diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 65fb288e..8058f9de 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -156,13 +156,13 @@ sequenceDiagram participant BackendConn as BackendConnection participant Downstream as MCP Backend - Client->>Router: POST /message?sessionId=1 (search_tools) - Router->>RoutingMgr: SearchToolsAsync("restart container") + Client->>Router: POST /message?sessionId=1 (search_tools: query, mode, dense_weight) + Router->>RoutingMgr: SearchToolsDetailedAsync("restart container", mode, denseWeight) RoutingMgr->>RoutingMgr: Compute Lexical Keyword Scoring (name, desc, tags, params) RoutingMgr->>VectorStore: SearchSimilarAsync(queryEmbedding) via .NET 10 SIMD VectorStore-->>RoutingMgr: Return Ranked Vector Similarities - RoutingMgr->>RoutingMgr: Fuse Ranks via Reciprocal Rank Fusion (RRF, k=60) - RoutingMgr-->>Router: Return Top Ranked Tools + RoutingMgr->>RoutingMgr: Calibrated Fusion: Score = alpha * DenseScore + (1 - alpha) * SparseScore + RoutingMgr-->>Router: Return Top Ranked Tools with Decomposed Scores Router-->>Client: Return namespaced search result JSON Client->>Router: POST /message?sessionId=1 (execute_tool: docker/restart_container) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ab2dfd..122d6da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ For summary details and quick references, see [README.md](README.md). | Version | Release Date | Summary of Key Changes | | :--- | :--- | :--- | +| **`v5.21.0`** | 2026-10-02 | feat(routing): calibrated hybrid semantic search scoring, explicit search modes, and test bench simulator overhaul | | **`v5.20.1`** | 2026-09-30 | refactor(testbench): Remove Pre-fill Example Button and Streamline Dynamic Argument Handling. Removed the legacy Pre-fill Example button and unused bulk argument modification plumbing from the Test Bench tool tester card, eliminating brittle and unhelpful parameter mocks while preserving interactive form validation and tool execution coverage. | | **`v5.20.0`** | 2026-09-30 | feat(auth,ui,docs,ci): AppKey & OAuth Client Last-Used Tracking, Safe Secret Rotation Workflow, Multi-Client Config Generator, VitePress Documentation Architecture Overhaul, and 30-Day Dependabot Stability Cooldown. Implemented `LastUsedAt` audit tracking for AppKeys and OAuth clients across SQLite, MySQL, and MSSQL with an in-memory 60-second sliding debounce `ActivityTracker`; added atomic credential rotation endpoints (`POST /api/appkeys/{id}/rotate`, `POST /api/clients/{id}/rotate-secret`) returning plaintext keys once with SHA-256 hashing at rest; built two-stage `AppKeyRotateModal` with backdrop click-away protection and multi-client configuration generator (Claude Desktop, Cursor, Windsurf, generic JSON); overhauled VitePress documentation architecture with DiΓ‘taxis decomposition, path-scoped multi-sidebars, purged legacy ASCII art to responsive Mermaid diagrams, modernized typography rhythm, and link integrity CI quality gates; updated GitHub Actions to v7/v5; and configured a 30-day stability cooldown across all Dependabot ecosystems (`AUTH-118` through `AUTH-123`, `UI-135` through `UI-137`). | | **`v5.19.1`** | 2026-09-29 | docs(slack): Purge Deprecated Wrapper References, Renumber Documentation Sections, and Refresh Wiki. Streamlined documentation for Slack Native Direct MCP (`https://mcp.slack.com/mcp`) and the Hybrid Egress Delegation Model; eliminated legacy self-hosted Node.js wrapper references across repository and wiki; updated release assets; and pinned container image tags. | diff --git a/Components/Capabilities/CapabilityEndpoints.cs b/Components/Capabilities/CapabilityEndpoints.cs index ae198f59..e197cb79 100644 --- a/Components/Capabilities/CapabilityEndpoints.cs +++ b/Components/Capabilities/CapabilityEndpoints.cs @@ -504,8 +504,27 @@ FROM AuditLogs } } - var scoredResults = await SemanticSearchService.SearchToolsSemanticAsync(model.Query, allTools, embeddingService, logger); - return Results.Ok(scoredResults); + var effectiveProvider = embeddingService != null ? new ModelContextGateway.Core.VectorSearch.EmbeddingServiceAdapter(embeddingService) : null; + var effectiveStore = httpContext.RequestServices.GetService(); + var routingManager = new ToolRoutingManager(effectiveProvider, effectiveStore); + var scoredResults = await routingManager.SearchToolsDetailedAsync( + model.Query ?? "", + allTools, + effectiveProvider, + effectiveStore, + logger, + model.Limit ?? 15, + searchMode: model.Mode ?? "hybrid", + denseWeight: model.DenseWeight ?? 0.5, + cancellationToken: httpContext.RequestAborted); + + return Results.Ok(new + { + query = model.Query ?? "", + mode = model.Mode ?? "hybrid", + denseWeight = model.DenseWeight ?? 0.5, + results = scoredResults + }); }); // 2b. Test Prompts List API diff --git a/Components/Capabilities/CapabilityModels.cs b/Components/Capabilities/CapabilityModels.cs index d8749274..d415cadf 100644 --- a/Components/Capabilities/CapabilityModels.cs +++ b/Components/Capabilities/CapabilityModels.cs @@ -38,5 +38,8 @@ public class TestResourceReadModel public class SearchModel { public string Query { get; set; } = string.Empty; + public string? Mode { get; set; } = "hybrid"; + public double? DenseWeight { get; set; } = 0.5; + public int? Limit { get; set; } = 15; } } diff --git a/Core/Routing/ToolRoutingManager.Execution.cs b/Core/Routing/ToolRoutingManager.Execution.cs index c8158df5..2f5e4688 100644 --- a/Core/Routing/ToolRoutingManager.Execution.cs +++ b/Core/Routing/ToolRoutingManager.Execution.cs @@ -67,11 +67,27 @@ private async Task CallToolInternalAsync( var root = doc.RootElement; string query = ""; + string mode = "hybrid"; + double denseWeight = 0.5; + if (root.TryGetProperty("params", out var paramsProp) && - paramsProp.TryGetProperty("arguments", out var argsProp) && - argsProp.TryGetProperty("query", out var queryProp)) + paramsProp.TryGetProperty("arguments", out var argsProp)) { - query = queryProp.GetString() ?? ""; + if (argsProp.TryGetProperty("query", out var queryProp)) + { + query = queryProp.GetString() ?? ""; + } + if (argsProp.TryGetProperty("mode", out var modeProp) && modeProp.ValueKind == JsonValueKind.String) + { + mode = modeProp.GetString() ?? "hybrid"; + } + if (argsProp.TryGetProperty("dense_weight", out var dwProp) || argsProp.TryGetProperty("denseWeight", out dwProp)) + { + if (dwProp.ValueKind == JsonValueKind.Number && dwProp.TryGetDouble(out var dwVal)) + { + denseWeight = dwVal; + } + } } var tools = new List(); @@ -192,9 +208,19 @@ private async Task CallToolInternalAsync( } var effectiveProvider = _embeddingProvider ?? (embeddingService != null ? new ModelContextGateway.Core.VectorSearch.EmbeddingServiceAdapter(embeddingService) : null); - var effectiveStore = _vectorStore ?? new ModelContextGateway.Core.VectorSearch.InMemorySimdToolVectorStore(); - var results = await SearchToolsAsync(query, tools, effectiveProvider, effectiveStore, logger, 15, cancellationToken); - var serialized = JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true }); + var effectiveStore = _vectorStore; + var detailedResults = await SearchToolsDetailedAsync( + query, + tools, + effectiveProvider, + effectiveStore, + logger, + 15, + searchMode: mode, + denseWeight: denseWeight, + cancellationToken: cancellationToken); + var toolSchemas = detailedResults.Select(r => r.Tool).ToList(); + var serialized = JsonSerializer.Serialize(toolSchemas, new JsonSerializerOptions { WriteIndented = true }); return new { resultType = "complete", diff --git a/Core/Routing/ToolRoutingManager.Search.cs b/Core/Routing/ToolRoutingManager.Search.cs index ed4fb890..19019f3a 100644 --- a/Core/Routing/ToolRoutingManager.Search.cs +++ b/Core/Routing/ToolRoutingManager.Search.cs @@ -5,12 +5,9 @@ namespace ModelContextGateway.Core.Routing { public partial class ToolRoutingManager { - private const double RrfK = 60.0; - /// - /// Asynchronously performs hybrid semantic search across candidate tools using lexical keyword scoring, - /// hardware-accelerated SIMD vector similarity, and Reciprocal Rank Fusion (RRF, k=60). - /// Gracefully falls back to pure keyword matching if the embedding provider is unconfigured, disabled, or offline. + /// Asynchronously performs calibrated hybrid semantic search across candidate tools, returning rich tool + /// search results with decomposed dense/sparse scores, ranks, and server metadata. /// /// The natural language intent query. /// Candidate tool list (defaults to cached tools if null). @@ -18,142 +15,258 @@ public partial class ToolRoutingManager /// Optional tool vector store override. /// Optional logger for telemetry and failure reporting. /// Maximum number of tool schemas to return. + /// Search mode: "hybrid", "semantic", or "lexical" (defaults to "hybrid"). + /// Weight alpha for dense similarity between 0.0 and 1.0 (defaults to 0.5). /// Cancellation token. - /// Ranked candidate tool schemas. - public async Task> SearchToolsAsync( + /// Ranked tool search results with decomposed diagnostics. + public async Task> SearchToolsDetailedAsync( string query, List? candidateTools = null, IEmbeddingProvider? embeddingProvider = null, IToolVectorStore? vectorStore = null, ILogger? logger = null, int limit = 15, + string searchMode = "hybrid", + double? denseWeight = 0.5, CancellationToken cancellationToken = default) { var tools = candidateTools ?? GetCachedTools(); if (tools.Count == 0) { - return new List(); + return new List(); } + var toolMetas = tools.Select(ExtractToolMetadata).ToList(); + if (string.IsNullOrWhiteSpace(query)) { - return tools.Take(limit).ToList(); + return toolMetas.Take(limit).Select(m => new ToolSearchResult + { + Tool = m.Tool, + ToolName = m.Name, + ServerId = ResolveServerId(m), + Score = 0.0, + DenseScore = null, + SparseScore = null, + DenseRank = null, + SparseRank = null + }).ToList(); } - // 1. Extract metadata from candidate tools - var toolMetas = tools.Select(ExtractToolMetadata).ToList(); - - // 2. Lexical / Keyword Scoring - var queryWords = TokenizeQuery(query); - var keywordScored = new List<(ToolMetadata Meta, double Score)>(); + var mode = searchMode?.Trim().ToLowerInvariant() ?? "hybrid"; + bool computeDense = mode == "hybrid" || mode == "semantic"; + bool computeSparse = mode == "hybrid" || mode == "lexical"; - foreach (var meta in toolMetas) + double alpha; + if (mode == "semantic") + { + alpha = 1.0; + } + else if (mode == "lexical") { - double score = CalculateLexicalScore(query, queryWords, meta); - if (score > 0) + alpha = 0.0; + } + else + { + double rawWeight = denseWeight ?? 0.5; + if (double.IsNaN(rawWeight) || double.IsInfinity(rawWeight)) { - keywordScored.Add((meta, score)); + rawWeight = 0.5; } + alpha = Math.Clamp(rawWeight, 0.0, 1.0); } - var keywordRanked = keywordScored - .OrderByDescending(x => x.Score) - .Select((item, index) => (item.Meta, item.Score, Rank: index + 1)) - .ToList(); + // 1. Lexical / Keyword Scoring + var rawLexicalScores = new Dictionary(); + var sparseRanks = new Dictionary(); + var normalizedSparseScores = new Dictionary(); - // 3. Vector Similarity Search (SIMD accelerated) - var effectiveProvider = embeddingProvider ?? _embeddingProvider; - var effectiveStore = vectorStore ?? _vectorStore; - - IReadOnlyList<(string ToolName, float Score)>? vectorMatches = null; - - if (effectiveProvider != null && !(effectiveProvider is NoOpEmbeddingProvider) && effectiveStore != null) + if (computeSparse) { - try + var queryWords = TokenizeQuery(query); + var keywordScored = new List<(ToolMetadata Meta, double Score)>(); + + foreach (var meta in toolMetas) { - float[]? queryEmbedding = null; - try + double score = CalculateLexicalScore(query, queryWords, meta); + rawLexicalScores[meta] = score; + if (score > 0) { - queryEmbedding = await effectiveProvider.GenerateEmbeddingAsync(query, cancellationToken); - } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to generate query embedding for '{Query}'. Falling back to keyword ranking.", query); + keywordScored.Add((meta, score)); } + } - if (queryEmbedding != null && queryEmbedding.Length > 0) - { - // Ensure tools are indexed in the vector store - await EnsureToolsIndexedAsync(toolMetas, effectiveProvider, effectiveStore, logger, cancellationToken); + var keywordRanked = keywordScored + .OrderByDescending(x => x.Score) + .ThenBy(x => x.Meta.Name, StringComparer.Ordinal) + .ToList(); - vectorMatches = await effectiveStore.SearchSimilarAsync(queryEmbedding, limit: 50, cancellationToken: cancellationToken); + for (int i = 0; i < keywordRanked.Count; i++) + { + sparseRanks[keywordRanked[i].Meta] = i + 1; + } + + if (keywordScored.Count > 0) + { + double maxLexical = keywordScored.Max(x => x.Score); + + foreach (var meta in toolMetas) + { + double raw = rawLexicalScores[meta]; + if (raw <= 0) + { + normalizedSparseScores[meta] = 0.0; + } + else if (maxLexical <= 0.0) + { + normalizedSparseScores[meta] = 1.0; + } + else + { + normalizedSparseScores[meta] = Math.Clamp(raw / maxLexical, 0.0, 1.0); + } } } - catch (Exception ex) + else { - logger?.LogWarning(ex, "Vector search encountered an error. Falling back gracefully to keyword ranking."); - vectorMatches = null; + foreach (var meta in toolMetas) + { + normalizedSparseScores[meta] = 0.0; + } } } - // 4. Reciprocal Rank Fusion (RRF, k=60) - if (vectorMatches != null && vectorMatches.Count > 0) - { - var vectorRanked = vectorMatches - .Where(m => !float.IsNaN(m.Score) && m.Score > 0f) - .Select((item, index) => (item.ToolName, item.Score, Rank: index + 1)) - .ToList(); + // 2. Vector Similarity Search (SIMD accelerated) + var denseScores = new Dictionary(); + var denseRanks = new Dictionary(); - var rrfScores = new Dictionary(); - var toolMetaLookup = toolMetas.ToDictionary(m => m.Tool, m => m); + if (computeDense) + { + var effectiveProvider = embeddingProvider ?? _embeddingProvider; + var effectiveStore = vectorStore ?? _vectorStore; - // Add reciprocal rank for keyword results - foreach (var kw in keywordRanked) - { - double rrf = 1.0 / (RrfK + kw.Rank); - rrfScores[kw.Meta.Tool] = rrf; - } + IReadOnlyList<(string ToolName, float Score)>? vectorMatches = null; - // Add reciprocal rank for vector results - foreach (var vec in vectorRanked) + if (effectiveProvider != null && !(effectiveProvider is NoOpEmbeddingProvider) && effectiveStore != null) { - var matched = FindMatchingToolMeta(vec.ToolName, toolMetas); - if (matched != null) + try { - double rrf = 1.0 / (RrfK + vec.Rank); - if (rrfScores.TryGetValue(matched.Tool, out var existing)) + float[]? queryEmbedding = null; + try { - rrfScores[matched.Tool] = existing + rrf; + queryEmbedding = await effectiveProvider.GenerateEmbeddingAsync(query, cancellationToken); } - else + catch (Exception ex) { - rrfScores[matched.Tool] = rrf; + logger?.LogWarning(ex, "Failed to generate query embedding for '{Query}'. Falling back to keyword ranking.", query); } + + if (queryEmbedding != null && queryEmbedding.Length > 0) + { + await EnsureToolsIndexedAsync(toolMetas, effectiveProvider, effectiveStore, logger, cancellationToken); + vectorMatches = await effectiveStore.SearchSimilarAsync(queryEmbedding, limit: 50, cancellationToken: cancellationToken); + } + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Vector search encountered an error. Falling back gracefully to keyword ranking."); + vectorMatches = null; } } - if (rrfScores.Count > 0) + if (vectorMatches != null && vectorMatches.Count > 0) { - var fused = rrfScores - .OrderByDescending(kvp => kvp.Value) - .Select(kvp => kvp.Key) - .Take(limit) + var vectorRanked = vectorMatches + .Where(m => !float.IsNaN(m.Score) && m.Score > 0f) .ToList(); - return fused; + for (int i = 0; i < vectorRanked.Count; i++) + { + var vec = vectorRanked[i]; + var matched = FindMatchingToolMeta(vec.ToolName, toolMetas); + if (matched != null && !denseScores.ContainsKey(matched)) + { + denseScores[matched] = Math.Clamp((double)vec.Score, 0.0, 1.0); + denseRanks[matched] = i + 1; + } + } + } + else if (mode == "hybrid" && (effectiveProvider == null || effectiveProvider is NoOpEmbeddingProvider || effectiveStore == null)) + { + // Graceful fallback to pure keyword scoring if vector search is unconfigured or disabled + alpha = 0.0; } } - // 5. Graceful Fallback: Pure Keyword Ranking - if (keywordRanked.Count > 0) + // 3. Fused Linear Combination Scoring + var searchResults = new List(toolMetas.Count); + + foreach (var meta in toolMetas) { - return keywordRanked - .Select(x => x.Meta.Tool) - .Take(limit) - .ToList(); + double? denseScore = denseScores.TryGetValue(meta, out var ds) ? ds : null; + int? denseRank = denseRanks.TryGetValue(meta, out var dr) ? dr : null; + + double? sparseScore = normalizedSparseScores.TryGetValue(meta, out var ss) ? ss : null; + int? sparseRank = sparseRanks.TryGetValue(meta, out var sr) ? sr : null; + + double combinedScore = (alpha * (denseScore ?? 0.0)) + ((1.0 - alpha) * (sparseScore ?? 0.0)); + + searchResults.Add(new ToolSearchResult + { + Tool = meta.Tool, + ToolName = meta.Name, + ServerId = ResolveServerId(meta), + Score = combinedScore, + DenseScore = denseScore, + SparseScore = sparseScore, + DenseRank = denseRank, + SparseRank = sparseRank + }); } - return tools.Take(Math.Min(10, limit)).ToList(); + return searchResults + .OrderByDescending(r => r.Score) + .ThenBy(r => r.SparseRank ?? int.MaxValue) + .ThenBy(r => r.DenseRank ?? int.MaxValue) + .ThenBy(r => r.ToolName, StringComparer.Ordinal) + .Take(limit) + .ToList(); + } + + /// + /// Asynchronously performs hybrid semantic search across candidate tools using calibrated linear combination + /// of dense vector similarity and sparse keyword scoring. + /// Gracefully falls back to pure keyword matching if the embedding provider is unconfigured, disabled, or offline. + /// + /// The natural language intent query. + /// Candidate tool list (defaults to cached tools if null). + /// Optional embedding provider override. + /// Optional tool vector store override. + /// Optional logger for telemetry and failure reporting. + /// Maximum number of tool schemas to return. + /// Cancellation token. + /// Ranked candidate tool schemas. + public async Task> SearchToolsAsync( + string query, + List? candidateTools = null, + IEmbeddingProvider? embeddingProvider = null, + IToolVectorStore? vectorStore = null, + ILogger? logger = null, + int limit = 15, + CancellationToken cancellationToken = default) + { + var detailedResults = await SearchToolsDetailedAsync( + query, + candidateTools, + embeddingProvider, + vectorStore, + logger, + limit, + searchMode: "hybrid", + denseWeight: 0.5, + cancellationToken: cancellationToken); + + return detailedResults.Select(r => r.Tool).ToList(); } /// @@ -352,6 +465,7 @@ private static ToolMetadata ExtractToolMetadata(object tool) { string name = ""; string description = ""; + string? serverId = null; var tags = new List(); var parameters = new List<(string Name, string Description)>(); @@ -365,14 +479,18 @@ private static ToolMetadata ExtractToolMetadata(object tool) { description = d.GetString() ?? ""; } + if (je.TryGetProperty("serverId", out var s) || je.TryGetProperty("server_id", out s)) + { + serverId = s.GetString(); + } if (je.TryGetProperty("tags", out var t) && t.ValueKind == JsonValueKind.Array) { foreach (var tag in t.EnumerateArray()) { - var s = tag.GetString(); - if (!string.IsNullOrEmpty(s)) + var str = tag.GetString(); + if (!string.IsNullOrEmpty(str)) { - tags.Add(s); + tags.Add(str); } } } @@ -397,6 +515,10 @@ private static ToolMetadata ExtractToolMetadata(object tool) { description = d?.ToString() ?? ""; } + if (dict.TryGetValue("serverId", out var s) || dict.TryGetValue("server_id", out s)) + { + serverId = s?.ToString(); + } if (dict.TryGetValue("tags", out var t)) { if (t is IEnumerable strSeq) @@ -407,10 +529,10 @@ private static ToolMetadata ExtractToolMetadata(object tool) { foreach (var tag in jeTags.EnumerateArray()) { - var s = tag.GetString(); - if (!string.IsNullOrEmpty(s)) + var str = tag.GetString(); + if (!string.IsNullOrEmpty(str)) { - tags.Add(s); + tags.Add(str); } } } @@ -431,6 +553,14 @@ private static ToolMetadata ExtractToolMetadata(object tool) { description = legacyDict["description"]?.ToString() ?? ""; } + if (legacyDict.Contains("serverId")) + { + serverId = legacyDict["serverId"]?.ToString(); + } + else if (legacyDict.Contains("server_id")) + { + serverId = legacyDict["server_id"]?.ToString(); + } } else { @@ -439,6 +569,8 @@ private static ToolMetadata ExtractToolMetadata(object tool) type.GetProperty("Name")?.GetValue(tool)?.ToString() ?? ""; description = type.GetProperty("description")?.GetValue(tool)?.ToString() ?? type.GetProperty("Description")?.GetValue(tool)?.ToString() ?? ""; + serverId = type.GetProperty("ServerId")?.GetValue(tool)?.ToString() ?? + type.GetProperty("serverId")?.GetValue(tool)?.ToString(); } var textToEmbed = $"{name}: {description}"; @@ -447,7 +579,41 @@ private static ToolMetadata ExtractToolMetadata(object tool) textToEmbed += " " + string.Join(" ", parameters.Select(p => $"{p.Name} {p.Description}")); } - return new ToolMetadata(tool, name, description, tags, parameters, textToEmbed); + return new ToolMetadata(tool, name, description, tags, parameters, textToEmbed, serverId); + } + + private string? ResolveServerId(ToolMetadata meta) + { + if (!string.IsNullOrWhiteSpace(meta.ServerId)) + { + return meta.ServerId; + } + + if (!string.IsNullOrWhiteSpace(meta.Name)) + { + if (_toolRoutingTable.TryGetValue(meta.Name, out var mappedId)) + { + return mappedId; + } + + var slashIdx = meta.Name.IndexOf('/'); + if (slashIdx > 0) + { + return meta.Name.Substring(0, slashIdx); + } + var underIdx = meta.Name.IndexOf("__", StringComparison.Ordinal); + if (underIdx > 0) + { + return meta.Name.Substring(0, underIdx); + } + var colonIdx = meta.Name.IndexOf(':'); + if (colonIdx > 0) + { + return meta.Name.Substring(0, colonIdx); + } + } + + return null; } private static void ExtractParametersFromSchemaObject(object? schemaObj, List<(string Name, string Description)> parameters) @@ -484,6 +650,7 @@ private record ToolMetadata( string Description, List Tags, List<(string Name, string Description)> Parameters, - string TextToEmbed); + string TextToEmbed, + string? ServerId = null); } } diff --git a/Core/Routing/ToolRoutingManager.cs b/Core/Routing/ToolRoutingManager.cs index 10821177..86322fb7 100644 --- a/Core/Routing/ToolRoutingManager.cs +++ b/Core/Routing/ToolRoutingManager.cs @@ -58,7 +58,9 @@ public static List GetMetaModeTools() type = "object", properties = new { - query = new { type = "string", description = "The natural language query describing what you want to do (e.g. 'read Excel file data', 'restart Docker container')." } + query = new { type = "string", description = "The natural language query or intent to find relevant tools." }, + mode = new { type = "string", @enum = new[] { "hybrid", "semantic", "lexical" }, description = "Search mode: 'hybrid' (default), 'semantic' (pure vector), or 'lexical' (pure keyword)." }, + dense_weight = new { type = "number", description = "Weight between 0.0 (pure lexical) and 1.0 (pure semantic). Default is 0.5 (balanced)." } }, required = new[] { "query" } } diff --git a/Core/Routing/ToolSearchResult.cs b/Core/Routing/ToolSearchResult.cs new file mode 100644 index 00000000..a77c2ea0 --- /dev/null +++ b/Core/Routing/ToolSearchResult.cs @@ -0,0 +1,48 @@ +namespace ModelContextGateway.Core.Routing +{ + /// + /// Represents a ranked tool search result with decomposed dense/sparse scores and ranking diagnostics. + /// + public class ToolSearchResult + { + /// + /// The raw underlying tool object or dictionary. + /// + public required object Tool { get; init; } + + /// + /// Resolved exposed tool identifier. + /// + public required string ToolName { get; init; } + + /// + /// Upstream server identifier if namespaced or resolvable. + /// + public string? ServerId { get; init; } + + /// + /// Calibrated fused score between 0.0 and 1.0. + /// + public double Score { get; init; } + + /// + /// Clamped dense vector cosine similarity score between 0.0 and 1.0 (null if dense search not executed or no match). + /// + public double? DenseScore { get; init; } + + /// + /// Min-max normalized sparse lexical keyword score between 0.0 and 1.0 (null if sparse search not executed). + /// + public double? SparseScore { get; init; } + + /// + /// 1-based rank among dense vector search hits. + /// + public int? DenseRank { get; init; } + + /// + /// 1-based rank among sparse lexical keyword hits. + /// + public int? SparseRank { get; init; } + } +} diff --git a/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs b/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs new file mode 100644 index 00000000..9fc66e54 --- /dev/null +++ b/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs @@ -0,0 +1,286 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Dapper; +using FluentAssertions; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Moq; + +namespace ModelContextGateway.Tests +{ + public class SemanticSearchEndpointTests : IClassFixture + { + private readonly PipelineIntegrationFactory _factory; + + public SemanticSearchEndpointTests(PipelineIntegrationFactory factory) + { + _factory = factory; + } + + private static (SqliteConnection conn, IDbConnectionFactory factory) CreateDbFactory() + { + var connection = new SqliteConnection("Filename=:memory:"); + connection.Open(); + connection.Execute("CREATE TABLE IF NOT EXISTS Settings (Id TEXT PRIMARY KEY);"); + + var mockDbFactory = new Mock(); + mockDbFactory.Setup(f => f.CreateConnection()).Returns(connection); + mockDbFactory.Setup(f => f.ProviderName).Returns("sqlite"); + return (connection, mockDbFactory.Object); + } + + private HttpClient CreateAuthenticatedClient() + { + var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "127.0.0.1"); + client.DefaultRequestHeaders.Add("Remote-User", "admin_user"); + client.DefaultRequestHeaders.Add("Remote-Groups", "full_admin"); + return client; + } + + [Fact] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task SemanticSearchEndpoint_ReturnsCalibratedHybridResultsWithDiagnostics() + { + var sessionManager = _factory.Services.GetRequiredService(); + var tool1 = new Dictionary + { + ["name"] = "docker__restart_container", + ["description"] = "Restart an active docker container service", + ["inputSchema"] = new Dictionary + { + ["type"] = "object", + ["properties"] = new Dictionary + { + ["container_id"] = new Dictionary { ["type"] = "string" } + } + } + }; + var tool2 = new Dictionary + { + ["name"] = "docker__inspect_container", + ["description"] = "Inspect container attributes", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + var tool3 = new Dictionary + { + ["name"] = "media__get_status", + ["description"] = "Get status of active media transcoder", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + sessionManager.SetServerToolsCache("docker", new List { tool1, tool2 }); + sessionManager.SetServerToolsCache("media", new List { tool3 }); + + var client = CreateAuthenticatedClient(); + var payload = new + { + query = "restart container", + mode = "hybrid", + denseWeight = 0.6, + limit = 10 + }; + + var response = await client.PostAsJsonAsync("/api/test/semantic-search", payload); + response.StatusCode.Should().Be(HttpStatusCode.OK); + + using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync()); + var root = doc.RootElement; + + // Verify top-level envelope + root.GetProperty("query").GetString().Should().Be("restart container"); + root.GetProperty("mode").GetString().Should().Be("hybrid"); + root.GetProperty("denseWeight").GetDouble().Should().BeApproximately(0.6, 0.001); + + // Verify decomposed score diagnostics in results + var results = root.GetProperty("results"); + results.GetArrayLength().Should().Be(3); + + var first = results[0]; + first.GetProperty("toolName").GetString().Should().Be("docker__restart_container"); + first.GetProperty("serverId").GetString().Should().Be("docker"); + + // Validate real numerical bounds and mathematical consistency + var score = first.GetProperty("score").GetDouble(); + score.Should().BeInRange(0.0, 1.0); + score.Should().BeGreaterThan(0.0); + + var sparseScore = first.GetProperty("sparseScore").GetDouble(); + sparseScore.Should().BeInRange(0.0, 1.0); + sparseScore.Should().BeApproximately(1.0, 0.01); // Top keyword hit + + // In absence of configured vector store in mock DI, denseScore is null and score equals sparseScore + if (first.TryGetProperty("denseScore", out var denseProp) && denseProp.ValueKind != JsonValueKind.Null) + { + var denseScore = denseProp.GetDouble(); + denseScore.Should().BeInRange(0.0, 1.0); + score.Should().BeApproximately(0.6 * denseScore + 0.4 * sparseScore, 0.01); + } + else + { + score.Should().BeApproximately(sparseScore, 0.01); + } + } + + [Fact] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task SemanticSearchEndpoint_RespectsSearchModeAndDenseWeightParameters() + { + var sessionManager = _factory.Services.GetRequiredService(); + var tool1 = new Dictionary + { + ["name"] = "media__scan_library", + ["description"] = "Trigger media refresh and index library sections", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + var tool2 = new Dictionary + { + ["name"] = "docker__list_containers", + ["description"] = "List active running containers", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + var tool3 = new Dictionary + { + ["name"] = "media__reboot_server", + ["description"] = "Bounce physical server host", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + sessionManager.SetServerToolsCache("media", new List { tool1, tool3 }); + sessionManager.SetServerToolsCache("docker", new List { tool2 }); + + var client = CreateAuthenticatedClient(); + var payload = new + { + query = "scan library", + mode = "lexical", + denseWeight = 0.0, + limit = 5 + }; + + var response = await client.PostAsJsonAsync("/api/test/semantic-search", payload); + response.StatusCode.Should().Be(HttpStatusCode.OK); + + using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync()); + var root = doc.RootElement; + + root.GetProperty("mode").GetString().Should().Be("lexical"); + root.GetProperty("denseWeight").GetDouble().Should().Be(0.0); + var results = root.GetProperty("results"); + results.GetArrayLength().Should().Be(3); + + // In lexical mode, media__scan_library MUST rank first with sparseScore = 1.0 and denseScore = null + var first = results[0]; + first.GetProperty("toolName").GetString().Should().Be("media__scan_library"); + first.GetProperty("denseScore").ValueKind.Should().Be(JsonValueKind.Null); + first.GetProperty("sparseScore").GetDouble().Should().BeApproximately(1.0, 0.01); + first.GetProperty("score").GetDouble().Should().BeApproximately(1.0, 0.01); + + // Other tools with zero keyword matches must have sparseScore = 0.0 and score = 0.0 + var second = results[1]; + second.GetProperty("sparseScore").GetDouble().Should().Be(0.0); + second.GetProperty("score").GetDouble().Should().Be(0.0); + } + + [Fact] + [Requirement("MCP-42", "MCP", RequirementType.Positive, "Meta-mode search_tools MCP tool accepts optional mode and dense_weight arguments and executes calibrated tool retrieval.")] + public void MetaModeTools_SearchToolsSchema_ExposesModeAndDenseWeight() + { + var metaTools = ToolRoutingManager.GetMetaModeTools(); + var searchToolsObj = metaTools.FirstOrDefault(t => + { + var json = JsonSerializer.Serialize(t); + using var doc = JsonDocument.Parse(json); + return doc.RootElement.TryGetProperty("name", out var n) && n.GetString() == "search_tools"; + }); + + searchToolsObj.Should().NotBeNull(); + var toolJson = JsonSerializer.Serialize(searchToolsObj); + using var toolDoc = JsonDocument.Parse(toolJson); + var props = toolDoc.RootElement.GetProperty("inputSchema").GetProperty("properties"); + + // Verify query property + props.TryGetProperty("query", out var queryProp).Should().BeTrue(); + queryProp.GetProperty("type").GetString().Should().Be("string"); + + // Verify mode property with enum + props.TryGetProperty("mode", out var modeProp).Should().BeTrue(); + modeProp.GetProperty("type").GetString().Should().Be("string"); + var enumVals = modeProp.GetProperty("enum").EnumerateArray().Select(e => e.GetString()).ToList(); + enumVals.Should().Contain(new[] { "hybrid", "semantic", "lexical" }); + + // Verify dense_weight property + props.TryGetProperty("dense_weight", out var weightProp).Should().BeTrue(); + weightProp.GetProperty("type").GetString().Should().Be("number"); + } + + [Fact] + [Requirement("MCP-42", "MCP", RequirementType.Positive, "Meta-mode search_tools MCP tool accepts optional mode and dense_weight arguments and executes calibrated tool retrieval.")] + public async Task SearchTools_CustomToolCall_ExecutesWithModeAndDenseWeight() + { + var manager = new ToolRoutingManager(); + var (conn, dbFactory) = CreateDbFactory(); + var connections = new ConcurrentDictionary(); + var servers = new List + { + new McpServer { Id = "docker", Enabled = true } + }; + + var mockEmbedding = new Mock(); + mockEmbedding.Setup(e => e.GetEmbeddingAsync(It.IsAny())) + .ReturnsAsync(new float[128]); + + var sessionManager = _factory.Services.GetRequiredService(); + var tool1 = new Dictionary + { + ["name"] = "docker__list_containers", + ["description"] = "List active running containers", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + var tool2 = new Dictionary + { + ["name"] = "docker__prune_volumes", + ["description"] = "Remove all unused local volumes", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + var tool3 = new Dictionary + { + ["name"] = "weather__forecast", + ["description"] = "Retrieve meteorology forecast", + ["inputSchema"] = new Dictionary { ["type"] = "object" } + }; + sessionManager.SetServerToolsCache("docker", new List { tool1, tool2, tool3 }); + + // Call search_tools with explicit lexical mode and dense_weight + var body = "{\"params\":{\"arguments\":{\"query\":\"list containers\",\"mode\":\"lexical\",\"dense_weight\":0.0}}}"; + var result = await manager.CallToolAsync( + "search_tools", + body, + dbFactory, + connections, + servers, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + new HttpClient(), + mockEmbedding.Object, + () => Task.CompletedTask, + (b, k, v) => b, + sessionManager: sessionManager + ); + + result.Should().NotBeNull(); + var resultJson = JsonSerializer.Serialize(result); + using var doc = JsonDocument.Parse(resultJson); + doc.RootElement.GetProperty("resultType").GetString().Should().Be("complete"); + var text = doc.RootElement.GetProperty("content")[0].GetProperty("text").GetString(); + text.Should().NotBeNull(); + + // Prove differential ranking: matching tool must rank #1 + var toolSchemas = JsonSerializer.Deserialize>(text!); + toolSchemas.Should().NotBeNull(); + toolSchemas![0].GetProperty("name").GetString().Should().Be("docker__list_containers"); + + // Prove lexical mode bypassed embedding generation entirely (anti-theater) + mockEmbedding.Verify(e => e.GetEmbeddingAsync(It.IsAny()), Times.Never); + } + } +} diff --git a/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs b/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs index a20002a3..85112617 100644 --- a/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs +++ b/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs @@ -37,7 +37,7 @@ public Task> GenerateEmbeddingsAsync(IReadOnlyList)results[0])["name"].Should().Be("git__push"); } + + [Fact] + [Requirement("MCP-40", "MCP", RequirementType.Positive, "Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination.")] + public async Task HybridSearch_WithBalancedWeight_ComputesLinearCombinationScore() + { + var provider = new MockDeterministicEmbeddingProvider(); + var vectorStore = new InMemorySimdToolVectorStore(); + + var tool1 = new Dictionary + { + ["name"] = "db__backup", + ["description"] = "Create database backup and snapshot" + }; + var tool2 = new Dictionary + { + ["name"] = "pg__dump", + ["description"] = "Export postgres sql archive" + }; + + var candidates = new List { tool1, tool2 }; + + provider.SetEmbedding("database backup", new float[] { 1.0f, 0.0f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("db__backup", new float[] { 0.4f, 0.9165f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("pg__dump", new float[] { 0.9f, 0.4359f, 0.0f }); + + var manager = new ToolRoutingManager(provider, vectorStore); + + var results = await manager.SearchToolsDetailedAsync("database backup", candidates, searchMode: "hybrid", denseWeight: 0.5); + + results.Should().HaveCount(2); + + results[0].ToolName.Should().Be("db__backup"); + results[0].Score.Should().BeApproximately(0.70, 0.02); + results[0].DenseScore.Should().BeApproximately(0.40, 0.02); + results[0].SparseScore.Should().BeApproximately(1.0, 0.01); + results[0].DenseRank.Should().Be(2); + results[0].SparseRank.Should().Be(1); + + results[1].ToolName.Should().Be("pg__dump"); + results[1].Score.Should().BeApproximately(0.45, 0.02); + results[1].DenseScore.Should().BeApproximately(0.90, 0.02); + results[1].SparseScore.Should().BeApproximately(0.0, 0.01); + results[1].DenseRank.Should().Be(1); + results[1].SparseRank.Should().BeNull(); + } + + [Fact] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task HybridSearch_WithSemanticMode_OnlyScoresDenseSimilarity() + { + var provider = new MockDeterministicEmbeddingProvider(); + var vectorStore = new InMemorySimdToolVectorStore(); + + var tool1 = new Dictionary + { + ["name"] = "db__backup", + ["description"] = "Create database backup and snapshot" + }; + var tool2 = new Dictionary + { + ["name"] = "pg__dump", + ["description"] = "Export postgres database archive" + }; + + var candidates = new List { tool1, tool2 }; + + provider.SetEmbedding("database backup", new float[] { 1.0f, 0.0f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("db__backup", new float[] { 0.4f, 0.9165f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("pg__dump", new float[] { 0.9f, 0.4359f, 0.0f }); + + var manager = new ToolRoutingManager(provider, vectorStore); + + var results = await manager.SearchToolsDetailedAsync("database backup", candidates, searchMode: "semantic"); + + results.Should().HaveCount(2); + + results[0].ToolName.Should().Be("pg__dump"); + results[0].Score.Should().BeApproximately(0.90, 0.02); + results[0].DenseScore.Should().BeApproximately(0.90, 0.02); + results[0].SparseScore.Should().BeNull(); + results[0].SparseRank.Should().BeNull(); + results[0].DenseRank.Should().Be(1); + + results[1].ToolName.Should().Be("db__backup"); + results[1].Score.Should().BeApproximately(0.40, 0.02); + results[1].DenseScore.Should().BeApproximately(0.40, 0.02); + results[1].SparseScore.Should().BeNull(); + results[1].SparseRank.Should().BeNull(); + results[1].DenseRank.Should().Be(2); + } + + [Fact] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task HybridSearch_WithLexicalMode_OnlyScoresLexicalMatch() + { + var provider = new MockDeterministicEmbeddingProvider(); + var vectorStore = new InMemorySimdToolVectorStore(); + + var tool1 = new Dictionary + { + ["name"] = "db__backup", + ["description"] = "Create database backup and snapshot" + }; + var tool2 = new Dictionary + { + ["name"] = "pg__dump", + ["description"] = "Export postgres sql archive" + }; + + var candidates = new List { tool1, tool2 }; + + provider.SetEmbedding("database backup", new float[] { 1.0f, 0.0f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("db__backup", new float[] { 0.4f, 0.9165f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("pg__dump", new float[] { 0.9f, 0.4359f, 0.0f }); + + var manager = new ToolRoutingManager(provider, vectorStore); + + var results = await manager.SearchToolsDetailedAsync("database backup", candidates, searchMode: "lexical"); + + results.Should().HaveCount(2); + + results[0].ToolName.Should().Be("db__backup"); + results[0].Score.Should().BeApproximately(1.0, 0.01); + results[0].SparseScore.Should().BeApproximately(1.0, 0.01); + results[0].DenseScore.Should().BeNull(); + results[0].DenseRank.Should().BeNull(); + results[0].SparseRank.Should().Be(1); + + results[1].ToolName.Should().Be("pg__dump"); + results[1].Score.Should().BeApproximately(0.0, 0.01); + results[1].SparseScore.Should().BeApproximately(0.0, 0.01); + results[1].DenseScore.Should().BeNull(); + results[1].DenseRank.Should().BeNull(); + results[1].SparseRank.Should().BeNull(); + } + + [Fact] + [Requirement("MCP-40", "MCP", RequirementType.Positive, "Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination.")] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task HybridSearch_ReturnsDecomposedScores() + { + var provider = new MockDeterministicEmbeddingProvider(); + var vectorStore = new InMemorySimdToolVectorStore(); + + var tool1 = new Dictionary + { + ["name"] = "docker__restart", + ["description"] = "Restart docker container" + }; + var tool2 = new Dictionary + { + ["name"] = "docker__stop", + ["description"] = "Stop docker container" + }; + var tool3 = new Dictionary + { + ["name"] = "plex__search", + ["description"] = "Find movies and music" + }; + + var candidates = new List { tool1, tool2, tool3 }; + + provider.SetEmbedding("restart container", new float[] { 1.0f, 0.0f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("docker__restart", new float[] { 0.9f, 0.4359f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("docker__stop", new float[] { 0.6f, 0.8f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("plex__search", new float[] { 0.0f, 1.0f, 0.0f }); + + var manager = new ToolRoutingManager(provider, vectorStore); + + var results = await manager.SearchToolsDetailedAsync("restart container", candidates, searchMode: "hybrid", denseWeight: 0.7); + + results.Should().NotBeEmpty(); + + foreach (var item in results) + { + item.Tool.Should().NotBeNull(); + item.ToolName.Should().NotBeNullOrEmpty(); + item.ServerId.Should().NotBeNullOrEmpty(); + item.Score.Should().BeInRange(0.0, 1.0); + + if (item.DenseScore.HasValue) + { + item.DenseScore.Value.Should().BeInRange(0.0, 1.0); + item.DenseRank.Should().BePositive(); + } + + if (item.SparseScore.HasValue) + { + item.SparseScore.Value.Should().BeInRange(0.0, 1.0); + } + + double expectedScore = (0.7 * (item.DenseScore ?? 0.0)) + (0.3 * (item.SparseScore ?? 0.0)); + item.Score.Should().BeApproximately(expectedScore, 0.001); + } + + results[0].ServerId.Should().Be("docker"); + } + + [Fact] + [Requirement("MCP-40", "MCP", RequirementType.Positive, "Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination.")] + public async Task HybridSearch_WithClampedDenseWeight_HandlesNegativeExcessiveAndNaN() + { + var provider = new MockDeterministicEmbeddingProvider(); + var vectorStore = new InMemorySimdToolVectorStore(); + + var tool = new Dictionary + { + ["name"] = "docker__start", + ["description"] = "Start existing container" + }; + var candidates = new List { tool }; + + provider.SetEmbedding("start", new float[] { 1.0f, 0.0f, 0.0f }); + await vectorStore.UpsertToolEmbeddingAsync("docker__start", new float[] { 0.8f, 0.6f, 0.0f }); + + var manager = new ToolRoutingManager(provider, vectorStore); + + // Negative dense weight (-2.5) -> clamped to 0.0 (pure lexical) + var negResults = await manager.SearchToolsDetailedAsync("start", candidates, denseWeight: -2.5); + negResults.Should().HaveCount(1); + negResults[0].Score.Should().BeApproximately(negResults[0].SparseScore!.Value, 0.001); + + // Excessive dense weight (3.5) -> clamped to 1.0 (pure dense) + var excessResults = await manager.SearchToolsDetailedAsync("start", candidates, denseWeight: 3.5); + excessResults.Should().HaveCount(1); + excessResults[0].Score.Should().BeApproximately(excessResults[0].DenseScore!.Value, 0.001); + + // NaN dense weight -> fallback to default 0.5 (balanced 50/50) + var nanResults = await manager.SearchToolsDetailedAsync("start", candidates, denseWeight: double.NaN); + nanResults.Should().HaveCount(1); + double expectedBalanced = 0.5 * nanResults[0].DenseScore!.Value + 0.5 * nanResults[0].SparseScore!.Value; + nanResults[0].Score.Should().BeApproximately(expectedBalanced, 0.001); + } + + [Fact] + [Requirement("MCP-40", "MCP", RequirementType.Positive, "Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination.")] + public async Task HybridSearch_WithIdenticalLexicalScores_NormalizesWithoutDivideByZero() + { + var provider = new MockDeterministicEmbeddingProvider(); + var vectorStore = new InMemorySimdToolVectorStore(); + + // Two tools that match the query with the exact same lexical signal + var toolA = new Dictionary + { + ["name"] = "alpha__deploy", + ["description"] = "Deploy deployment package" + }; + var toolB = new Dictionary + { + ["name"] = "beta__deploy", + ["description"] = "Deploy deployment package" + }; + var candidates = new List { toolB, toolA }; // intentionally out of alphabetical order + + var manager = new ToolRoutingManager(provider, vectorStore); + + var results = await manager.SearchToolsDetailedAsync("deploy", candidates, searchMode: "lexical"); + + results.Should().HaveCount(2); + // Both tools should have raw / maxLexical = 1.0 (not crushed or NaN) + results[0].SparseScore.Should().Be(1.0); + results[1].SparseScore.Should().Be(1.0); + results[0].Score.Should().Be(1.0); + results[1].Score.Should().Be(1.0); + + // Deterministic secondary sort: alpha__deploy before beta__deploy + results[0].ToolName.Should().Be("alpha__deploy"); + results[1].ToolName.Should().Be("beta__deploy"); + } + + [Fact] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task HybridSearch_WithEmptyQueryOrEmptyTools_ReturnsSafeGracefulList() + { + var manager = new ToolRoutingManager(); + + // Empty candidate list -> returns empty list gracefully + var emptyToolResults = await manager.SearchToolsDetailedAsync("restart", new List()); + emptyToolResults.Should().BeEmpty(); + + // Empty query string -> returns unranked candidate tools with 0.0 score and null subscores + var tools = new List + { + new Dictionary { ["name"] = "docker__ps", ["description"] = "List containers" } + }; + var emptyQueryResults = await manager.SearchToolsDetailedAsync("", tools); + emptyQueryResults.Should().HaveCount(1); + emptyQueryResults[0].ToolName.Should().Be("docker__ps"); + emptyQueryResults[0].ServerId.Should().Be("docker"); + emptyQueryResults[0].Score.Should().Be(0.0); + emptyQueryResults[0].DenseScore.Should().BeNull(); + emptyQueryResults[0].SparseScore.Should().BeNull(); + } + + [Fact] + [Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")] + public async Task HybridSearch_ResolveServerId_ResolvesVariousConventions() + { + var manager = new ToolRoutingManager(); + + var toolSlash = new Dictionary { ["name"] = "github/create_issue", ["description"] = "Create GitHub issue" }; + var toolUnderscore = new Dictionary { ["name"] = "docker__run", ["description"] = "Run container" }; + var toolColon = new Dictionary { ["name"] = "k8s:get_pods", ["description"] = "Get kubernetes pods" }; + var toolExplicit = new Dictionary { ["name"] = "custom_tool", ["description"] = "Custom server tool", ["serverId"] = "custom_srv" }; + + var candidates = new List { toolSlash, toolUnderscore, toolColon, toolExplicit }; + + var results = await manager.SearchToolsDetailedAsync("issue run pods custom", candidates, searchMode: "lexical"); + + results.Should().HaveCount(4); + results.First(r => r.ToolName == "github/create_issue").ServerId.Should().Be("github"); + results.First(r => r.ToolName == "docker__run").ServerId.Should().Be("docker"); + results.First(r => r.ToolName == "k8s:get_pods").ServerId.Should().Be("k8s"); + results.First(r => r.ToolName == "custom_tool").ServerId.Should().Be("custom_srv"); + } } } diff --git a/ModelContextGateway.csproj b/ModelContextGateway.csproj index ca934cb8..134f1ab9 100644 --- a/ModelContextGateway.csproj +++ b/ModelContextGateway.csproj @@ -6,9 +6,9 @@ enable mcg ModelContextGateway - 5.20.1 - 5.20.1.0 - 5.20.1.0 + 5.21.0 + 5.21.0.0 + 5.21.0.0 true true $(NoWarn);1591 diff --git a/README.md b/README.md index dd869bf3..7dae876e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Model Context Gateway (MCG) -![Version](https://img.shields.io/badge/version-v5.20.1-orange?style=for-the-badge) +![Version](https://img.shields.io/badge/version-v5.21.0-orange?style=for-the-badge) [![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue?style=for-the-badge&logo=githubpages&logoColor=white)](https://spelech.github.io/model-context-gateway/) ![.NET 10.0](https://img.shields.io/badge/.NET-10.0-512BD4?style=for-the-badge&logo=dotnet&logoColor=white) ![MCP Spec](https://img.shields.io/badge/MCP%20Spec-2026--07--28-0052CC?style=for-the-badge) @@ -175,11 +175,11 @@ For complete release history and version logs, see [**CHANGELOG.md**](CHANGELOG. | Version | Release Date | Summary of Key Changes | | :--- | :--- | :--- | +| **`v5.21.0`** | 2026-10-02 | feat(routing): calibrated hybrid semantic search scoring, explicit search modes, and test bench simulator overhaul | | **`v5.20.1`** | 2026-09-30 | refactor(testbench): Remove Pre-fill Example Button and Streamline Dynamic Argument Handling. Removed the legacy Pre-fill Example button and unused bulk argument modification plumbing from the Test Bench tool tester card, eliminating brittle and unhelpful parameter mocks while preserving interactive form validation and tool execution coverage. | | **`v5.20.0`** | 2026-09-30 | feat(auth,ui,docs,ci): AppKey & OAuth Client Last-Used Tracking, Safe Secret Rotation Workflow, Multi-Client Config Generator, VitePress Documentation Architecture Overhaul, and 30-Day Dependabot Stability Cooldown. Implemented `LastUsedAt` audit tracking for AppKeys and OAuth clients across SQLite, MySQL, and MSSQL with an in-memory 60-second sliding debounce `ActivityTracker`; added atomic credential rotation endpoints (`POST /api/appkeys/{id}/rotate`, `POST /api/clients/{id}/rotate-secret`) returning plaintext keys once with SHA-256 hashing at rest; built two-stage `AppKeyRotateModal` with backdrop click-away protection and multi-client configuration generator (Claude Desktop, Cursor, Windsurf, generic JSON); overhauled VitePress documentation architecture with DiΓ‘taxis decomposition, path-scoped multi-sidebars, purged legacy ASCII art to responsive Mermaid diagrams, modernized typography rhythm, and link integrity CI quality gates; updated GitHub Actions to v7/v5; and configured a 30-day stability cooldown across all Dependabot ecosystems (`AUTH-118` through `AUTH-123`, `UI-135` through `UI-137`). | | **`v5.19.1`** | 2026-09-29 | docs(slack): Purge Deprecated Wrapper References, Renumber Documentation Sections, and Refresh Wiki. Streamlined documentation for Slack Native Direct MCP (`https://mcp.slack.com/mcp`) and the Hybrid Egress Delegation Model; eliminated legacy self-hosted Node.js wrapper references across repository and wiki; updated release assets; and pinned container image tags. | | **`v5.19.0`** | 2026-09-29 | feat(slack,oauth,ui): Slack Direct Native MCP Server Integration, 3LO OAuth Egress UI, and Hybrid Token Delegation Model. Added native integration with Slack's cloud MCP server (`https://mcp.slack.com/mcp`) over streamable HTTP; implemented Hybrid Egress fallback enabling autonomous background daemons (e.g. OpenClaw) to authenticate via static User OAuth tokens (`xoxp-...`) while allowing interactive users to authorize individually via 3-legged OAuth (3LO); relocated injected trace telemetry into `params._meta` to satisfy TypeScript SDK Zod `strict()` schema validation; added 3LO user delegation toggle and configuration fields to `ServerModal` UI; exposed OAuth properties in Admin MCP `manage_servers` tooling; added Slack OAuth callback handling for `user_scope` and non-expiring credentials; and resolved ESLint `prefer-const` rules. | -| **`v5.18.0`** | 2026-09-27 | feat(core,ui,e2e): Comprehensive E2E Real Wiring Overhaul, Cold-Start Dynamic Delimiter Routing, Full data-testid UI Instrumentation, and Deterministic E2E Test Suite. Added on-demand delimiter-tolerant (`/`, `:`, `__`) routing across `PromptRoutingManager` and `ResourceRoutingManager` allowing connected agents to retrieve prompts and read un-virtualized `mcp://` resources without prior list handshakes; safely returned 404 for missing/disabled servers in `CapabilityEndpoints` testbench APIs; resolved runtime crashes in `AccessControlTab` with defensive array handling; instrumented all frontend components with standard `data-testid` attributes; created in-memory stateful `mockApi.ts` fixture eliminating 502 proxy errors during isolated E2E testing; eliminated all conditional skips and brittle locators in Playwright test suites (64/64 tests passing 100% across all 20 spec files); upgraded `mock-mcp-server` in `docker-compose.test.yml` to full Model Context Protocol parity; and achieved 1,209 total passing automated tests with zero requirements catalog drift (`MCP-36`, `MCP-37`, `MCP-38`, `MCP-39`, `UI-133`, `UI-134`, `TRANS-01`, `TRANS-02`, `SEC-01`, `AUTH-05`). | --- diff --git a/docs/assets/add_appkey_modal.jpg b/docs/assets/add_appkey_modal.jpg index d6e0cd7d..afecfb6c 100644 Binary files a/docs/assets/add_appkey_modal.jpg and b/docs/assets/add_appkey_modal.jpg differ diff --git a/docs/assets/add_server_modal.jpg b/docs/assets/add_server_modal.jpg index 7a6bf352..f150a051 100644 Binary files a/docs/assets/add_server_modal.jpg and b/docs/assets/add_server_modal.jpg differ diff --git a/docs/assets/dashboard.jpg b/docs/assets/dashboard.jpg index 39c1c308..1821d5c4 100644 Binary files a/docs/assets/dashboard.jpg and b/docs/assets/dashboard.jpg differ diff --git a/docs/assets/my_mcp_servers_view.jpg b/docs/assets/my_mcp_servers_view.jpg index 04bc019d..0f35e1dc 100644 Binary files a/docs/assets/my_mcp_servers_view.jpg and b/docs/assets/my_mcp_servers_view.jpg differ diff --git a/docs/assets/registered_client_modal.jpg b/docs/assets/registered_client_modal.jpg index 054329d8..a679c4bc 100644 Binary files a/docs/assets/registered_client_modal.jpg and b/docs/assets/registered_client_modal.jpg differ diff --git a/docs/assets/security_view.jpg b/docs/assets/security_view.jpg index 398629c4..31f75830 100644 Binary files a/docs/assets/security_view.jpg and b/docs/assets/security_view.jpg differ diff --git a/docs/assets/semantic_router_simulator.jpg b/docs/assets/semantic_router_simulator.jpg new file mode 100644 index 00000000..4a1464d4 Binary files /dev/null and b/docs/assets/semantic_router_simulator.jpg differ diff --git a/docs/assets/server_inspect_modal.jpg b/docs/assets/server_inspect_modal.jpg index c99509e5..319f8fff 100644 Binary files a/docs/assets/server_inspect_modal.jpg and b/docs/assets/server_inspect_modal.jpg differ diff --git a/docs/assets/settings_access_control.jpg b/docs/assets/settings_access_control.jpg index cb115482..141f5b0e 100644 Binary files a/docs/assets/settings_access_control.jpg and b/docs/assets/settings_access_control.jpg differ diff --git a/docs/assets/settings_identity_auth.jpg b/docs/assets/settings_identity_auth.jpg index ed4975d5..e214c9b6 100644 Binary files a/docs/assets/settings_identity_auth.jpg and b/docs/assets/settings_identity_auth.jpg differ diff --git a/docs/assets/settings_prompts_resources.jpg b/docs/assets/settings_prompts_resources.jpg index 3666d9a1..14a9db85 100644 Binary files a/docs/assets/settings_prompts_resources.jpg and b/docs/assets/settings_prompts_resources.jpg differ diff --git a/docs/assets/settings_secret_providers.jpg b/docs/assets/settings_secret_providers.jpg index 5d7beeec..09972d3b 100644 Binary files a/docs/assets/settings_secret_providers.jpg and b/docs/assets/settings_secret_providers.jpg differ diff --git a/docs/assets/settings_vector_search.jpg b/docs/assets/settings_vector_search.jpg index 2aa4b2a9..d0d9c6ff 100644 Binary files a/docs/assets/settings_vector_search.jpg and b/docs/assets/settings_vector_search.jpg differ diff --git a/docs/assets/settings_view.jpg b/docs/assets/settings_view.jpg index cb115482..141f5b0e 100644 Binary files a/docs/assets/settings_view.jpg and b/docs/assets/settings_view.jpg differ diff --git a/docs/assets/test_bench_view.jpg b/docs/assets/test_bench_view.jpg index 9fc89f7e..3759875c 100644 Binary files a/docs/assets/test_bench_view.jpg and b/docs/assets/test_bench_view.jpg differ diff --git a/docs/requirements-catalog.json b/docs/requirements-catalog.json index e1ea94a1..108f8fd4 100644 --- a/docs/requirements-catalog.json +++ b/docs/requirements-catalog.json @@ -1,10 +1,10 @@ { "metadata": { - "generatedAt": "2026-10-01T05:55:06.3348768Z", - "totalRequirements": 459, - "positiveCount": 364, - "guardrailCount": 95, - "totalProofs": 1054 + "generatedAt": "2026-10-03T03:52:25.4441412Z", + "totalRequirements": 464, + "positiveCount": 368, + "guardrailCount": 96, + "totalProofs": 1082 }, "requirements": [ { @@ -6182,7 +6182,7 @@ "category": "MCP", "type": "Positive", "description": "OpenAiEmbeddingProvider generates embeddings via OpenAI and Ollama compatible endpoints.", - "proofCount": 5, + "proofCount": 4, "proofs": [ { "suite": "Backend xUnit", @@ -6198,13 +6198,6 @@ "testName": "GenerateEmbeddingsAsync_PreservesInputOrder_AcrossBatch", "details": null }, - { - "suite": "Backend xUnit", - "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", - "lineNumber": 39, - "testName": "SearchToolsAsync_CombinesKeywordAndVectorRanks_UsingRRF", - "details": null - }, { "suite": "Backend xUnit", "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", @@ -6387,6 +6380,131 @@ } ] }, + { + "id": "MCP-40", + "category": "MCP", + "type": "Positive", + "description": "Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination.", + "proofCount": 5, + "proofs": [ + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 39, + "testName": "SearchToolsAsync_CombinesKeywordAndVectorRanks_UsingRRF", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 158, + "testName": "HybridSearch_WithBalancedWeight_ComputesLinearCombinationScore", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 293, + "testName": "HybridSearch_ReturnsDecomposedScores", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 355, + "testName": "HybridSearch_WithClampedDenseWeight_HandlesNegativeExcessiveAndNaN", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 391, + "testName": "HybridSearch_WithIdenticalLexicalScores_NormalizesWithoutDivideByZero", + "details": null + } + ] + }, + { + "id": "MCP-41", + "category": "MCP", + "type": "Positive", + "description": "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.", + "proofCount": 7, + "proofs": [ + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 203, + "testName": "HybridSearch_WithSemanticMode_OnlyScoresDenseSimilarity", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 248, + "testName": "HybridSearch_WithLexicalMode_OnlyScoresLexicalMatch", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 293, + "testName": "HybridSearch_ReturnsDecomposedScores", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 427, + "testName": "HybridSearch_WithEmptyQueryOrEmptyTools_ReturnsSafeGracefulList", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs", + "lineNumber": 451, + "testName": "HybridSearch_ResolveServerId_ResolvesVariousConventions", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/SemanticSearchEndpointTests.cs", + "lineNumber": 43, + "testName": "SemanticSearchEndpoint_ReturnsCalibratedHybridResultsWithDiagnostics", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/SemanticSearchEndpointTests.cs", + "lineNumber": 126, + "testName": "SemanticSearchEndpoint_RespectsSearchModeAndDenseWeightParameters", + "details": null + } + ] + }, + { + "id": "MCP-42", + "category": "MCP", + "type": "Positive", + "description": "Meta-mode search_tools MCP tool accepts optional mode and dense_weight arguments and executes calibrated tool retrieval.", + "proofCount": 2, + "proofs": [ + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/SemanticSearchEndpointTests.cs", + "lineNumber": 185, + "testName": "MetaModeTools_SearchToolsSchema_ExposesModeAndDenseWeight", + "details": null + }, + { + "suite": "Backend xUnit", + "filePath": "ModelContextGateway.Tests/SemanticSearchEndpointTests.cs", + "lineNumber": 217, + "testName": "SearchTools_CustomToolCall_ExecutesWithModeAndDenseWeight", + "details": null + } + ] + }, { "id": "MCP-ADMIN-ENDPOINT-CALL-TOOL", "category": "MCP", @@ -10419,252 +10537,252 @@ { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 255, + "lineNumber": 276, "testName": "should pass layout audit on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 276, + "lineNumber": 297, "testName": "should pass layout audit on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 321, + "lineNumber": 342, "testName": "should pass layout audit on Samsung Galaxy Tab S10 Lite tablet viewport (portrait)", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 342, + "lineNumber": 363, "testName": "should pass layout audit on Samsung Galaxy Tab S10 Lite tablet viewport (landscape)", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 363, + "lineNumber": 384, "testName": "should pass layout audit for Add Server modal on Samsung Galaxy Tab S10 Lite tablet", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 404, + "lineNumber": 425, "testName": "should pass layout audit for Add Server modal on desktop 1080p", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 444, + "lineNumber": 465, "testName": "should pass layout audit for Add Server modal on Samsung Galaxy S25\u002B mobile", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 486, + "lineNumber": 507, "testName": "should pass layout audit across Capabilities Inspect modal tabs", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 542, + "lineNumber": 563, "testName": "should pass layout audit on App Keys \u0026 Security tab", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 570, + "lineNumber": 591, "testName": "should pass layout audit on Settings tab", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 598, + "lineNumber": 619, "testName": "should maintain layout stability during tab navigation", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 620, + "lineNumber": 641, "testName": "should pass layout audit on Test Bench tab on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 662, + "lineNumber": 683, "testName": "should pass layout audit on Test Bench tab on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 693, + "lineNumber": 714, "testName": "should pass layout audit on Test Bench tab on Samsung Galaxy Tab S10 Lite tablet viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 724, + "lineNumber": 745, "testName": "should pass layout audit on Test Bench Prompts and Resources tabs", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 776, + "lineNumber": 797, "testName": "should pass layout audit on Test Bench across Half-Ultrawide viewports", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 817, + "lineNumber": 838, "testName": "should pass layout audit on Test Bench Semantic Router tab", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 848, + "lineNumber": 880, "testName": "should pass layout audit on App Keys \u0026 Security sub-tabs on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 905, + "lineNumber": 937, "testName": "should pass layout audit on App Keys \u0026 Security sub-tabs on Samsung Galaxy Tab S10 Lite tablet viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 958, + "lineNumber": 990, "testName": "should pass layout audit on Settings sub-tabs on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1025, + "lineNumber": 1057, "testName": "should pass layout audit on Settings sub-tabs on Samsung Galaxy Tab S10 Lite tablet viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1091, + "lineNumber": 1123, "testName": "should pass layout audit on My MCP Servers tab on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1118, + "lineNumber": 1150, "testName": "should pass layout audit on My MCP Servers tab on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1146, + "lineNumber": 1178, "testName": "should pass layout audit on My MCP Servers tab on Samsung Galaxy Tab S10 Lite tablet viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1173, + "lineNumber": 1205, "testName": "should pass layout audit across Capabilities Inspect modal tabs on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1240, + "lineNumber": 1272, "testName": "should pass layout audit across Capabilities Inspect modal tabs on Samsung Galaxy Tab S10 Lite tablet viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1307, + "lineNumber": 1339, "testName": "should pass layout audit for AppKey modal on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1351, + "lineNumber": 1383, "testName": "should pass layout audit for AppKey modal on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1396, + "lineNumber": 1428, "testName": "should pass layout audit for Client modal on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1440, + "lineNumber": 1472, "testName": "should pass layout audit for Client modal on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1485, + "lineNumber": 1517, "testName": "should pass layout audit for CustomFile modal on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1534, + "lineNumber": 1566, "testName": "should pass layout audit for CustomFile modal on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1584, + "lineNumber": 1616, "testName": "should pass layout audit for Policy modal on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1633, + "lineNumber": 1665, "testName": "should pass layout audit for Policy modal on Samsung Galaxy S25\u002B mobile viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1683, + "lineNumber": 1715, "testName": "should pass layout audit for Mapping modal on desktop 1080p viewport", "details": null }, { "suite": "Playwright E2E", "filePath": "frontend/e2e/layout-inspector.spec.ts", - "lineNumber": 1732, + "lineNumber": 1764, "testName": "should pass layout audit for Mapping modal on Samsung Galaxy S25\u002B mobile viewport", "details": null } @@ -11215,7 +11333,7 @@ "category": "UI", "type": "Positive", "description": "verifies deterministic data-testid attributes and interactive flow across test bench tabs including dedicated semantic router", - "proofCount": 1, + "proofCount": 2, "proofs": [ { "suite": "Frontend Vitest", @@ -11223,6 +11341,129 @@ "lineNumber": 473, "testName": "renders and interacts with test bench elements via deterministic data-testid attributes", "details": null + }, + { + "suite": "Playwright E2E", + "filePath": "frontend/e2e/testbench.spec.ts", + "lineNumber": 54, + "testName": "should interact with semantic router simulator, change modes and presets, and test tool transition", + "details": null + } + ] + }, + { + "id": "UI-142", + "category": "UI", + "type": "Negative", + "description": "verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes", + "proofCount": 10, + "proofs": [ + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/SemanticRouterCard.test.tsx", + "lineNumber": 35, + "testName": "verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/SemanticRouterCard.test.tsx", + "lineNumber": 98, + "testName": "verifies hybrid split slider and quick preset chips appear in hybrid mode and update denseWeight", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/SemanticRouterCard.test.tsx", + "lineNumber": 176, + "testName": "verifies search limit dropdown changes value", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/useTestBenchState.test.ts", + "lineNumber": 14, + "testName": "initializes semantic search state with defaults", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/useTestBenchState.test.ts", + "lineNumber": 34, + "testName": "updates searchMode, denseWeight, and searchLimit state", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/useTestBenchState.test.ts", + "lineNumber": 58, + "testName": "executes handleSemanticSearch using current state parameters", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/useTestBenchState.test.ts", + "lineNumber": 102, + "testName": "handles semantic search failures gracefully", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/api/testbenchApi.test.ts", + "lineNumber": 11, + "testName": "calls apiRequest with query, mode, denseWeight, and limit parameters", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/api/testbenchApi.test.ts", + "lineNumber": 53, + "testName": "uses default values for mode, denseWeight, and limit when not specified", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/api/testbenchApi.test.ts", + "lineNumber": 84, + "testName": "returns fallback object when apiRequest returns null or undefined", + "details": null + } + ] + }, + { + "id": "UI-143", + "category": "UI", + "type": "Positive", + "description": "renders decomposed score badges with formatted percentages in hybrid mode", + "proofCount": 4, + "proofs": [ + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/SemanticRouterCard.test.tsx", + "lineNumber": 209, + "testName": "renders decomposed score badges with formatted percentages in hybrid mode", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/SemanticRouterCard.test.tsx", + "lineNumber": 251, + "testName": "hides dense and sparse badges in pure semantic or lexical modes", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/SemanticRouterCard.test.tsx", + "lineNumber": 279, + "testName": "clicking Test Tool button dispatches onSelectTool with serverId and toolName", + "details": null + }, + { + "suite": "Frontend Vitest", + "filePath": "frontend/src/test/components/TestBenchView.test.tsx", + "lineNumber": 512, + "testName": "clicking Test Tool in semantic search results switches to Tools tab with tool and server preselected", + "details": null } ] }, diff --git a/docs/software-requirements-and-test-catalog.md b/docs/software-requirements-and-test-catalog.md index 760ab981..f9e9ca17 100644 --- a/docs/software-requirements-and-test-catalog.md +++ b/docs/software-requirements-and-test-catalog.md @@ -1,7 +1,7 @@ # Software Requirements Specification (SRS) & Test Verification Catalog > **Automated Verification Document:** Generated via `dotnet run --project scripts/CatalogGenerator` -> **Catalog Statistics:** **459 Requirements Verified** across **1054 Test Proofs** (364 Functional Capabilities, 95 Safety Guardrails). +> **Catalog Statistics:** **464 Requirements Verified** across **1082 Test Proofs** (368 Functional Capabilities, 96 Safety Guardrails). --- @@ -15,10 +15,10 @@ | **`DB`** | Multi-Database Persistence & Migrations | **23** | 22 | 1 | 35 proofs | | **`DOC`** | DOC | **4** | 4 | 0 | 4 proofs | | **`GUARD`** | Universal Safety & Fail-Closed Guardrails | **66** | 3 | 63 | 138 proofs | -| **`MCP`** | Model Context Protocol Engine & Tool Routing | **117** | 110 | 7 | 251 proofs | +| **`MCP`** | Model Context Protocol Engine & Tool Routing | **120** | 113 | 7 | 264 proofs | | **`SEC`** | Secrets Providers & Encryption | **65** | 56 | 9 | 138 proofs | | **`TRANS`** | Transports (SSE, HTTP, STDIO, Proxy) | **36** | 32 | 4 | 42 proofs | -| **`UI`** | Dashboard, Test Bench & Settings UI | **38** | 32 | 6 | 197 proofs | +| **`UI`** | Dashboard, Test Bench & Settings UI | **40** | 33 | 7 | 212 proofs | --- @@ -1216,10 +1216,9 @@ ### `[MCP-33]` OpenAiEmbeddingProvider generates embeddings via OpenAI and Ollama compatible endpoints. * **Category:** `MCP` (Model Context Protocol Engine & Tool Routing) * **Type:** Positive Feature Capability -* **Verification Proofs (5):** +* **Verification Proofs (4):** - [Backend xUnit] [`ModelContextGateway.Tests/OpenAiEmbeddingProviderTests.cs#L26`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/OpenAiEmbeddingProviderTests.cs#L26) (`GenerateEmbeddingAsync_SendsValidPayload_AndParsesResponse`) - [Backend xUnit] [`ModelContextGateway.Tests/OpenAiEmbeddingProviderTests.cs#L75`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/OpenAiEmbeddingProviderTests.cs#L75) (`GenerateEmbeddingsAsync_PreservesInputOrder_AcrossBatch`) - - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L39`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L39) (`SearchToolsAsync_CombinesKeywordAndVectorRanks_UsingRRF`) - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L92`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L92) (`SearchToolsAsync_ScoresLexicalSignals_AcrossNameDescriptionTagsAndParameters`) - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L141`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L141) (`SearchTools_SynchronousMethod_ReturnsRankedCandidates`) @@ -1258,6 +1257,35 @@ * **Verification Proofs (1):** - [Backend xUnit] [`ModelContextGateway.Tests/PromptAndResourceResilienceTests.cs#L165`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/PromptAndResourceResilienceTests.cs#L165) (`CapabilityEndpoints_TestBench_HandlesMissingServer_GracefullyWith404`) +### `[MCP-40]` Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination. +* **Category:** `MCP` (Model Context Protocol Engine & Tool Routing) +* **Type:** Positive Feature Capability +* **Verification Proofs (5):** + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L39`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L39) (`SearchToolsAsync_CombinesKeywordAndVectorRanks_UsingRRF`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L158`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L158) (`HybridSearch_WithBalancedWeight_ComputesLinearCombinationScore`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L293`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L293) (`HybridSearch_ReturnsDecomposedScores`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L355`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L355) (`HybridSearch_WithClampedDenseWeight_HandlesNegativeExcessiveAndNaN`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L391`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L391) (`HybridSearch_WithIdenticalLexicalScores_NormalizesWithoutDivideByZero`) + +### `[MCP-41]` Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager. +* **Category:** `MCP` (Model Context Protocol Engine & Tool Routing) +* **Type:** Positive Feature Capability +* **Verification Proofs (7):** + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L203`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L203) (`HybridSearch_WithSemanticMode_OnlyScoresDenseSimilarity`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L248`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L248) (`HybridSearch_WithLexicalMode_OnlyScoresLexicalMatch`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L293`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L293) (`HybridSearch_ReturnsDecomposedScores`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L427`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L427) (`HybridSearch_WithEmptyQueryOrEmptyTools_ReturnsSafeGracefulList`) + - [Backend xUnit] [`ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L451`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L451) (`HybridSearch_ResolveServerId_ResolvesVariousConventions`) + - [Backend xUnit] [`ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L43`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L43) (`SemanticSearchEndpoint_ReturnsCalibratedHybridResultsWithDiagnostics`) + - [Backend xUnit] [`ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L126`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L126) (`SemanticSearchEndpoint_RespectsSearchModeAndDenseWeightParameters`) + +### `[MCP-42]` Meta-mode search_tools MCP tool accepts optional mode and dense_weight arguments and executes calibrated tool retrieval. +* **Category:** `MCP` (Model Context Protocol Engine & Tool Routing) +* **Type:** Positive Feature Capability +* **Verification Proofs (2):** + - [Backend xUnit] [`ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L185`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L185) (`MetaModeTools_SearchToolsSchema_ExposesModeAndDenseWeight`) + - [Backend xUnit] [`ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L217`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L217) (`SearchTools_CustomToolCall_ExecutesWithModeAndDenseWeight`) + ### `[MCP-ADMIN-ENDPOINT-CALL-TOOL]` Admin endpoint /admin/message executes tools/call for manage_system diagnostics. * **Category:** `MCP` (Model Context Protocol Engine & Tool Routing) * **Type:** Positive Feature Capability @@ -2416,42 +2444,42 @@ * **Category:** `UI` (Dashboard, Test Bench & Settings UI) * **Type:** Positive Feature Capability * **Verification Proofs (36):** - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L255`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L255) (`should pass layout audit on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L276`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L276) (`should pass layout audit on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L321`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L321) (`should pass layout audit on Samsung Galaxy Tab S10 Lite tablet viewport (portrait)`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L342`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L342) (`should pass layout audit on Samsung Galaxy Tab S10 Lite tablet viewport (landscape)`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L363`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L363) (`should pass layout audit for Add Server modal on Samsung Galaxy Tab S10 Lite tablet`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L404`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L404) (`should pass layout audit for Add Server modal on desktop 1080p`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L444`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L444) (`should pass layout audit for Add Server modal on Samsung Galaxy S25+ mobile`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L486`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L486) (`should pass layout audit across Capabilities Inspect modal tabs`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L542`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L542) (`should pass layout audit on App Keys & Security tab`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L570`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L570) (`should pass layout audit on Settings tab`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L598`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L598) (`should maintain layout stability during tab navigation`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L620`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L620) (`should pass layout audit on Test Bench tab on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L662`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L662) (`should pass layout audit on Test Bench tab on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L693`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L693) (`should pass layout audit on Test Bench tab on Samsung Galaxy Tab S10 Lite tablet viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L724`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L724) (`should pass layout audit on Test Bench Prompts and Resources tabs`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L776`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L776) (`should pass layout audit on Test Bench across Half-Ultrawide viewports`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L817`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L817) (`should pass layout audit on Test Bench Semantic Router tab`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L848`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L848) (`should pass layout audit on App Keys & Security sub-tabs on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L905`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L905) (`should pass layout audit on App Keys & Security sub-tabs on Samsung Galaxy Tab S10 Lite tablet viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L958`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L958) (`should pass layout audit on Settings sub-tabs on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1025`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1025) (`should pass layout audit on Settings sub-tabs on Samsung Galaxy Tab S10 Lite tablet viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1091`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1091) (`should pass layout audit on My MCP Servers tab on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1118`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1118) (`should pass layout audit on My MCP Servers tab on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1146`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1146) (`should pass layout audit on My MCP Servers tab on Samsung Galaxy Tab S10 Lite tablet viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1173`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1173) (`should pass layout audit across Capabilities Inspect modal tabs on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1240`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1240) (`should pass layout audit across Capabilities Inspect modal tabs on Samsung Galaxy Tab S10 Lite tablet viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1307`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1307) (`should pass layout audit for AppKey modal on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1351`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1351) (`should pass layout audit for AppKey modal on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1396`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1396) (`should pass layout audit for Client modal on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1440`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1440) (`should pass layout audit for Client modal on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1485`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1485) (`should pass layout audit for CustomFile modal on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1534`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1534) (`should pass layout audit for CustomFile modal on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1584`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1584) (`should pass layout audit for Policy modal on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1633`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1633) (`should pass layout audit for Policy modal on Samsung Galaxy S25+ mobile viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1683`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1683) (`should pass layout audit for Mapping modal on desktop 1080p viewport`) - - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1732`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1732) (`should pass layout audit for Mapping modal on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L276`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L276) (`should pass layout audit on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L297`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L297) (`should pass layout audit on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L342`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L342) (`should pass layout audit on Samsung Galaxy Tab S10 Lite tablet viewport (portrait)`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L363`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L363) (`should pass layout audit on Samsung Galaxy Tab S10 Lite tablet viewport (landscape)`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L384`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L384) (`should pass layout audit for Add Server modal on Samsung Galaxy Tab S10 Lite tablet`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L425`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L425) (`should pass layout audit for Add Server modal on desktop 1080p`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L465`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L465) (`should pass layout audit for Add Server modal on Samsung Galaxy S25+ mobile`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L507`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L507) (`should pass layout audit across Capabilities Inspect modal tabs`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L563`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L563) (`should pass layout audit on App Keys & Security tab`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L591`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L591) (`should pass layout audit on Settings tab`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L619`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L619) (`should maintain layout stability during tab navigation`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L641`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L641) (`should pass layout audit on Test Bench tab on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L683`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L683) (`should pass layout audit on Test Bench tab on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L714`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L714) (`should pass layout audit on Test Bench tab on Samsung Galaxy Tab S10 Lite tablet viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L745`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L745) (`should pass layout audit on Test Bench Prompts and Resources tabs`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L797`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L797) (`should pass layout audit on Test Bench across Half-Ultrawide viewports`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L838`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L838) (`should pass layout audit on Test Bench Semantic Router tab`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L880`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L880) (`should pass layout audit on App Keys & Security sub-tabs on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L937`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L937) (`should pass layout audit on App Keys & Security sub-tabs on Samsung Galaxy Tab S10 Lite tablet viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L990`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L990) (`should pass layout audit on Settings sub-tabs on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1057`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1057) (`should pass layout audit on Settings sub-tabs on Samsung Galaxy Tab S10 Lite tablet viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1123`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1123) (`should pass layout audit on My MCP Servers tab on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1150`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1150) (`should pass layout audit on My MCP Servers tab on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1178`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1178) (`should pass layout audit on My MCP Servers tab on Samsung Galaxy Tab S10 Lite tablet viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1205`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1205) (`should pass layout audit across Capabilities Inspect modal tabs on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1272`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1272) (`should pass layout audit across Capabilities Inspect modal tabs on Samsung Galaxy Tab S10 Lite tablet viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1339`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1339) (`should pass layout audit for AppKey modal on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1383`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1383) (`should pass layout audit for AppKey modal on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1428`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1428) (`should pass layout audit for Client modal on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1472`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1472) (`should pass layout audit for Client modal on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1517`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1517) (`should pass layout audit for CustomFile modal on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1566`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1566) (`should pass layout audit for CustomFile modal on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1616`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1616) (`should pass layout audit for Policy modal on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1665`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1665) (`should pass layout audit for Policy modal on Samsung Galaxy S25+ mobile viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1715`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1715) (`should pass layout audit for Mapping modal on desktop 1080p viewport`) + - [Playwright E2E] [`frontend/e2e/layout-inspector.spec.ts#L1764`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L1764) (`should pass layout audit for Mapping modal on Samsung Galaxy S25+ mobile viewport`) ### `[UI-102]` Dashboard renders stats card, connected server list, and setup instructions * **Category:** `UI` (Dashboard, Test Bench & Settings UI) @@ -2600,8 +2628,18 @@ ### `[UI-141]` verifies deterministic data-testid attributes and interactive flow across test bench tabs including dedicated semantic router * **Category:** `UI` (Dashboard, Test Bench & Settings UI) * **Type:** Positive Feature Capability -* **Verification Proofs (1):** +* **Verification Proofs (2):** - [Frontend Vitest] [`frontend/src/test/components/TestBenchView.test.tsx#L473`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/TestBenchView.test.tsx#L473) (`renders and interacts with test bench elements via deterministic data-testid attributes`) + - [Playwright E2E] [`frontend/e2e/testbench.spec.ts#L54`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/testbench.spec.ts#L54) (`should interact with semantic router simulator, change modes and presets, and test tool transition`) + +### `[UI-143]` renders decomposed score badges with formatted percentages in hybrid mode +* **Category:** `UI` (Dashboard, Test Bench & Settings UI) +* **Type:** Positive Feature Capability +* **Verification Proofs (4):** + - [Frontend Vitest] [`frontend/src/test/components/SemanticRouterCard.test.tsx#L209`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L209) (`renders decomposed score badges with formatted percentages in hybrid mode`) + - [Frontend Vitest] [`frontend/src/test/components/SemanticRouterCard.test.tsx#L251`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L251) (`hides dense and sparse badges in pure semantic or lexical modes`) + - [Frontend Vitest] [`frontend/src/test/components/SemanticRouterCard.test.tsx#L279`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L279) (`clicking Test Tool button dispatches onSelectTool with serverId and toolName`) + - [Frontend Vitest] [`frontend/src/test/components/TestBenchView.test.tsx#L512`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/TestBenchView.test.tsx#L512) (`clicking Test Tool in semantic search results switches to Tools tab with tool and server preselected`) ### `[UI-30]` Renders client registration form with inputs for name, client type, redirect URIs, grant types, scopes, and expiration. * **Category:** `UI` (Dashboard, Test Bench & Settings UI) @@ -3334,6 +3372,21 @@ - [Frontend Vitest] [`frontend/src/test/components/RedirectUrisModal.test.tsx#L71`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/RedirectUrisModal.test.tsx#L71) (`calls onClose when close button or dismiss is clicked`) - [Frontend Vitest] [`frontend/src/test/components/RedirectUrisModal.test.tsx#L93`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/RedirectUrisModal.test.tsx#L93) (`renders nothing when isOpen is false`) +### `[UI-142]` verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes +* **Category:** `UI` (Dashboard, Test Bench & Settings UI) +* **Type:** Negative / Safety Guardrail (Fail-Closed) +* **Verification Proofs (10):** + - [Frontend Vitest] [`frontend/src/test/components/SemanticRouterCard.test.tsx#L35`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L35) (`verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes`) + - [Frontend Vitest] [`frontend/src/test/components/SemanticRouterCard.test.tsx#L98`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L98) (`verifies hybrid split slider and quick preset chips appear in hybrid mode and update denseWeight`) + - [Frontend Vitest] [`frontend/src/test/components/SemanticRouterCard.test.tsx#L176`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L176) (`verifies search limit dropdown changes value`) + - [Frontend Vitest] [`frontend/src/test/components/useTestBenchState.test.ts#L14`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/useTestBenchState.test.ts#L14) (`initializes semantic search state with defaults`) + - [Frontend Vitest] [`frontend/src/test/components/useTestBenchState.test.ts#L34`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/useTestBenchState.test.ts#L34) (`updates searchMode, denseWeight, and searchLimit state`) + - [Frontend Vitest] [`frontend/src/test/components/useTestBenchState.test.ts#L58`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/useTestBenchState.test.ts#L58) (`executes handleSemanticSearch using current state parameters`) + - [Frontend Vitest] [`frontend/src/test/components/useTestBenchState.test.ts#L102`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/useTestBenchState.test.ts#L102) (`handles semantic search failures gracefully`) + - [Frontend Vitest] [`frontend/src/test/api/testbenchApi.test.ts#L11`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/api/testbenchApi.test.ts#L11) (`calls apiRequest with query, mode, denseWeight, and limit parameters`) + - [Frontend Vitest] [`frontend/src/test/api/testbenchApi.test.ts#L53`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/api/testbenchApi.test.ts#L53) (`uses default values for mode, denseWeight, and limit when not specified`) + - [Frontend Vitest] [`frontend/src/test/api/testbenchApi.test.ts#L84`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/api/testbenchApi.test.ts#L84) (`returns fallback object when apiRequest returns null or undefined`) + ### `[UI-31]` Fetches registered OAuth clients and updates store state. * **Category:** `UI` (Dashboard, Test Bench & Settings UI) * **Type:** Negative / Safety Guardrail (Fail-Closed) @@ -3629,6 +3682,9 @@ | `MCP-37` | Positive | `MCP` | ResourceRoutingManager resolves mcp:// URIs directly on cold-start without prior resources/list. | [`PromptAndResourceResilienceTests.cs:L120`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/PromptAndResourceResilienceTests.cs#L120) | Backend xUnit | | `MCP-38` | Positive | `MCP` | PromptRoutingManager supports slash, colon, and double-underscore delimiters for prompts/get. | [`PromptAndResourceResilienceTests.cs:L56`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/PromptAndResourceResilienceTests.cs#L56) | Backend xUnit | | `MCP-39` | Positive | `MCP` | CapabilityEndpoints test bench APIs return 404 Not Found gracefully when target server is missing. | [`PromptAndResourceResilienceTests.cs:L165`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/PromptAndResourceResilienceTests.cs#L165) | Backend xUnit | +| `MCP-40` | Positive | `MCP` | Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination. | [`ToolRoutingManagerHybridSearchTests.cs:L39`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L39) | Backend xUnit | +| `MCP-41` | Positive | `MCP` | Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager. | [`ToolRoutingManagerHybridSearchTests.cs:L203`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs#L203) | Backend xUnit | +| `MCP-42` | Positive | `MCP` | Meta-mode search_tools MCP tool accepts optional mode and dense_weight arguments and executes calibrated tool retrieval. | [`SemanticSearchEndpointTests.cs:L185`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/SemanticSearchEndpointTests.cs#L185) | Backend xUnit | | `MCP-ADMIN-ENDPOINT-CALL-TOOL` | Positive | `MCP` | Admin endpoint /admin/message executes tools/call for manage_system diagnostics. | [`AdminEndpointsTests.cs:L294`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/AdminEndpointsTests.cs#L294) | Backend xUnit | | `MCP-ADMIN-ENDPOINT-HEAD-REQUEST` | Positive | `MCP` | Admin endpoint /admin handles HEAD request returning text/event-stream headers. | [`AdminEndpointsTests.cs:L212`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/AdminEndpointsTests.cs#L212) | Backend xUnit | | `MCP-ADMIN-ENDPOINT-LIST-TOOLS` | Positive | `MCP` | Admin endpoint /admin/message executes tools/list over active SSE session and returns 10 admin tools. | [`AdminEndpointsTests.cs:L224`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/AdminEndpointsTests.cs#L224) | Backend xUnit | @@ -3813,7 +3869,7 @@ | `UI-04` | Positive | `UI` | Tool selector filters available tools by selected backend server | [`ToolTesterCard.test.tsx:L77`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/ToolTesterCard.test.tsx#L77) | Frontend Vitest | | `UI-05` | Positive | `UI` | Router allows customized branding parameters (DashboardTitle, DashboardIcon) to be saved and retrieved via the API. | [`PipelineIntegrationTests.cs:L254`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/PipelineIntegrationTests.cs#L254) | Backend xUnit | | `UI-06` | Positive | `UI` | Router supports uploading and retrieving custom branding logo images via dedicated endpoints. | [`PipelineIntegrationTests.cs:L448`](https://github.com/spelech/model-context-gateway/blob/main/ModelContextGateway.Tests/PipelineIntegrationTests.cs#L448) | Backend xUnit | -| `UI-07` | Positive | `UI` | Audits desktop viewport layout for zero horizontal overflow and high UX score. | [`layout-inspector.spec.ts:L255`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L255) | Playwright E2E | +| `UI-07` | Positive | `UI` | Audits desktop viewport layout for zero horizontal overflow and high UX score. | [`layout-inspector.spec.ts:L276`](https://github.com/spelech/model-context-gateway/blob/main/frontend/e2e/layout-inspector.spec.ts#L276) | Playwright E2E | | `UI-102` | Positive | `UI` | Dashboard renders stats card, connected server list, and setup instructions | [`DashboardView.test.tsx:L1`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/DashboardView.test.tsx#L1) | Frontend Vitest | | `UI-103` | Positive | `UI` | Interactive tool tester renders server and tool selection dropdowns | [`ToolTesterCard.test.tsx:L1`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/ToolTesterCard.test.tsx#L1) | Frontend Vitest | | `UI-108` | Positive | `UI` | renders nothing when isMappingModalOpen is false | [`MappingModal.test.tsx:L1`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/MappingModal.test.tsx#L1) | Frontend Vitest | @@ -3840,6 +3896,8 @@ | `UI-139` | **Guardrail** | `UI` | Compacts multiple redirect URIs with +N more button and opens RedirectUrisModal on click. | [`RegisteredClientsCard.test.tsx:L199`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/RegisteredClientsCard.test.tsx#L199) | Frontend Vitest | | `UI-140` | Positive | `UI` | Navigation tabs maintain static 1px baseline border to eliminate vertical height jumps across active toggles | [`LayoutCentering.test.tsx:L132`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/LayoutCentering.test.tsx#L132) | Frontend Vitest | | `UI-141` | Positive | `UI` | verifies deterministic data-testid attributes and interactive flow across test bench tabs including dedicated semantic router | [`TestBenchView.test.tsx:L473`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/TestBenchView.test.tsx#L473) | Frontend Vitest | +| `UI-142` | **Guardrail** | `UI` | verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes | [`SemanticRouterCard.test.tsx:L35`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L35) | Frontend Vitest | +| `UI-143` | Positive | `UI` | renders decomposed score badges with formatted percentages in hybrid mode | [`SemanticRouterCard.test.tsx:L209`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/SemanticRouterCard.test.tsx#L209) | Frontend Vitest | | `UI-30` | Positive | `UI` | Renders client registration form with inputs for name, client type, redirect URIs, grant types, scopes, and expiration. | [`ClientModal.test.tsx:L27`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/components/ClientModal.test.tsx#L27) | Frontend Vitest | | `UI-31` | **Guardrail** | `UI` | Fetches registered OAuth clients and updates store state. | [`useClientStore.test.ts:L37`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/stores/useClientStore.test.ts#L37) | Frontend Vitest | | `UI-32` | Positive | `UI` | Registers OAuth client with extended metadata (redirect URIs, grant types, client type, expiration) and captures one-time credentials. | [`useClientStore.test.ts:L76`](https://github.com/spelech/model-context-gateway/blob/main/frontend/src/test/stores/useClientStore.test.ts#L76) | Frontend Vitest | diff --git a/docs/superpowers/plans/2026-10-02-semantic-search-hybrid-calibration.md b/docs/superpowers/plans/2026-10-02-semantic-search-hybrid-calibration.md new file mode 100644 index 00000000..6cd5eaaf --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-semantic-search-hybrid-calibration.md @@ -0,0 +1,291 @@ +# Semantic Search Hybrid Calibration & Test Bench Alignment Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Calibrate hybrid semantic search with configurable dense weights ($\alpha \in [0.0, 1.0]$), explicit search modes (`hybrid`, `semantic`, `lexical`), score decomposition diagnostics, and a redesigned Semantic Router Simulator on the Test Bench matching ContextCortex. + +**Architecture:** Extend `ToolRoutingManager.Search.cs` with a normalized linear combination scoring model ($\text{Score} = \alpha \cdot \text{DenseScore} + (1-\alpha) \cdot \text{SparseScore}$) and return rich `ToolSearchResult` models with decomposed scores. Update `POST /api/test/semantic-search` and meta-mode `search_tools`. Overhaul `SemanticRouterCard.tsx` with a 3-way mode toggle, interactive slider with quick presets, score chips, and a 1-click "Test Tool" delegation button. + +**Tech Stack:** C# .NET 10, ASP.NET Core, React 19, TypeScript, Vitest, Playwright, Puppeteer. + +## Global Constraints + +- Every code change commit or merge to `main` must bump the version number simultaneously in `ModelContextGateway.csproj`, `frontend/src/stores/useUserStore.ts`, `CHANGELOG.md`, and `README.md`. +- All new and modified tests must be annotated with formal requirement metadata (`MCP-40`, `MCP-41`, `MCP-42`, `UI-142`, `UI-143`). +- Requirements catalog must be verified with zero drift: `dotnet run --project scripts/CatalogGenerator -- --verify-only`. +- Real screenshots must be captured using Puppeteer (`scripts/take_screenshots.js`) on `net_cloud` and saved under `docs/assets/` (no placeholders or mockups). +- Atomic commits for each logical task. + +--- + +### Task 1: Core Scoring Engine & Data Models (`MCP-40`, `MCP-41`) + +**Files:** +- Create: `Core/Routing/ToolSearchResult.cs` +- Modify: `Core/Routing/ToolRoutingManager.Search.cs:60-220` +- Test: `ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs` + +**Interfaces:** +- Consumes: `IEmbeddingProvider`, `IToolVectorStore`, `ToolMetadata` +- Produces: `ToolSearchResult`, `ToolRoutingManager.SearchToolsDetailedAsync(...)` + +- [ ] **Step 1: Write the failing tests** +Create `ToolSearchResult` model and add unit tests in `ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs`: + - `HybridSearch_WithBalancedWeight_ComputesLinearCombinationScore`: tests $\alpha = 0.5$ combines dense cosine similarity and normalized lexical score. + - `HybridSearch_WithSemanticMode_OnlyScoresDenseSimilarity`: tests `searchMode = "semantic"` or $\alpha = 1.0$. + - `HybridSearch_WithLexicalMode_OnlyScoresLexicalMatch`: tests `searchMode = "lexical"` or $\alpha = 0.0$. + - `HybridSearch_ReturnsDecomposedScores`: tests `DenseScore`, `SparseScore`, `DenseRank`, `SparseRank` populated correctly in $[0.0, 1.0]$. + Annotate tests with `[Requirement("MCP-40", "MCP", RequirementType.Positive, "Calibrated hybrid semantic search scoring with configurable dense weight and normalized linear combination.")]` and `[Requirement("MCP-41", "MCP", RequirementType.Positive, "Explicit search modes (hybrid, semantic, lexical) and decomposed score diagnostics in ToolRoutingManager.")]`. + +- [ ] **Step 2: Run tests to verify they fail** +```bash +dotnet test ModelContextGateway.slnx --filter "FullyQualifiedName~ToolRoutingManagerHybridSearchTests" +``` + +- [ ] **Step 3: Implement `ToolSearchResult` and `SearchToolsDetailedAsync`** + - Define `ToolSearchResult` in `Core/Routing/ToolSearchResult.cs`. + - In `Core/Routing/ToolRoutingManager.Search.cs`: + - Implement `SearchToolsDetailedAsync(string query, List? candidateTools, IEmbeddingProvider? embeddingProvider, IToolVectorStore? vectorStore, ILogger? logger, int limit = 15, string searchMode = "hybrid", double? denseWeight = 0.5, CancellationToken cancellationToken = default)`. + - Handle `searchMode` ("hybrid", "semantic", "lexical") and clamp `denseWeight` to $[0.0, 1.0]$. + - Compute min-max normalized lexical scores for candidates with matches. + - Compute dense cosine similarity clamped to $[0.0, 1.0]$. + - Calculate linear combination: $\text{Score} = \alpha \cdot \text{DenseScore} + (1.0 - \alpha) \cdot \text{SparseScore}$. + - Return sorted `List` with populated ranks and scores. + - Keep `SearchToolsAsync` as a backward-compatible wrapper returning `List`. + +- [ ] **Step 4: Run tests to verify they pass** +```bash +dotnet test ModelContextGateway.slnx --filter "FullyQualifiedName~ToolRoutingManagerHybridSearchTests" +``` + +- [ ] **Step 5: Commit work** +```bash +git add Core/Routing/ToolSearchResult.cs Core/Routing/ToolRoutingManager.Search.cs ModelContextGateway.Tests/ToolRoutingManagerHybridSearchTests.cs +git commit -m "feat(routing): implement calibrated hybrid scoring and explicit search modes (MCP-40, MCP-41)" +``` + +--- + +### Task 2: API Endpoint & Meta-Mode MCP Tool Integration (`MCP-41`, `MCP-42`) + +**Files:** +- Modify: `Components/Capabilities/CapabilityEndpoints.cs:425-515` +- Modify: `Core/Routing/ToolRoutingManager.cs:50-80` +- Modify: `Core/Routing/ToolRoutingManager.Execution.cs:64-210` +- Test: `ModelContextGateway.Tests/SemanticSearchEndpointTests.cs` + +**Interfaces:** +- Consumes: `ToolRoutingManager.SearchToolsDetailedAsync` +- Produces: Enhanced `POST /api/test/semantic-search` endpoint response and meta-mode `search_tools` tool execution + +- [ ] **Step 1: Write the failing tests** +Add tests in `ModelContextGateway.Tests/SemanticSearchEndpointTests.cs`: + - `SemanticSearchEndpoint_AcceptsModeAndDenseWeight_ReturnsDetailedScores`: tests `POST /api/test/semantic-search` with `{ query, mode, denseWeight, limit }` returning `{ query, mode, denseWeight, results: [{ tool, toolName, serverId, score, denseScore, sparseScore }] }`. + - `MetaMode_SearchTools_AcceptsModeAndDenseWeight`: tests `tools/call` for `search_tools` with arguments `{ query: "...", mode: "semantic", dense_weight: 0.8 }`. + Annotate tests with `[Requirement("MCP-41", ...)]` and `[Requirement("MCP-42", "MCP", RequirementType.Positive, "Meta-mode search_tools MCP tool accepts optional mode and dense_weight arguments and executes calibrated tool retrieval.")]`. + +- [ ] **Step 2: Run tests to verify they fail** +```bash +dotnet test ModelContextGateway.slnx --filter "FullyQualifiedName~SemanticSearchEndpointTests" +``` + +- [ ] **Step 3: Implement endpoint and MCP tool parameters** + - In `Components/Capabilities/CapabilityEndpoints.cs`: + - Update `SearchModel` (or create `SemanticSearchRequest`) to include `Mode`, `DenseWeight`, `Limit`. + - Route `POST /api/test/semantic-search` through `ToolRoutingManager.SearchToolsDetailedAsync`. + - Return `Results.Ok(new { query = model.Query, mode = model.Mode, denseWeight = model.DenseWeight, results = scoredResults })`. + - In `Core/Routing/ToolRoutingManager.cs`: + - Update `search_tools` declaration `inputSchema` to include optional `mode` (string enum: `["hybrid", "semantic", "lexical"]`) and `dense_weight` (number, range 0.0 to 1.0). + - In `Core/Routing/ToolRoutingManager.Execution.cs`: + - Parse `mode` and `dense_weight` from `params.arguments`. + - Pass to `SearchToolsAsync` / `SearchToolsDetailedAsync`. + +- [ ] **Step 4: Run tests to verify they pass** +```bash +dotnet test ModelContextGateway.slnx --filter "FullyQualifiedName~SemanticSearchEndpointTests" +``` + +- [ ] **Step 5: Commit work** +```bash +git add Components/Capabilities/CapabilityEndpoints.cs Core/Routing/ToolRoutingManager.cs Core/Routing/ToolRoutingManager.Execution.cs ModelContextGateway.Tests/SemanticSearchEndpointTests.cs +git commit -m "feat(api): expose search mode and dense weight in semantic-search API and search_tools MCP tool (MCP-41, MCP-42)" +``` + +--- + +### Task 3: Frontend API & State Updates (`UI-142`) + +**Files:** +- Modify: `frontend/src/shared/types.ts` +- Modify: `frontend/src/api/testbenchApi.ts:60-70` +- Modify: `frontend/src/components/testbench/useTestBenchState.ts:25-90` +- Test: `frontend/src/test/api/testbenchApi.test.ts` + +**Interfaces:** +- Consumes: Backend `/api/test/semantic-search` +- Produces: `ToolSearchResultItem`, `SearchMode`, `semanticSearchApi`, `useTestBenchState` semantic controls + +- [ ] **Step 1: Write the failing tests** +Update `frontend/src/test/api/testbenchApi.test.ts` to test calling `semanticSearchApi` with `query`, `mode`, `denseWeight`, and `limit`, receiving structured `results` with scores. +Annotate test with `@requirement UI-142`. + +- [ ] **Step 2: Run tests to verify they fail** +```bash +npm --prefix frontend test frontend/src/test/api/testbenchApi.test.ts +``` + +- [ ] **Step 3: Implement types, API client, and hook state** + - In `frontend/src/shared/types.ts`: + - Add `SearchMode = 'hybrid' | 'semantic' | 'lexical'`. + - Add `ToolSearchResultItem` interface with `tool`, `toolName`, `serverId`, `score`, `denseScore`, `sparseScore`, `denseRank`, `sparseRank`. + - Add `SemanticSearchResponse` interface. + - In `frontend/src/api/testbenchApi.ts`: + - Update `semanticSearchApi(query: string, mode: SearchMode = 'hybrid', denseWeight: number = 0.5, limit: number = 15): Promise`. + - In `frontend/src/components/testbench/useTestBenchState.ts`: + - Add state: `searchMode`, `setSearchMode`, `denseWeight`, `setDenseWeight`, `searchLimit`, `setSearchLimit`. + - Update `handleSemanticSearch` to pass these options to `semanticSearchApi` and store results. + +- [ ] **Step 4: Run tests to verify they pass** +```bash +npm --prefix frontend test frontend/src/test/api/testbenchApi.test.ts +``` + +- [ ] **Step 5: Commit work** +```bash +git add frontend/src/shared/types.ts frontend/src/api/testbenchApi.ts frontend/src/components/testbench/useTestBenchState.ts frontend/src/test/api/testbenchApi.test.ts +git commit -m "feat(ui): add search mode, dense weight, and limit state to test bench (UI-142)" +``` + +--- + +### Task 4: Frontend UI: Semantic Router Simulator Redesign (`UI-142`, `UI-143`) + +**Files:** +- Modify: `frontend/src/components/testbench/SemanticRouterCard.tsx` +- Modify: `frontend/src/components/testbench/TestBenchView.tsx` +- Modify: `frontend/src/styles/tester.css` +- Test: `frontend/src/test/components/SemanticRouterCard.test.tsx` + +**Interfaces:** +- Consumes: `useTestBenchState` semantic state and `onSelectTool` navigation callback +- Produces: Redesigned `SemanticRouterCard` component with mode selector, split slider, presets, score chips, and "Test Tool" action + +- [ ] **Step 1: Write the failing tests** +Create `frontend/src/test/components/SemanticRouterCard.test.tsx`: + - Verifies mode selector toggles between `hybrid`, `semantic`, and `lexical`. + - Verifies hybrid calibration slider and quick preset chips appear in hybrid mode. + - Verifies preset buttons update `denseWeight` (`50/50`, `70/30`, `30/70`, `100/0`, `0/100`). + - Verifies limit dropdown changes `limit`. + - Verifies score breakdown badges (`Total %`, `Semantic %`, `Keyword %`) render with formatted percentages. + - Verifies clicking **"Test Tool"** dispatches `onSelectTool(serverId, toolName)`. + Annotate tests with `@requirement UI-142` and `@requirement UI-143`. + +- [ ] **Step 2: Run tests to verify they fail** +```bash +npm --prefix frontend test frontend/src/test/components/SemanticRouterCard.test.tsx +``` + +- [ ] **Step 3: Implement `SemanticRouterCard.tsx` & styles** + - Implement 3-way mode toggle: + `` + - Implement hybrid split slider & dynamic label: + `` + `Semantic ${Math.round(denseWeight * 100)}% / Keyword ${Math.round((1 - denseWeight) * 100)}%` + - Implement preset chips (`Balanced (50/50)`, `Semantic Bias (70/30)`, `Keyword Bias (30/70)`, `Pure Semantic (100/0)`, `Pure Keyword (0/100)`). + - Implement limit selector dropdown. + - Implement result hit cards with badges: + `Score: ${(item.score * 100).toFixed(1)}%` + `Semantic: ${(item.denseScore * 100).toFixed(1)}%` + `Keyword: ${(item.sparseScore * 100).toFixed(1)}%` + - Implement "Test Tool" button calling `onSelectTool(item.serverId, item.toolName)`. + - In `TestBenchView.tsx`, pass `onSelectTool = (serverId, toolName) => { setSelectedToolServer(serverId); setSelectedToolName(toolName); setActiveTab('tools'); }`. + - Add styles in `frontend/src/styles/tester.css` using centralized CSS variables. + +- [ ] **Step 4: Run tests to verify they pass** +```bash +npm --prefix frontend test frontend/src/test/components/SemanticRouterCard.test.tsx +``` + +- [ ] **Step 5: Run linter and typecheck** +```bash +npm --prefix frontend run lint +npm --prefix frontend run build +``` + +- [ ] **Step 6: Commit work** +```bash +git add frontend/src/components/testbench/SemanticRouterCard.tsx frontend/src/components/testbench/TestBenchView.tsx frontend/src/styles/tester.css frontend/src/test/components/SemanticRouterCard.test.tsx +git commit -m "feat(ui): redesign semantic router simulator with mode toggle, slider presets, score badges, and tool testing transition (UI-142, UI-143)" +``` + +--- + +### Task 5: Layout Inspector Audit & E2E Tests (`UI-07`, `UI-141`) + +**Files:** +- Modify: `frontend/e2e/layout-inspector.spec.ts` +- Modify: `frontend/e2e/testbench.spec.ts` + +- [ ] **Step 1: Update Playwright tests** + - Add test in `frontend/e2e/testbench.spec.ts` testing the Semantic Router simulator: + - Navigates to `Semantic Router` tab. + - Selects `Semantic (Vector)` mode. + - Selects `Hybrid Fusion` mode, clicks `Semantic Bias (70/30)` preset chip. + - Inputs query "search media" and submits search. + - Verifies score badges appear. + - Clicks "Test Tool" on first hit and verifies browser navigates to `Tools` tab with tool preselected. + - Verify layout inspector in `frontend/e2e/layout-inspector.spec.ts` passes with 0 layout overflows across Desktop, Half-Ultrawide 1440p, and Half-Ultrawide 1080p. + +- [ ] **Step 2: Run Playwright tests** +```bash +npx playwright test --config=frontend/playwright.config.ts +``` + +- [ ] **Step 3: Commit work** +```bash +git add frontend/e2e/layout-inspector.spec.ts frontend/e2e/testbench.spec.ts +git commit -m "test(e2e): add semantic router simulator interaction and multi-viewport layout inspector tests (UI-07, UI-141)" +``` + +--- + +### Task 6: Documentation, Real Screenshots, Version Bump & Full Verification + +**Files:** +- Modify: `README.md` +- Modify: `ARCHITECTURE.md` +- Modify: `docs/features-guide.md` +- Modify: `ModelContextGateway.csproj` +- Modify: `frontend/src/stores/useUserStore.ts` +- Modify: `CHANGELOG.md` +- Generate: `docs/requirements-catalog.json`, `docs/software-requirements-and-test-catalog.md` +- Capture: `docs/assets/semantic-router-simulator.png` (via Puppeteer script) + +- [ ] **Step 1: Update documentation** + - Update `ARCHITECTURE.md` with the calibrated hybrid scoring formula and search modes. + - Update `docs/features-guide.md` with instructions on using the Semantic Router simulator, adjusting the slider, and using presets. +- [ ] **Step 2: Version bump to 5.21.0** + - `ModelContextGateway.csproj`: `5.21.0`, `5.21.0`, `5.21.0`. + - `frontend/src/stores/useUserStore.ts`: fallback version string `'5.21.0'`. + - `CHANGELOG.md`: add release entry for `5.21.0` (feat: calibrated hybrid scoring, search modes, score decomposition, and redesigned semantic router simulator). + - `README.md`: update top-5 release preview table. +- [ ] **Step 3: Regenerate and verify requirements catalog** +```bash +dotnet run --project scripts/CatalogGenerator +dotnet run --project scripts/CatalogGenerator -- --verify-only +``` +- [ ] **Step 4: Run full verification test suites** +```bash +dotnet test ModelContextGateway.slnx +npm --prefix frontend test +npm --prefix frontend run lint +npm --prefix frontend run build +``` +- [ ] **Step 5: Capture live screenshots** + - Build and start container on `net_cloud`: `docker compose up -d --build mcg` + - Run Puppeteer script to capture real screenshot of the redesigned Semantic Router Simulator to `docs/assets/semantic-router-simulator.png`. +- [ ] **Step 6: Commit release bump and documentation** +```bash +git add ModelContextGateway.csproj frontend/src/stores/useUserStore.ts CHANGELOG.md README.md ARCHITECTURE.md docs/ +git commit -m "chore(release): bump version to 5.21.0, update docs, and capture live screenshots" +``` diff --git a/docs/superpowers/specs/2026-10-02-semantic-search-hybrid-calibration-design.md b/docs/superpowers/specs/2026-10-02-semantic-search-hybrid-calibration-design.md new file mode 100644 index 00000000..a21db380 --- /dev/null +++ b/docs/superpowers/specs/2026-10-02-semantic-search-hybrid-calibration-design.md @@ -0,0 +1,178 @@ +# Semantic Search Hybrid Calibration & Scoring Alignment Design Spec + +## 1. Context & Motivation + +Model Context Gateway (MCG) routes natural language queries and tool invocations to backend MCP servers. In meta-mode (`/sse`), the gateway exposes `search_tools` and `execute_tool` to AI agents to prevent context bloat. In the admin dashboard, the **Test Bench** provides a **Semantic Router Simulator** (`SemanticRouterCard.tsx`) to evaluate tool retrieval against simulated natural language user intents. + +Recently, **ContextCortex** (PR #36) addressed hybrid search calibration, dense/sparse weighting, and search inspector UX. Comparing ContextCortex with MCG revealed key opportunities for alignment: +1. **Fixed Fusion Without Tuning**: MCG currently uses Reciprocal Rank Fusion (RRF, $k=60$) with an unweighted 50/50 reciprocal rank sum in `ToolRoutingManager.Search.cs`, or legacy unnormalized cosine+boosting in `SemanticSearchService.cs`. Users cannot adjust the balance between dense vector similarity and lexical keyword matching. +2. **Missing Search Modes**: Search is restricted to hybrid fallback; users and AI agents cannot explicitly request pure semantic (dense vector only) or pure lexical (keyword only) retrieval. +3. **Score Discarding & Opacity**: Search returns a raw list of tool objects, discarding the computed match scores. In the UI, the Semantic Router Simulator only shows `Rank #1`, `Rank #2`, with zero visibility into semantic similarity or keyword scores. +4. **Test Bench Workflow Gap**: When the simulator finds a tool, there is no direct action to transition to the Tool Tester with that tool pre-selected. + +This specification aligns MCG's semantic search and simulator with ContextCortex's calibrated hybrid model, explicit search modes, decomposed score diagnostics, and enhanced test bench UX. + +--- + +## 2. Goals & Non-Goals + +### Goals +- **Configurable Hybrid Calibration**: Support dynamic dense weight $\alpha \in [0.0, 1.0]$ (default $0.5$ balanced) across backend search, HTTP APIs, MCP tools, and Web UI. +- **Explicit Search Modes**: Support `hybrid`, `semantic` (pure dense vector similarity), and `lexical` (pure keyword matching). +- **Normalized Linear Combination Scoring**: Normalize dense cosine similarity and lexical match scores into $[0.0, 1.0]$, computing: + $$\text{score} = \alpha \cdot \text{dense\_score} + (1.0 - \alpha) \cdot \text{sparse\_score}$$ +- **Score Transparency & Diagnostics**: Expose decomposed score components (`score`, `dense_score`, `sparse_score`, `dense_rank`, `sparse_rank`) in search result models, HTTP endpoints, and the UI. +- **Enhanced Test Bench UI (`SemanticRouterCard.tsx`)**: + - 3-way search mode toggle (`Hybrid`, `Semantic`, `Lexical`). + - Interactive hybrid split slider with dynamic label (`Semantic X% / Keyword Y%`) and presets (`Balanced 50/50`, `Semantic Bias 70/30`, `Keyword Bias 30/70`, `Pure Semantic 100/0`, `Pure Keyword 0/100`). + - Result limit selector (`5`, `10`, `15`, `25`). + - Score breakdown badges on result cards (`Total %`, `Semantic %`, `Keyword %`). + - 1-click **"Test Tool"** action button that transitions to the Tools tab and preselects the tool in the Tool Tester form. +- **Meta-Mode MCP Tool Enhancement**: Update `search_tools` to accept optional `mode` and `dense_weight` parameters while remaining 100% backward compatible. +- **Real Screenshots & Documentation**: Capture live screenshots of the updated simulator with real tools and update documentation (`README.md`, `ARCHITECTURE.md`, `docs/features-guide.md`). + +### Non-Goals +- Modifying underlying embedding models or ONNX runtimes. +- Breaking backward compatibility with existing `search_tools(query: "...")` calls. +- Modifying custom tool registry or non-search routing logic. + +--- + +## 3. Architecture & Detailed Design + +### 3.1 Backend Models & Data Structures +In `Core/VectorSearch/ToolSearchResult.cs` (or `Models/Routing/ToolSearchResult.cs`): +```csharp +namespace ModelContextGateway.Core.Routing +{ + public class ToolSearchResult + { + public required object Tool { get; init; } + public required string ToolName { get; init; } + public string? ServerId { get; init; } + public double Score { get; init; } // Fused score: 0.0 to 1.0 + public double? DenseScore { get; init; } // Vector similarity: 0.0 to 1.0 + public double? SparseScore { get; init; } // Normalized lexical score: 0.0 to 1.0 + public int? DenseRank { get; init; } + public int? SparseRank { get; init; } + } +} +``` + +### 3.2 Hybrid Scoring & Weight Calibration +In `Core/Routing/ToolRoutingManager.Search.cs`: +1. **Search Modes**: + - `hybrid`: Computes both lexical and dense scores, then fuses using weight $\alpha = \max(0.0, \min(1.0, \text{denseWeight} ?? 0.5))$. + - `semantic`: Computes dense vector similarity only ($\alpha = 1.0$). + - `lexical`: Computes keyword lexical match only ($\alpha = 0.0$). +2. **Score Normalization & Fusion**: + - **Dense Score**: Cosine similarity between query embedding and tool embedding, clamped to $[0.0, 1.0]$. + - **Sparse Score**: Lexical score calculated via `CalculateLexicalScore`. Scores are min-max normalized across all non-zero candidates in the batch to $[0.0, 1.0]$. If only one candidate matches, normalized score is $1.0$. + - **Combined Score**: + $$\text{FinalScore} = \alpha \cdot \text{DenseScore} + (1.0 - \alpha) \cdot \text{SparseScore}$$ +3. **Methods**: + - `SearchToolsDetailedAsync(...)`: Returns `List` with complete diagnostics. + - `SearchToolsAsync(...)`: Backwards-compatible wrapper returning `List`. + +### 3.3 HTTP API (`POST /api/test/semantic-search`) +In `Components/Capabilities/CapabilityEndpoints.cs`: +- Request schema: + ```json + { + "query": "string (required)", + "mode": "hybrid | semantic | lexical (optional, default hybrid)", + "denseWeight": 0.5, + "limit": 15 + } + ``` +- Response schema: + ```json + { + "query": "search movie", + "mode": "hybrid", + "denseWeight": 0.5, + "results": [ + { + "tool": { "name": "plex__search_media", "description": "..." }, + "toolName": "plex__search_media", + "serverId": "plex", + "score": 0.852, + "denseScore": 0.910, + "sparseScore": 0.794, + "denseRank": 1, + "sparseRank": 2 + } + ] + } + ``` + +### 3.4 MCP Tool `search_tools` in Meta-Mode +In `Core/Routing/ToolRoutingManager.cs` & `Core/Routing/ToolRoutingManager.Execution.cs`: +- Tool schema definition updated in `tools/list`: + - `query` (string, required): Search query or intent description. + - `mode` (string, optional, enum: `["hybrid", "semantic", "lexical"]`): Search mode. + - `dense_weight` (number, optional, range: `0.0` - `1.0`): Weight between vector (1.0) and lexical (0.0). +- Tool execution: + - Extracts optional `mode` and `dense_weight` from `params.arguments`. + - Executes calibrated search and returns matching tools. + +### 3.5 Frontend UI: `SemanticRouterCard.tsx` & Test Bench +1. **Mode Controls**: + - Segmented buttons for `Hybrid Fusion`, `Semantic (Vector)`, and `Lexical (Keyword)`. +2. **Hybrid Calibration Slider**: + - Displayed conditionally when `mode === 'hybrid'`. + - Range input `0.0` to `1.0` (step `0.05`). + - Live label: `Semantic X% / Keyword Y%`. + - Preset chips: + - `Balanced 50/50` + - `Semantic Bias 70/30` + - `Keyword Bias 30/70` + - `Pure Semantic 100/0` + - `Pure Keyword 0/100` +3. **Limit Filter**: + - Dropdown with `5`, `10`, `15`, `25`. +4. **Enhanced Result Cards**: + - Header with tool name and server badge. + - Score badges: + - `Total Score: XX.X%` (accent color) + - `Semantic: XX.X%` (cyan/blue badge) + - `Keyword: XX.X%` (purple badge) + - Formatted tool description with clean markdown rendering. + - **"Test Tool"** button: clicking dispatches `onSelectTool(serverId, toolName)`, switching the Test Bench to the `Tools` tab and preselecting the server & tool in `ToolTesterCard`. + +--- + +## 4. Documentation & Visual Assets Plan + +1. **Features Guide (`docs/features-guide.md`)**: + - Document the calibrated hybrid scoring model, search modes, and slider usage. +2. **Architecture Guide (`ARCHITECTURE.md`)**: + - Detail the normalized linear combination formula and score decomposition. +3. **Real Screenshots (`docs/assets/`)**: + - Capture real screenshots of the Semantic Router Simulator showing the mode selector, hybrid slider with presets, and score badges on live tools. + +--- + +## 5. Testing & Verification Plan + +### Backend Tests (`ModelContextGateway.Tests`) +- `ToolRoutingManagerHybridSearchTests.cs`: + - Test pure semantic mode ($\alpha = 1.0$). + - Test pure lexical mode ($\alpha = 0.0$). + - Test hybrid calibration with varying `denseWeight` ($0.2, 0.5, 0.8$). + - Test score decomposition: verify `DenseScore`, `SparseScore`, `Score` are in $[0.0, 1.0]$. + - Test `POST /api/test/semantic-search` endpoint contract. + - Test `search_tools` MCP execution with custom `mode` and `dense_weight`. + - All tests annotated with formal requirement attributes (`ROUT-XX`, `MCP-XX`). + +### Frontend Tests (`frontend/src/test`) +- `SemanticRouterCard.test.tsx`: + - Verify mode toggle changes mode state. + - Verify hybrid slider and presets adjust `denseWeight`. + - Verify score badges render formatted percentages. + - Verify "Test Tool" button triggers callback with server and tool. +- E2E layout audit (`frontend/e2e/layout-inspector.spec.ts`): + - Audit Semantic Router tab across desktop, half-ultrawide 1440p, and half-ultrawide 1080p viewports. + +### Verification Gate +- Run `CatalogGenerator` and verify zero requirements drift (`--verify-only`). diff --git a/docs/user-guide/test-bench/semantic-search.md b/docs/user-guide/test-bench/semantic-search.md index 5f97f04a..77c12fbd 100644 --- a/docs/user-guide/test-bench/semantic-search.md +++ b/docs/user-guide/test-bench/semantic-search.md @@ -1,60 +1,112 @@ # Semantic Router & Search Simulator -The **Semantic Router Simulator** (`SemanticRouterCard`) in the Test Bench enables you to test and fine-tune how MCG matches natural language intents to backend tools when AI clients call `search_tools` in **Meta-Mode**. +The **Semantic Router Simulator** (`SemanticRouterCard`) in the Test Bench enables you to test, fine-tune, and inspect how MCG matches natural language intents to backend tools when AI clients call `search_tools` in **Meta-Mode**. --- ## 🧠 Simulator Interface & Layout -![Semantic Router Simulator in Action](../../assets/test_bench_view.jpg) +![Semantic Router Simulator in Action](../../assets/semantic_router_simulator.jpg) + +The simulator features: +1. **3-Way Search Mode Toggle**: Switch between **Hybrid Fusion**, **Semantic (Vector)**, and **Lexical (Keyword)** modes. +2. **Interactive Hybrid Split Slider & Presets**: In *Hybrid* mode, dynamically balance dense vector similarity against sparse lexical keyword scores with 5 quick presets: + - **Balanced (50/50)**: Equal 0.5/0.5 weighting between semantic intent and technical keyword matches. + - **Semantic Bias (70/30)**: Prioritizes vector embeddings for conceptual queries. + - **Keyword Bias (30/70)**: Prioritizes exact command, tool name, or parameter token matches. + - **Pure Semantic (100/0)**: Dense vector similarity only. + - **Pure Keyword (0/100)**: Sparse BM25 / token match only. +3. **Result Limit Selector**: Choose `5`, `10`, `15`, or `25` candidate tools. +4. **Decomposed Score Breakdown**: Hit cards display the total combined score alongside individual `Semantic` and `Keyword` percentage badges. +5. **1-Click "Test Tool" Action**: Seamlessly transitions to the **Tools** tab with the selected tool and its upstream server pre-loaded in the Tool Tester. --- -## πŸ” Understanding the Hybrid Scoring Algorithm (Reciprocal Rank Fusion) +## πŸ” Understanding the Calibrated Scoring Model -MCG uses an enterprise-grade **Reciprocal Rank Fusion (RRF, $k=60$)** hybrid ranking pipeline that merges lexical keyword precision with dense vector semantic understanding: +MCG combines hardware-accelerated vector similarity (.NET 10 SIMD `TensorPrimitives.CosineSimilarity`) with normalized lexical matching into a calibrated linear combination: ```mermaid flowchart TD - Query["Client Natural Language Query
search_tools ('restart web proxy')"] --> Split{"Parallel Candidate Ranking"} + Query["Client Natural Language Query
search_tools ('restart web proxy')"] --> Split{"Parallel Candidate Scoring"} - Split --> Lexical["1. Lexical Keyword Engine
BM25 exact phrase & token matches
across tool names, descriptions & tags"] - Split --> Vector["2. SIMD Vector Search Engine
Dense embedding cosine similarity
via .NET 10 hardware SIMD intrinsics"] + Split --> Lexical["1. Lexical Keyword Engine
Token & phrase matches across tool names,
descriptions & schemas; normalized to [0, 1]"] + Split --> Vector["2. SIMD Vector Engine
Dense embedding cosine similarity
via hardware SIMD intrinsics clamped to [0, 1]"] - Lexical --> RRF["3. Reciprocal Rank Fusion (RRF, k=60)
RRF_Score = 1.0/(60 + Rank_lexical) + 1.0/(60 + Rank_vector)"] - Vector --> RRF + Lexical --> Fusion["3. Calibrated Linear Combination
Score = alpha * DenseScore + (1 - alpha) * SparseScore"] + Vector --> Fusion - RRF --> Output["Final Tool Ranking
Top-k namespaced tools & JSON input schemas"] + Fusion --> Output["Final Tool Ranking
Sorted by Score with decomposed diagnostics"] ``` -### Reciprocal Rank Fusion (RRF) Formula -$$RRF\_Score(tool) = \frac{1.0}{60 + Rank_{keyword}(tool)} + \frac{1.0}{60 + Rank_{vector}(tool)}$$ +### Scoring Formula +$$\text{Score} = \alpha \cdot \text{DenseScore} + (1.0 - \alpha) \cdot \text{SparseScore}$$ -* **$Rank_{keyword}$**: Ranked position derived from lexical matches across tool names, documentation descriptions, categories, and JSON Schema argument properties. -* **$Rank_{vector}$**: Ranked position derived from dense embedding cosine similarity using **.NET 10 hardware SIMD intrinsics** (`TensorPrimitives.CosineSimilarity`). -* **Why RRF?**: Unlike linear additive scoring, RRF is immune to semantic scale compression, prevents score saturation, guarantees that exact technical command names rank highest, and gracefully falls back to pure keyword rankings if vector providers are offline. +* **$\alpha$ (`dense_weight`)**: Configurable weight in $[0.0, 1.0]$ (default $0.5$). + - When Mode is `semantic`: $\alpha = 1.0$. + - When Mode is `lexical`: $\alpha = 0.0$. + - When Mode is `hybrid`: $\alpha$ matches the slider setting. +* **$\text{DenseScore}$**: SIMD cosine similarity clamped between $0.0$ and $1.0$. +* **$\text{SparseScore}$**: Min-max normalized lexical match score across candidate tools. --- ## πŸ“‹ How to Run a Simulation -1. **Enter Natural Language Query**: Type a representative prompt or intent (e.g. *"check disk usage"*, *"turn off living room lights"*, or *"reboot proxy"*). -2. **Set Result Limit**: Choose the maximum number of results to return (default: `5`). -3. **Simulate**: Click **Simulate Semantic Search**. -4. **Evaluate Results**: Review the ranked list of matching tools. Each entry displays its calculated composite score, match tier (🟒 High, 🟑 Moderate, βšͺ Low), and description snippet. -5. **Optimize**: If a desired tool does not rank highly, refine the tool's description or add relevant category tags in server settings to boost its semantic relevance. +1. **Select Search Mode**: Choose `Hybrid Fusion`, `Semantic (Vector)`, or `Lexical (Keyword)`. +2. **Calibrate Dense Weight**: In Hybrid mode, adjust the slider or click a preset chip (e.g., `Balanced (50/50)` or `Semantic Bias (70/30)`). +3. **Enter Intent Query**: Type a natural language prompt (e.g. *"restart docker container"* or *"fetch plex media"*). +4. **Select Limit**: Set the maximum tools to inspect (`5`, `10`, `15`, or `25`). +5. **Evaluate Results**: Inspect total score %, semantic %, and keyword % badges, along with formatted tool descriptions. +6. **Test the Tool**: Click **Test Tool** on any hit card to immediately test execution in the Tool Tester form. --- -## πŸ’» API & cURL Invocation - -You can simulate semantic routing via REST API calls to `/api/test/semantic-search`: +## πŸ’» API & Meta-Mode MCP Tool Invocation +### REST API (`POST /api/test/semantic-search`) ```bash curl -X POST http://localhost:8080/api/test/semantic-search \ -H "Authorization: Bearer " \ -H "Content-Type: application/json" \ -d '{ - "query": "restart web proxy container" + "query": "restart web proxy container", + "mode": "hybrid", + "denseWeight": 0.7, + "limit": 15 }' ``` + +Response includes decomposed score diagnostics: +```json +{ + "query": "restart web proxy container", + "mode": "hybrid", + "denseWeight": 0.7, + "results": [ + { + "tool": { "name": "docker__restart_container", "description": "..." }, + "toolName": "docker__restart_container", + "serverId": "docker", + "score": 0.884, + "denseScore": 0.920, + "sparseScore": 0.800, + "denseRank": 1, + "sparseRank": 1 + } + ] +} +``` + +### Meta-Mode MCP Tool (`search_tools`) +AI agents calling `search_tools` can optionally pass `mode` and `dense_weight`: +```json +{ + "name": "search_tools", + "arguments": { + "query": "find proxy tools", + "mode": "hybrid", + "dense_weight": 0.5 + } +} +``` diff --git a/frontend/e2e/layout-inspector.spec.ts b/frontend/e2e/layout-inspector.spec.ts index 7626cd0f..61ffef62 100644 --- a/frontend/e2e/layout-inspector.spec.ts +++ b/frontend/e2e/layout-inspector.spec.ts @@ -115,6 +115,27 @@ test.describe('Dashboard Layout & UX Audit', () => { }, }); } + if (path === '/api/test/semantic-search') { + return route.fulfill({ + json: { + query: 'restart container', + mode: 'hybrid', + denseWeight: 0.7, + results: [ + { + tool: { name: 'mock-docker/docker_restart', description: 'Restart a running container' }, + toolName: 'mock-docker/docker_restart', + serverId: 'mock-docker', + score: 0.885, + denseScore: 0.92, + sparseScore: 0.80, + denseRank: 1, + sparseRank: 1, + }, + ], + }, + }); + } if (path === '/api/logs') { return route.fulfill({ json: [] }); } @@ -821,6 +842,7 @@ test.describe('Dashboard Layout & UX Audit', () => { * @description Audits Test Bench Semantic Router tab on Desktop 1080p and Half-Ultrawide viewports for zero overflow and high UX score. */ test('should pass layout audit on Test Bench Semantic Router tab', async ({ page }) => { + // 1. Audit on 1720x1440 (Half-Ultrawide 1440p) await page.setViewportSize({ width: 1720, height: 1440 }); await page.goto('/'); await page.waitForLoadState('domcontentloaded'); @@ -836,13 +858,23 @@ test.describe('Dashboard Layout & UX Audit', () => { await expect(page.locator('#view-testbench [data-testid="semantic-router-card"]')).toBeVisible(); const inspector = new LayoutInspector(page); - const result = await inspector.audit({ + let result = await inspector.audit({ viewport: { width: 1720, height: 1440 }, includeScreenshot: false, }); expect(result.overflowIssues.length).toBe(0); expect(result.uxScore.totalScore).toBeGreaterThanOrEqual(85); + + // 2. Audit on 1280x1080 (Half-Ultrawide 1080p) + await page.setViewportSize({ width: 1280, height: 1080 }); + result = await inspector.audit({ + viewport: { width: 1280, height: 1080 }, + includeScreenshot: false, + }); + + expect(result.overflowIssues.length).toBe(0); + expect(result.uxScore.totalScore).toBeGreaterThanOrEqual(85); }); /** diff --git a/frontend/e2e/testbench.spec.ts b/frontend/e2e/testbench.spec.ts index cb9d0fa2..5a265644 100644 --- a/frontend/e2e/testbench.spec.ts +++ b/frontend/e2e/testbench.spec.ts @@ -51,4 +51,98 @@ test.describe('Test Bench View Flow', () => { await expect(page.locator('[data-testid="resource-server-select"]')).toBeVisible(); }); + /** + * @requirement UI-141 + * @category UI + * @type PositiveFeature + * @description should interact with semantic router simulator, change modes and presets, and test tool transition + */ + test('should interact with semantic router simulator, change modes and presets, and test tool transition', async ({ page }) => { + await page.route('**/api/test/tools', async (route) => { + return route.fulfill({ + json: [ + { + name: 'mock-docker/docker_restart', + description: 'Restart a running container', + inputSchema: { + type: 'object', + properties: { + container_id: { type: 'string', description: 'Target container ID' }, + }, + required: ['container_id'], + }, + }, + ], + }); + }); + + await page.route('**/api/test/semantic-search', async (route) => { + return route.fulfill({ + json: { + query: 'restart container', + mode: 'hybrid', + denseWeight: 0.7, + results: [ + { + tool: { name: 'mock-docker/docker_restart', description: 'Restart a running container' }, + toolName: 'mock-docker/docker_restart', + serverId: 'mock-docker', + score: 0.885, + denseScore: 0.92, + sparseScore: 0.80, + denseRank: 1, + sparseRank: 1, + }, + ], + }, + }); + }); + + await page.goto('/'); + + // Switch to Test Bench view + const testbenchTab = page.locator('[data-testid="tab-testbench"]'); + await expect(testbenchTab).toBeVisible(); + await testbenchTab.click(); + await expect(page.locator('[data-testid="view-testbench"]')).toBeVisible(); + + // Switch to Semantic Router Tab + await page.locator('[data-testid="testbench-tab-semantic"]').click(); + await expect(page.locator('[data-testid="semantic-router-card"]')).toBeVisible(); + + // Test mode toggling: click semantic mode -> slider disappears + await page.locator('[data-testid="mode-btn-semantic"]').click(); + await expect(page.locator('[data-testid="hybrid-weight-slider"]')).toBeHidden(); + + // Click hybrid mode -> slider reappears + await page.locator('[data-testid="mode-btn-hybrid"]').click(); + await expect(page.locator('[data-testid="hybrid-weight-slider"]')).toBeVisible(); + + // Test preset button: click Semantic Bias -> denseWeight becomes 0.7 + await page.locator('[data-testid="preset-semantic-bias"]').click(); + await expect(page.locator('[data-testid="hybrid-weight-slider"]')).toHaveValue('0.7'); + + // Fill query and execute search + const searchInput = page.locator('[data-testid="semantic-query-input"]'); + await expect(searchInput).toBeVisible(); + await searchInput.fill('restart container'); + await expect(searchInput).toHaveValue('restart container'); + await page.locator('[data-testid="semantic-search-btn"]').click(); + + // Verify score badges appear + await expect(page.locator('[data-testid="score-total"]')).toBeVisible(); + await expect(page.locator('[data-testid="score-dense"]')).toBeVisible(); + await expect(page.locator('[data-testid="score-sparse"]')).toBeVisible(); + + // Click "Test Tool" transition button + const testToolBtn = page.locator('[data-testid="test-tool-btn-mock-docker/docker_restart"]'); + await expect(testToolBtn).toBeVisible(); + await testToolBtn.click(); + + // Verify active tab switches to Tools card with selected server + await expect(page.locator('[data-testid="tool-tester-card"]')).toBeVisible(); + await expect(page.locator('[data-testid="tool-server-select"]')).toHaveValue('mock-docker'); + }); + }); + diff --git a/frontend/package.json b/frontend/package.json index aa7c2e47..2aebc19f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "model-context-gateway-ui", "private": true, - "version": "5.20.1", + "version": "5.21.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/api/testbenchApi.ts b/frontend/src/api/testbenchApi.ts index 6b0cab9d..7b407250 100644 --- a/frontend/src/api/testbenchApi.ts +++ b/frontend/src/api/testbenchApi.ts @@ -1,5 +1,5 @@ import { apiRequest } from '../shared/api/api'; -import { ToolItem, PromptItem, ResourcesData, LogEntry } from '../shared/types'; +import { ToolItem, PromptItem, ResourcesData, LogEntry, SearchMode, SemanticSearchResponse } from '../shared/types'; export async function fetchTestToolsApi(): Promise { const data = await apiRequest('/api/test/tools'); @@ -59,10 +59,18 @@ export async function readResourceApi(serverId: string, uri: string): Promise { - const data = await apiRequest('/api/test/semantic-search', { +export async function semanticSearchApi( + query: string, + mode: SearchMode = 'hybrid', + denseWeight: number = 0.5, + limit: number = 15 +): Promise { + const data = await apiRequest('/api/test/semantic-search', { method: 'POST', - body: { query } + body: { query, mode, denseWeight, limit } }); - return data || []; + if (Array.isArray(data)) { + return { query, mode, denseWeight, results: data as any }; + } + return data || { query, mode, denseWeight, results: [] }; } diff --git a/frontend/src/components/testbench/FormattedDescription.tsx b/frontend/src/components/testbench/FormattedDescription.tsx index 8b4595a6..ccc54d05 100644 --- a/frontend/src/components/testbench/FormattedDescription.tsx +++ b/frontend/src/components/testbench/FormattedDescription.tsx @@ -4,16 +4,18 @@ import { cleanDescription, renderInline, parseBlocks } from './descriptionUtils' interface FormattedDescriptionProps { text?: string; + description?: string; serverId?: string; className?: string; } export const FormattedDescription: React.FC = ({ text, + description, serverId, className, }) => { - const cleaned = cleanDescription(text, serverId); + const cleaned = cleanDescription(text ?? description, serverId); if (!cleaned) return null; const blocks = parseBlocks(cleaned); diff --git a/frontend/src/components/testbench/SemanticRouterCard.tsx b/frontend/src/components/testbench/SemanticRouterCard.tsx index a29a3d63..4c6b890f 100644 --- a/frontend/src/components/testbench/SemanticRouterCard.tsx +++ b/frontend/src/components/testbench/SemanticRouterCard.tsx @@ -1,58 +1,244 @@ import React from 'react'; -import { cleanDescription } from './descriptionUtils'; +import { SearchMode, ToolSearchResultItem } from '../../shared/types'; +import { FormattedDescription } from './FormattedDescription'; -interface SemanticRouterCardProps { +export interface SemanticRouterCardProps { semanticQuery: string; - semanticResults: any[]; + semanticResults: ToolSearchResultItem[]; isSearchingSemantic: boolean; + searchMode: SearchMode; + denseWeight: number; + searchLimit: number; onQueryChange: (q: string) => void; + onModeChange: (mode: SearchMode) => void; + onDenseWeightChange: (weight: number) => void; + onSearchLimitChange: (limit: number) => void; onSearch: () => void; + onSelectTool?: (serverId: string, toolName: string) => void; } export const SemanticRouterCard: React.FC = ({ semanticQuery, semanticResults, isSearchingSemantic, + searchMode, + denseWeight, + searchLimit, onQueryChange, + onModeChange, + onDenseWeightChange, + onSearchLimitChange, onSearch, + onSelectTool, }) => { return (

Semantic Router Simulator

-
- - onQueryChange(e.target.value)} - /> + + {/* 3-way Search Mode Toggle */} +
+ + + +
+ + {/* Hybrid Split Slider & Quick Presets (Only in Hybrid Mode) */} + {searchMode === 'hybrid' && ( +
+
+ + + Semantic {Math.round(denseWeight * 100)}% / Keyword {Math.round((1 - denseWeight) * 100)}% + +
+ onDenseWeightChange(parseFloat(e.target.value))} + data-testid="hybrid-weight-slider" + /> +
+ + + + + +
+
+ )} + + {/* Query input and Limit selector */} +
+
+ + onQueryChange(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + onSearch(); + } + }} + /> +
+
+ + +
- + {/* Search Hit Cards */}
{semanticResults.length === 0 ? (
{isSearchingSemantic ? 'Searching...' : 'Enter a prompt query to test tool matching scoring.'}
) : ( - semanticResults.map((tool: any, idx: number) => ( -
-
- {tool.name} - Rank #{idx + 1} + semanticResults.map((item: ToolSearchResultItem, idx: number) => { + const toolName = item.toolName || item.tool?.name || (item as any).name || `tool-${idx}`; + const serverId = item.serverId || (item as any).server || ''; + const desc = item.tool?.description || (item as any).description || ''; + const totalScore = typeof item.score === 'number' ? item.score : 0; + + return ( +
+
+
+ {toolName} + {serverId && {serverId}} +
+
+ + Score: {(totalScore * 100).toFixed(1)}% + + {searchMode === 'hybrid' && ( + <> + {typeof item.denseScore === 'number' && ( + + Semantic: {(item.denseScore * 100).toFixed(1)}% + + )} + {typeof item.sparseScore === 'number' && ( + + Keyword: {(item.sparseScore * 100).toFixed(1)}% + + )} + + )} +
+
+ +
+ {desc ? ( + + ) : ( + No description provided. + )} +
+ +
+ +
- {cleanDescription(tool.description) || 'No description provided.'} -
- )) + ); + }) )}
diff --git a/frontend/src/components/testbench/TestBenchView.tsx b/frontend/src/components/testbench/TestBenchView.tsx index 35308e7f..e1e71986 100644 --- a/frontend/src/components/testbench/TestBenchView.tsx +++ b/frontend/src/components/testbench/TestBenchView.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { executeToolApi, getPromptApi, readResourceApi, semanticSearchApi } from '../../api/testbenchApi'; +import { executeToolApi, getPromptApi, readResourceApi } from '../../api/testbenchApi'; import { showToast } from '../../stores/useToastStore'; import { useTestBenchState } from './useTestBenchState'; @@ -27,8 +27,12 @@ export const TestBenchView: React.FC = () => { selectedResourceUri, setSelectedResourceUri, selectedResourceValue, setSelectedResourceValue, semanticQuery, setSemanticQuery, - semanticResults, setSemanticResults, - isSearchingSemantic, setIsSearchingSemantic, + searchMode, setSearchMode, + denseWeight, setDenseWeight, + searchLimit, setSearchLimit, + semanticResults, + isSearchingSemantic, + handleSemanticSearch, consoleRequest, setConsoleRequest, consoleResponse, setConsoleResponse } = useTestBenchState(); @@ -199,21 +203,10 @@ export const TestBenchView: React.FC = () => { } }; - // Semantic - const runSemanticSearch = async (e?: React.FormEvent) => { - if (e && e.preventDefault) e.preventDefault(); - if (!semanticQuery.trim()) return; - - setIsSearchingSemantic(true); - try { - const results = await semanticSearchApi(semanticQuery.trim()); - setSemanticResults(results || []); - } catch (err: any) { - showToast(`Semantic search failed: ${err.message}`, 'error'); - setSemanticResults([]); - } finally { - setIsSearchingSemantic(false); - } + const handleSelectTool = (serverId: string, toolName: string) => { + setSelectedToolServer(serverId); + handleToolNameChange(toolName); + setActiveTab('tools'); }; return ( @@ -261,8 +254,15 @@ export const TestBenchView: React.FC = () => { semanticQuery={semanticQuery} semanticResults={semanticResults} isSearchingSemantic={isSearchingSemantic} + searchMode={searchMode} + denseWeight={denseWeight} + searchLimit={searchLimit} onQueryChange={setSemanticQuery} - onSearch={runSemanticSearch} + onModeChange={setSearchMode} + onDenseWeightChange={setDenseWeight} + onSearchLimitChange={setSearchLimit} + onSearch={handleSemanticSearch} + onSelectTool={handleSelectTool} />
diff --git a/frontend/src/components/testbench/useTestBenchState.ts b/frontend/src/components/testbench/useTestBenchState.ts index f8e843d8..67fa7461 100644 --- a/frontend/src/components/testbench/useTestBenchState.ts +++ b/frontend/src/components/testbench/useTestBenchState.ts @@ -1,6 +1,6 @@ import { useState, useEffect } from 'react'; -import { ToolItem, PromptItem, ResourceItem, TemplateItem } from '../../shared/types'; -import { fetchTestToolsApi, fetchTestPromptsApi, fetchTestResourcesApi } from '../../api/testbenchApi'; +import { ToolItem, PromptItem, ResourceItem, TemplateItem, SearchMode, ToolSearchResultItem } from '../../shared/types'; +import { fetchTestToolsApi, fetchTestPromptsApi, fetchTestResourcesApi, semanticSearchApi } from '../../api/testbenchApi'; export const useTestBenchState = () => { const [activeTab, setActiveTab] = useState<'tools' | 'prompts' | 'resources' | 'semantic'>('tools'); @@ -28,13 +28,30 @@ export const useTestBenchState = () => { // Semantic const [semanticQuery, setSemanticQuery] = useState(''); - const [semanticResults, setSemanticResults] = useState([]); + const [searchMode, setSearchMode] = useState('hybrid'); + const [denseWeight, setDenseWeight] = useState(0.5); + const [searchLimit, setSearchLimit] = useState(15); + const [semanticResults, setSemanticResults] = useState([]); const [isSearchingSemantic, setIsSearchingSemantic] = useState(false); // Console const [consoleRequest, setConsoleRequest] = useState('Ready'); const [consoleResponse, setConsoleResponse] = useState('Waiting for execution...'); + const handleSemanticSearch = async () => { + if (!semanticQuery.trim()) return; + setIsSearchingSemantic(true); + try { + const res = await semanticSearchApi(semanticQuery, searchMode, denseWeight, searchLimit); + setSemanticResults(res?.results || []); + } catch (err) { + console.error('Semantic search failed:', err); + setSemanticResults([]); + } finally { + setIsSearchingSemantic(false); + } + }; + useEffect(() => { let ignore = false; const fetchInitialData = async () => { @@ -82,9 +99,14 @@ export const useTestBenchState = () => { selectedResourceUri, setSelectedResourceUri, selectedResourceValue, setSelectedResourceValue, semanticQuery, setSemanticQuery, + searchMode, setSearchMode, + denseWeight, setDenseWeight, + searchLimit, setSearchLimit, semanticResults, setSemanticResults, isSearchingSemantic, setIsSearchingSemantic, + handleSemanticSearch, consoleRequest, setConsoleRequest, consoleResponse, setConsoleResponse }; }; + diff --git a/frontend/src/shared/stores/useUserStore.ts b/frontend/src/shared/stores/useUserStore.ts index 0f1bd5a5..5f21d735 100644 --- a/frontend/src/shared/stores/useUserStore.ts +++ b/frontend/src/shared/stores/useUserStore.ts @@ -13,7 +13,7 @@ export interface UserStore { export const useUserStore = create((set) => ({ user: null, - version: '5.20.1', // fallback default + version: '5.21.0', // fallback default service: 'ModelContextGateway', isLoadingUser: false, loadUser: async () => { diff --git a/frontend/src/shared/types/testbench.ts b/frontend/src/shared/types/testbench.ts index 74277b84..8b3c18e3 100644 --- a/frontend/src/shared/types/testbench.ts +++ b/frontend/src/shared/types/testbench.ts @@ -44,3 +44,23 @@ export interface LogEntry { message: string; exception?: string; } + +export type SearchMode = 'hybrid' | 'semantic' | 'lexical'; + +export interface ToolSearchResultItem { + tool: ToolItem; + toolName: string; + serverId?: string; + score: number; + denseScore?: number; + sparseScore?: number; + denseRank?: number; + sparseRank?: number; +} + +export interface SemanticSearchResponse { + query: string; + mode: SearchMode; + denseWeight: number; + results: ToolSearchResultItem[]; +} diff --git a/frontend/src/styles/tester.css b/frontend/src/styles/tester.css index 50821717..74af9433 100644 --- a/frontend/src/styles/tester.css +++ b/frontend/src/styles/tester.css @@ -172,13 +172,139 @@ display: block; } +/* Semantic Router Mode Selector & Controls */ +.search-mode-selector { + display: flex; + gap: var(--space-2); + margin-bottom: var(--space-4); + flex-wrap: wrap; +} + +.mode-toggle-btn { + background: var(--bg-input); + border: 1px solid var(--border-color); + color: var(--text-muted); + padding: var(--space-2) var(--space-4); + border-radius: var(--radius-sm); + font-size: var(--font-size-xs); + font-weight: var(--font-weight-semibold); + cursor: pointer; + display: inline-flex; + align-items: center; + gap: var(--space-2); + box-sizing: border-box; + transition: all var(--transition-fast); +} + +.mode-toggle-btn:hover { + color: var(--text-main); + border-color: var(--border-hover); +} + +.mode-toggle-btn.active { + background: rgb(var(--primary-rgb) / 0.15); + border-color: var(--primary); + color: var(--primary); +} + +/* Hybrid Split Slider Container & Presets */ +.hybrid-slider-container { + background: var(--bg-input); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + padding: var(--space-3) var(--space-4); + margin-bottom: var(--space-4); + display: flex; + flex-direction: column; + gap: var(--space-2); +} + +.hybrid-slider-header { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: var(--space-2); +} + +.hybrid-slider-header label { + font-size: var(--font-size-xs); + font-weight: var(--font-weight-semibold); + color: var(--text-main); +} + +.hybrid-slider-ratio { + font-family: var(--font-family-code); + font-size: var(--font-size-xs); + color: var(--accent); +} + +.hybrid-slider-container input[type="range"] { + width: 100%; + accent-color: var(--primary); + cursor: pointer; + margin: 4px 0; + min-height: 24px; + height: 24px; +} + +.slider-presets { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 4px; +} + +.preset-chip { + background: var(--bg-card); + border: 1px solid var(--border-color); + color: var(--text-muted); + font-size: var(--font-size-xs); + padding: 3px 10px; + border-radius: var(--radius-full); + cursor: pointer; + box-sizing: border-box; + transition: all var(--transition-fast); + min-height: 24px; + display: inline-flex; + align-items: center; + justify-content: center; +} + +.preset-chip:hover { + color: var(--text-main); + border-color: var(--border-hover); +} + +.preset-chip.active { + border-color: var(--accent); + color: var(--accent); + background: rgb(var(--accent-rgb) / 0.1); +} + +/* Query Controls (Input + Limit Select) */ +.semantic-query-controls { + display: flex; + gap: var(--space-3); + align-items: flex-start; +} + +.semantic-query-controls .query-input-group { + flex: 1; +} + +.semantic-query-controls .limit-select-group { + width: 90px; + flex-shrink: 0; +} + /* Semantic Search Tester Results */ .semantic-search-results { display: flex; flex-direction: column; - gap: var(--space-2); + gap: var(--space-3); margin-top: var(--space-4); - max-height: 300px; + max-height: 400px; overflow-y: auto; padding-right: 5px; } @@ -190,13 +316,22 @@ padding: var(--space-3) var(--space-4); display: flex; flex-direction: column; - gap: 4px; + gap: 6px; } .search-result-header { display: flex; justify-content: space-between; align-items: center; + flex-wrap: wrap; + gap: var(--space-2); +} + +.search-result-title-group { + display: flex; + align-items: center; + gap: var(--space-2); + flex-wrap: wrap; } .search-result-name { @@ -204,20 +339,79 @@ font-size: var(--font-size-sm); } -.search-result-score { +.search-result-scores { + display: flex; + align-items: center; + gap: 6px; + flex-wrap: wrap; +} + +.badge-server { + background: rgb(var(--color-info-rgb) / 0.15); + color: var(--color-info); + border: 1px solid rgb(var(--color-info-rgb) / 0.3); font-family: var(--font-family-code); - font-size: var(--font-size-xs); + font-size: 11px; + padding: 2px 6px; + border-radius: var(--radius-sm); +} + +.badge-score-total { + background: rgb(var(--primary-rgb) / 0.15); + color: var(--primary); + border: 1px solid rgb(var(--primary-rgb) / 0.3); + font-family: var(--font-family-code); + font-size: 11px; + padding: 2px 6px; + border-radius: var(--radius-sm); +} + +.badge-score-dense { + background: rgb(var(--color-purple-rgb) / 0.15); + color: var(--color-purple); + border: 1px solid rgb(var(--color-purple-rgb) / 0.3); + font-family: var(--font-family-code); + font-size: 11px; + padding: 2px 6px; + border-radius: var(--radius-sm); +} + +.badge-score-sparse { + background: rgb(var(--accent-rgb) / 0.15); color: var(--accent); - background: rgb(var(--color-success-rgb) / 0.1); + border: 1px solid rgb(var(--accent-rgb) / 0.3); + font-family: var(--font-family-code); + font-size: 11px; padding: 2px 6px; border-radius: var(--radius-sm); } +.search-result-body { + margin-top: 2px; +} + .search-result-desc { font-size: var(--font-size-xs); color: var(--text-muted); } +.search-result-actions { + display: flex; + justify-content: flex-end; + margin-top: 4px; +} + +.test-tool-btn { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: var(--font-size-xs); + padding: 4px 10px; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all var(--transition-fast); +} + /* Full Width span for bottom grid row (System Logs) */ .tester-full-width { grid-column: 1 / -1; diff --git a/frontend/src/test/api/testbenchApi.test.ts b/frontend/src/test/api/testbenchApi.test.ts new file mode 100644 index 00000000..b6c20877 --- /dev/null +++ b/frontend/src/test/api/testbenchApi.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { semanticSearchApi } from '../../api/testbenchApi'; +import * as api from '../../shared/api/api'; +import { SemanticSearchResponse } from '../../shared/types/testbench'; + +describe('testbenchApi', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description Verifies semantic search API client passes searchMode, denseWeight, and limit to backend. + */ + it('calls apiRequest with query, mode, denseWeight, and limit parameters', async () => { + const mockResponse: SemanticSearchResponse = { + query: 'docker list containers', + mode: 'hybrid', + denseWeight: 0.7, + results: [ + { + tool: { name: 'docker__list_containers', description: 'List containers' }, + toolName: 'docker__list_containers', + serverId: 'docker', + score: 0.88, + denseScore: 0.92, + sparseScore: 0.81, + denseRank: 1, + sparseRank: 2, + }, + ], + }; + + const spy = vi.spyOn(api, 'apiRequest').mockResolvedValue(mockResponse); + + const result = await semanticSearchApi('docker list containers', 'hybrid', 0.7, 10); + + expect(spy).toHaveBeenCalledWith('/api/test/semantic-search', { + method: 'POST', + body: { + query: 'docker list containers', + mode: 'hybrid', + denseWeight: 0.7, + limit: 10, + }, + }); + + expect(result).toEqual(mockResponse); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description Verifies semantic search API client uses default arguments for mode, denseWeight, and limit. + */ + it('uses default values for mode, denseWeight, and limit when not specified', async () => { + const mockResponse: SemanticSearchResponse = { + query: 'status', + mode: 'hybrid', + denseWeight: 0.5, + results: [], + }; + + const spy = vi.spyOn(api, 'apiRequest').mockResolvedValue(mockResponse); + + const result = await semanticSearchApi('status'); + + expect(spy).toHaveBeenCalledWith('/api/test/semantic-search', { + method: 'POST', + body: { + query: 'status', + mode: 'hybrid', + denseWeight: 0.5, + limit: 15, + }, + }); + + expect(result).toEqual(mockResponse); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description Returns fallback object when apiRequest returns null or undefined. + */ + it('returns fallback object when apiRequest returns null or undefined', async () => { + vi.spyOn(api, 'apiRequest').mockResolvedValue(null as any); + + const result = await semanticSearchApi('test', 'semantic', 0.8, 5); + + expect(result).toEqual({ + query: 'test', + mode: 'semantic', + denseWeight: 0.8, + results: [], + }); + }); +}); diff --git a/frontend/src/test/api/typedApi.test.ts b/frontend/src/test/api/typedApi.test.ts index 527c27c1..a633ad55 100644 --- a/frontend/src/test/api/typedApi.test.ts +++ b/frontend/src/test/api/typedApi.test.ts @@ -277,9 +277,14 @@ describe('Typed API Client Layer', () => { const resResult = await readResourceApi('srv1', 'mcp://srv1/res'); expect(resResult.content).toBe('res_output'); - mockApiResponse('/api/test/semantic-search', [{ name: 'tool1', score: 0.9 }]); + mockApiResponse('/api/test/semantic-search', { + query: 'query', + mode: 'hybrid', + denseWeight: 0.5, + results: [{ toolName: 'tool1', score: 0.9 }], + }); const searchRes = await semanticSearchApi('query'); - expect(searchRes).toHaveLength(1); + expect(searchRes.results).toHaveLength(1); }); }); }); diff --git a/frontend/src/test/components/SemanticRouterCard.test.tsx b/frontend/src/test/components/SemanticRouterCard.test.tsx new file mode 100644 index 00000000..42701ab1 --- /dev/null +++ b/frontend/src/test/components/SemanticRouterCard.test.tsx @@ -0,0 +1,317 @@ +import { describe, it, expect, vi } from 'vitest'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { SemanticRouterCard } from '../../components/testbench/SemanticRouterCard'; +import { ToolSearchResultItem } from '../../shared/types'; + +describe('SemanticRouterCard Component', () => { + const sampleResults: ToolSearchResultItem[] = [ + { + tool: { + name: 'docker__list_containers', + description: 'List running Docker containers', + }, + toolName: 'docker__list_containers', + serverId: 'docker', + score: 0.8765, + denseScore: 0.9123, + sparseScore: 0.8407, + denseRank: 1, + sparseRank: 2, + }, + { + tool: { + name: 'native_calc', + description: 'Perform basic math operations', + }, + toolName: 'native_calc', + score: 0.6543, + denseScore: 0.7, + sparseScore: 0.6, + denseRank: 2, + sparseRank: 3, + }, + ]; + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes + */ + it('verifies search mode toggle renders buttons and switches between hybrid, semantic, and lexical modes', () => { + const onModeChange = vi.fn(); + + const { rerender } = render( + + ); + + const hybridBtn = screen.getByTestId('mode-btn-hybrid'); + const semanticBtn = screen.getByTestId('mode-btn-semantic'); + const lexicalBtn = screen.getByTestId('mode-btn-lexical'); + + expect(hybridBtn).toBeInTheDocument(); + expect(semanticBtn).toBeInTheDocument(); + expect(lexicalBtn).toBeInTheDocument(); + + expect(hybridBtn).toHaveClass('active'); + expect(semanticBtn).not.toHaveClass('active'); + expect(lexicalBtn).not.toHaveClass('active'); + + fireEvent.click(semanticBtn); + expect(onModeChange).toHaveBeenCalledWith('semantic'); + + fireEvent.click(lexicalBtn); + expect(onModeChange).toHaveBeenCalledWith('lexical'); + + rerender( + + ); + + expect(hybridBtn).not.toHaveClass('active'); + expect(lexicalBtn).toHaveClass('active'); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description verifies hybrid split slider and quick preset chips appear in hybrid mode and update denseWeight + */ + it('verifies hybrid split slider and quick preset chips appear in hybrid mode and update denseWeight', () => { + const onDenseWeightChange = vi.fn(); + + const { rerender } = render( + + ); + + expect(screen.getByText(/Dense \/ Lexical Balance/i)).toBeInTheDocument(); + expect(screen.getByText(/Semantic 70% \/ Keyword 30%/i)).toBeInTheDocument(); + + const slider = screen.getByTestId('hybrid-weight-slider'); + expect(slider).toBeInTheDocument(); + expect(slider).toHaveValue('0.7'); + + fireEvent.change(slider, { target: { value: '0.4' } }); + expect(onDenseWeightChange).toHaveBeenCalledWith(0.4); + + // Test quick preset chips + const presetBalanced = screen.getByTestId('preset-balanced'); + const presetSemanticBias = screen.getByTestId('preset-semantic-bias'); + const presetKeywordBias = screen.getByTestId('preset-keyword-bias'); + const presetPureSemantic = screen.getByTestId('preset-pure-semantic'); + const presetPureKeyword = screen.getByTestId('preset-pure-keyword'); + + fireEvent.click(presetBalanced); + expect(onDenseWeightChange).toHaveBeenCalledWith(0.5); + + fireEvent.click(presetSemanticBias); + expect(onDenseWeightChange).toHaveBeenCalledWith(0.7); + + fireEvent.click(presetKeywordBias); + expect(onDenseWeightChange).toHaveBeenCalledWith(0.3); + + fireEvent.click(presetPureSemantic); + expect(onDenseWeightChange).toHaveBeenCalledWith(1.0); + + fireEvent.click(presetPureKeyword); + expect(onDenseWeightChange).toHaveBeenCalledWith(0.0); + + // Verify slider & presets hidden in non-hybrid mode + rerender( + + ); + + expect(screen.queryByTestId('hybrid-weight-slider')).not.toBeInTheDocument(); + expect(screen.queryByTestId('preset-balanced')).not.toBeInTheDocument(); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description verifies search limit dropdown changes value + */ + it('verifies search limit dropdown changes value', () => { + const onSearchLimitChange = vi.fn(); + + render( + + ); + + const limitSelect = screen.getByTestId('search-limit-select') as HTMLSelectElement; + expect(limitSelect).toBeInTheDocument(); + expect(limitSelect.value).toBe('15'); + + fireEvent.change(limitSelect, { target: { value: '25' } }); + expect(onSearchLimitChange).toHaveBeenCalledWith(25); + }); + + /** + * @requirement UI-143 + * @category UI + * @type PositiveFeature + * @description renders decomposed score badges with formatted percentages in hybrid mode + */ + it('renders decomposed score badges with formatted percentages in hybrid mode', () => { + render( + + ); + + const scoreTotals = screen.getAllByTestId('score-total'); + expect(scoreTotals).toHaveLength(2); + expect(scoreTotals[0]).toHaveTextContent('Score: 87.6%'); + expect(scoreTotals[1]).toHaveTextContent('Score: 65.4%'); + + const scoreDenses = screen.getAllByTestId('score-dense'); + expect(scoreDenses).toHaveLength(2); + expect(scoreDenses[0]).toHaveTextContent('Semantic: 91.2%'); + expect(scoreDenses[1]).toHaveTextContent('Semantic: 70.0%'); + + const scoreSparses = screen.getAllByTestId('score-sparse'); + expect(scoreSparses).toHaveLength(2); + expect(scoreSparses[0]).toHaveTextContent('Keyword: 84.1%'); + expect(scoreSparses[1]).toHaveTextContent('Keyword: 60.0%'); + + // Server badge when present + expect(screen.getByText('docker')).toHaveClass('badge', 'badge-server'); + }); + + /** + * @requirement UI-143 + * @category UI + * @type PositiveFeature + * @description hides dense and sparse badges in pure semantic or lexical modes + */ + it('hides dense and sparse badges in pure semantic or lexical modes', () => { + render( + + ); + + expect(screen.getAllByTestId('score-total')).toHaveLength(2); + expect(screen.queryByTestId('score-dense')).not.toBeInTheDocument(); + expect(screen.queryByTestId('score-sparse')).not.toBeInTheDocument(); + }); + + /** + * @requirement UI-143 + * @category UI + * @type PositiveFeature + * @description clicking Test Tool button dispatches onSelectTool with serverId and toolName + */ + it('clicking Test Tool button dispatches onSelectTool with serverId and toolName', () => { + const onSelectTool = vi.fn(); + + render( + + ); + + const testToolBtn1 = screen.getByTestId('test-tool-btn-docker__list_containers'); + expect(testToolBtn1).toBeInTheDocument(); + + fireEvent.click(testToolBtn1); + expect(onSelectTool).toHaveBeenCalledWith('docker', 'docker__list_containers'); + + const testToolBtn2 = screen.getByTestId('test-tool-btn-native_calc'); + expect(testToolBtn2).toBeInTheDocument(); + + fireEvent.click(testToolBtn2); + expect(onSelectTool).toHaveBeenCalledWith('', 'native_calc'); + }); +}); diff --git a/frontend/src/test/components/TestBenchView.test.tsx b/frontend/src/test/components/TestBenchView.test.tsx index fe30683f..7e5482b4 100644 --- a/frontend/src/test/components/TestBenchView.test.tsx +++ b/frontend/src/test/components/TestBenchView.test.tsx @@ -508,4 +508,54 @@ describe('TestBenchView Component', () => { expect(screen.getByTestId('semantic-query-input')).toBeInTheDocument(); expect(screen.getByTestId('semantic-search-btn')).toBeInTheDocument(); }); + + /** + * @requirement UI-143 + * @category UI + * @type PositiveFeature + * @description clicking Test Tool in semantic search results switches to Tools tab with tool and server preselected + */ + it('clicking Test Tool in semantic search results switches to Tools tab with tool and server preselected', async () => { + vi.spyOn(api, 'apiRequest').mockResolvedValue({ + query: 'containers', + mode: 'hybrid', + denseWeight: 0.5, + results: [ + { + tool: { name: 'docker__list_containers', description: 'List containers' }, + toolName: 'docker__list_containers', + serverId: 'docker', + score: 0.95, + }, + ], + }); + + render(); + + const semanticTab = screen.getByRole('button', { name: /Semantic Router/i }); + fireEvent.click(semanticTab); + + const searchInput = screen.getByTestId('semantic-query-input'); + fireEvent.change(searchInput, { target: { value: 'containers' } }); + + const searchBtn = screen.getByTestId('semantic-search-btn'); + fireEvent.click(searchBtn); + + await waitFor(() => { + expect(screen.getByTestId('test-tool-btn-docker__list_containers')).toBeInTheDocument(); + }); + + const testToolBtn = screen.getByTestId('test-tool-btn-docker__list_containers'); + fireEvent.click(testToolBtn); + + await waitFor(() => { + expect(screen.getByTestId('tool-tester-card')).toBeInTheDocument(); + }); + + const serverSelect = screen.getByLabelText('Server') as HTMLSelectElement; + expect(serverSelect.value).toBe('docker'); + + const toolSelect = screen.getByLabelText('Tool') as HTMLSelectElement; + expect(toolSelect.value).toBe('docker__list_containers'); + }); }); diff --git a/frontend/src/test/components/useTestBenchState.test.ts b/frontend/src/test/components/useTestBenchState.test.ts new file mode 100644 index 00000000..502e22c2 --- /dev/null +++ b/frontend/src/test/components/useTestBenchState.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { renderHook, act, waitFor } from '@testing-library/react'; +import { useTestBenchState } from '../../components/testbench/useTestBenchState'; +import * as testbenchApi from '../../api/testbenchApi'; + +describe('useTestBenchState', () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(testbenchApi, 'fetchTestToolsApi').mockResolvedValue([]); + vi.spyOn(testbenchApi, 'fetchTestPromptsApi').mockResolvedValue([]); + vi.spyOn(testbenchApi, 'fetchTestResourcesApi').mockResolvedValue({ resources: [], templates: [] }); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description Initializes semantic search state with default mode, weight, limit, and empty results. + */ + it('initializes semantic search state with defaults', async () => { + const { result } = renderHook(() => useTestBenchState()); + + await waitFor(() => { + expect(result.current.tools).toEqual([]); + }); + + expect(result.current.searchMode).toBe('hybrid'); + expect(result.current.denseWeight).toBe(0.5); + expect(result.current.searchLimit).toBe(15); + expect(result.current.semanticResults).toEqual([]); + expect(result.current.isSearchingSemantic).toBe(false); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description Updates searchMode, denseWeight, and searchLimit via their respective state setters. + */ + it('updates searchMode, denseWeight, and searchLimit state', async () => { + const { result } = renderHook(() => useTestBenchState()); + + await waitFor(() => { + expect(result.current.tools).toEqual([]); + }); + + act(() => { + result.current.setSearchMode('semantic'); + result.current.setDenseWeight(0.85); + result.current.setSearchLimit(25); + }); + + expect(result.current.searchMode).toBe('semantic'); + expect(result.current.denseWeight).toBe(0.85); + expect(result.current.searchLimit).toBe(25); + }); + + /** + * @requirement UI-142 + * @category UI + * @type PositiveFeature + * @description Executes semantic search using state parameters and updates semanticResults. + */ + it('executes handleSemanticSearch using current state parameters', async () => { + const mockResults = [ + { + tool: { name: 'docker__run', description: 'Run container' }, + toolName: 'docker__run', + serverId: 'docker', + score: 0.91, + denseScore: 0.95, + sparseScore: 0.82, + denseRank: 1, + sparseRank: 2, + }, + ]; + + const searchSpy = vi.spyOn(testbenchApi, 'semanticSearchApi').mockResolvedValue({ + query: 'deploy container', + mode: 'hybrid', + denseWeight: 0.7, + results: mockResults, + }); + + const { result } = renderHook(() => useTestBenchState()); + + act(() => { + result.current.setSemanticQuery('deploy container'); + result.current.setDenseWeight(0.7); + result.current.setSearchLimit(10); + }); + + await act(async () => { + await result.current.handleSemanticSearch(); + }); + + expect(searchSpy).toHaveBeenCalledWith('deploy container', 'hybrid', 0.7, 10); + expect(result.current.semanticResults).toEqual(mockResults); + expect(result.current.isSearchingSemantic).toBe(false); + }); + + /** + * @requirement UI-142 + * @category UI + * @type FailClosedGuardrail + * @description Handles semantic search API errors gracefully by resetting results. + */ + it('handles semantic search failures gracefully', async () => { + const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(testbenchApi, 'semanticSearchApi').mockRejectedValue(new Error('Network error')); + + const { result } = renderHook(() => useTestBenchState()); + + act(() => { + result.current.setSemanticQuery('test error query'); + }); + + await act(async () => { + await result.current.handleSemanticSearch(); + }); + + expect(result.current.semanticResults).toEqual([]); + expect(result.current.isSearchingSemantic).toBe(false); + expect(consoleSpy).toHaveBeenCalled(); + }); +}); diff --git a/scripts/capture_guide_screenshots.mjs b/scripts/capture_guide_screenshots.mjs index a4d10f35..335166c0 100644 --- a/scripts/capture_guide_screenshots.mjs +++ b/scripts/capture_guide_screenshots.mjs @@ -111,6 +111,24 @@ async function capture() { await testbenchTab.click(); await page.waitForTimeout(1000); await page.screenshot({ path: path.join(outputDir, 'test_bench_view.jpg'), quality: 90, type: 'jpeg' }); + + // 8b. Semantic Router Simulator + console.log('Capturing Semantic Router Simulator...'); + const semanticTabBtn = page.locator('[data-testid="testbench-tab-semantic"], button:has-text("Semantic Router")').first(); + if (await semanticTabBtn.isVisible()) { + await semanticTabBtn.click(); + await page.waitForTimeout(1000); + const queryInput = page.locator('[data-testid="semantic-query-input"]'); + if (await queryInput.isVisible()) { + await queryInput.fill('search media in plex'); + const searchBtn = page.locator('[data-testid="semantic-search-btn"]'); + if (await searchBtn.isVisible()) { + await searchBtn.click(); + await page.waitForTimeout(1500); + } + } + await page.screenshot({ path: path.join(outputDir, 'semantic_router_simulator.jpg'), quality: 90, type: 'jpeg' }); + } } // 9. Settings View & Tabs