diff --git a/.changeset/tidy-ravens-search.md b/.changeset/tidy-ravens-search.md new file mode 100644 index 0000000..30e6cb6 --- /dev/null +++ b/.changeset/tidy-ravens-search.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Handle search topics as literal keywords with optional quoted phrases, preventing malformed quotes and FTS operator words from causing SQLite errors. Preserve Unicode keywords and add local ingestion-to-retrieval regression fixtures. diff --git a/packages/context/README.md b/packages/context/README.md index f8c182e..08e762d 100644 --- a/packages/context/README.md +++ b/packages/context/README.md @@ -556,6 +556,17 @@ context query nextjs 'middleware authentication' # Returns the same JSON format as the MCP get_docs tool ``` +Topics are literal keywords, matched together (AND). Punctuation separates words, +so `ExecStart=`, `systemctl --user`, and `spring.main.banner-mode` can be used +directly. Words such as `AND`, `OR`, `NOT`, and `NEAR` are searched as text; +advanced FTS syntax is not supported. + +Use paired double quotes for a phrase, for example +`context query nextjs '"server components" rendering'`. Phrase words must appear +next to each other in order, with the usual search stemming. An unmatched double +quote is ignored and the remaining words are searched as keywords. Empty or +punctuation-only topics return no results. These rules also apply to MCP `get_docs`. + --- ## :gear: Architecture diff --git a/packages/context/src/fixtures/retrieval/README.md b/packages/context/src/fixtures/retrieval/README.md new file mode 100644 index 0000000..b6da4d0 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/README.md @@ -0,0 +1,18 @@ +# Retrieval fixtures + +These small, hand-written documentation samples exercise the real package builder +(parsing, chunking, deduplication, and FTS indexing) without network access. +They are regression examples, not copies of upstream documentation. + +`search.retrieval.test.ts` defines the query expectations. Each expected source +must appear among the first **three distinct documents** returned. Language +queries additionally require the intended document first: both signal documents +mention GDScript and C#, providing competing matches for the same API concept. +Assertions cover document and section attribution (Markdown title metadata and +HTML filename fallback), complete fenced examples, +and the search budget of 2,000 estimated content tokens (one token per four +characters, rounded up per snippet). + +A separate budget test builds numbered copies of the GDScript example. Unique +content keeps deduplication from collapsing them, so matching content exceeds +the budget and retrieval must select a subset while preserving examples. diff --git a/packages/context/src/fixtures/retrieval/csharp-signals.md b/packages/context/src/fixtures/retrieval/csharp-signals.md new file mode 100644 index 0000000..4852f69 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/csharp-signals.md @@ -0,0 +1,27 @@ +--- +title: C# signals +--- + +# C# signals + +## Declaring and emitting signals + +C# signals use a delegate marked with the Signal attribute. Connect a listener +with the generated event and notify it with EmitSignal. Unlike GDScript signals, +C# signal delegates must have names ending in EventHandler. + +```csharp +using Godot; + +public partial class Health : Node +{ + [Signal] + public delegate void HealthChangedEventHandler(int value); + + public override void _Ready() + { + HealthChanged += value => GD.Print(value); + EmitSignal(SignalName.HealthChanged, 80); + } +} +``` diff --git a/packages/context/src/fixtures/retrieval/gdscript-signals.md b/packages/context/src/fixtures/retrieval/gdscript-signals.md new file mode 100644 index 0000000..123ef49 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/gdscript-signals.md @@ -0,0 +1,24 @@ +--- +title: GDScript signals +--- + +# GDScript signals + +## Declaring and emitting signals + +GDScript signals let a node notify listeners when an event happens. Declare a +signal with the signal keyword, connect a callable, and emit it with the value +listeners need. Unlike C# events, GDScript uses the signal object's emit method. + +```gdscript +extends Node + +signal health_changed(value: int) + +func _ready() -> void: + health_changed.connect(_on_health_changed) + health_changed.emit(80) + +func _on_health_changed(value: int) -> void: + print(value) +``` diff --git a/packages/context/src/fixtures/retrieval/spring-configuration.md b/packages/context/src/fixtures/retrieval/spring-configuration.md new file mode 100644 index 0000000..017e66b --- /dev/null +++ b/packages/context/src/fixtures/retrieval/spring-configuration.md @@ -0,0 +1,16 @@ +--- +title: Spring Boot configuration +--- + +# Spring Boot configuration + +## Banner mode + +Set spring.main.banner-mode in application.properties to disable the startup +banner. The setting accepts off, console, or log. This example disables the +banner while choosing an HTTP port for the application. + +```properties +spring.main.banner-mode=off +server.port=8080 +``` diff --git a/packages/context/src/fixtures/retrieval/systemctl.html b/packages/context/src/fixtures/retrieval/systemctl.html new file mode 100644 index 0000000..877cfa4 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/systemctl.html @@ -0,0 +1,15 @@ + + +systemctl + +

systemctl

+

User services

+

Use systemctl --user to manage services belonging to the current user. +After editing a user unit, reload the user manager, start the service, +and inspect its status.

+
systemctl --user daemon-reload
+systemctl --user start example.service
+systemctl --user status example.service
+
+ + diff --git a/packages/context/src/fixtures/retrieval/systemd.service.html b/packages/context/src/fixtures/retrieval/systemd.service.html new file mode 100644 index 0000000..f9f728f --- /dev/null +++ b/packages/context/src/fixtures/retrieval/systemd.service.html @@ -0,0 +1,19 @@ + + +systemd.service + +

systemd.service

+

ExecStart

+

ExecStart= specifies the command to execute when a service starts. +Place ExecStart in the Service section of a unit file. This example starts +an application after the network is available.

+
[Unit]
+Description=Example application
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/example --serve
+Restart=on-failure
+
+ + diff --git a/packages/context/src/guidance.ts b/packages/context/src/guidance.ts index 99f8349..cb1c960 100644 --- a/packages/context/src/guidance.ts +++ b/packages/context/src/guidance.ts @@ -5,7 +5,7 @@ export const GET_DOCS_LIBRARY_DESCRIPTION = "Installed library to search (name@version). If it is not installed, use search_packages, then download_package, then retry get_docs."; export const GET_DOCS_TOPIC_DESCRIPTION = - "Use a short API name, keyword, or phrase (for example: 'createServer', 'cors middleware'). Search terms are all matched together, so extra words will narrow but can also eliminate results."; + 'Use a short API name, keyword, or phrase (for example: createServer, cors middleware). Search terms are all matched together, so extra words will narrow but can also eliminate results. Punctuation separates words. Paired double quotes require a phrase ("server components"); unmatched quotes are ignored. FTS operator words are literal.'; export const SEARCH_PACKAGES_DESCRIPTION = "Search for documentation packages available on the registry server. Use short package names like 'react', 'next', or 'fastapi'. If you find a match, call download_package, then retry get_docs. If the registry package is unavailable or insufficient, ask the user to run `context add` to build docs from source."; diff --git a/packages/context/src/search.retrieval.test.ts b/packages/context/src/search.retrieval.test.ts new file mode 100644 index 0000000..b7555bf --- /dev/null +++ b/packages/context/src/search.retrieval.test.ts @@ -0,0 +1,141 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { DatabaseConnection } from "./database.js"; +import { initDatabase, openDatabase } from "./database.js"; +import { buildPackage } from "./package-builder.js"; +import type { DocSnippet } from "./search.js"; +import { search } from "./search.js"; + +const FIXTURE_DIR = new URL("./fixtures/retrieval/", import.meta.url); +const FIXTURE_NAMES = [ + "systemd.service.html", + "systemctl.html", + "spring-configuration.md", + "gdscript-signals.md", + "csharp-signals.md", +]; +const TOP_K = 3; +const TOKEN_BUDGET = 2000; + +function contentTokens(results: DocSnippet[]): number { + return results.reduce((sum, r) => sum + Math.ceil(r.content.length / 4), 0); +} + +describe("retrieval through package ingestion", () => { + let testDir: string; + let db: DatabaseConnection; + + beforeAll(async () => { + await initDatabase(); + testDir = mkdtempSync(join(tmpdir(), "context-retrieval-")); + const packagePath = join(testDir, "retrieval.db"); + const built = buildPackage( + packagePath, + FIXTURE_NAMES.map((name) => ({ + path: `docs/${name}`, + content: readFileSync(new URL(name, FIXTURE_DIR), "utf8"), + })), + { name: "retrieval-fixtures", version: "1.0.0" }, + ); + expect(built.skippedFiles).toBe(0); + db = openDatabase(packagePath, { readonly: true }); + }); + + afterAll(() => { + db?.close(); + if (testDir) rmSync(testDir, { recursive: true, force: true }); + }); + + it.each([ + { + topic: "ExecStart=", + file: "systemd.service.html", + title: "systemd.service.html > ExecStart", + code: "ExecStart=/usr/bin/example --serve", + }, + { + topic: "systemctl --user", + file: "systemctl.html", + title: "systemctl.html > User services", + code: "systemctl --user start example.service", + }, + { + topic: "spring.main.banner-mode", + file: "spring-configuration.md", + title: "Spring Boot configuration > Banner mode", + code: "spring.main.banner-mode=off", + }, + { + topic: "GDScript signals", + file: "gdscript-signals.md", + title: "GDScript signals > Declaring and emitting signals", + code: "health_changed.emit(80)", + }, + { + topic: "C# signals", + file: "csharp-signals.md", + title: "C# signals > Declaring and emitting signals", + code: "EmitSignal(SignalName.HealthChanged, 80);", + }, + ])("retrieves attributed documentation and code for $topic", (fixture) => { + const result = search(db, fixture.topic); + const sources = [...new Set(result.results.map((r) => r.source))]; + expect(result.library).toBe("retrieval-fixtures@1.0.0"); + expect(result.version).toBe("1.0.0"); + expect(sources.slice(0, TOP_K)).toContain(`docs/${fixture.file}`); + expect(contentTokens(result.results)).toBeLessThanOrEqual(TOKEN_BUDGET); + + const snippet = result.results.find((r) => r.title === fixture.title); + expect(snippet?.source).toBe(`docs/${fixture.file}`); + const codeBlocks = snippet?.content.match(/```[^\n]*\n[\s\S]*?\n```/g); + expect(codeBlocks?.some((block) => block.includes(fixture.code))).toBe( + true, + ); + }); + + it.each([ + ["GDScript signals", "gdscript-signals.md"], + ["C# signals", "csharp-signals.md"], + ])("ranks the intended language first for %s", (topic, file) => { + expect(search(db, topic).results[0]?.source).toBe(`docs/${file}`); + }); + + it("preserves examples when matching content exceeds the token budget", () => { + const content = readFileSync( + new URL("gdscript-signals.md", FIXTURE_DIR), + "utf8", + ); + const files = Array.from({ length: 20 }, (_, i) => ({ + path: `docs/example-${i}.md`, + content: `${content}\nThis is signal example ${i}.\n`, + })); + const packagePath = join(testDir, "budget.db"); + const built = buildPackage(packagePath, files, { + name: "budget-fixtures", + version: "1.0.0", + }); + expect(built.skippedFiles).toBe(0); + expect(built.totalTokens).toBeGreaterThan(TOKEN_BUDGET); + + const budgetDb = openDatabase(packagePath, { readonly: true }); + try { + const result = search(budgetDb, "GDScript signals"); + expect(result.results.length).toBeGreaterThan(0); + expect(result.results.length).toBeLessThan(files.length); + expect(contentTokens(result.results)).toBeLessThanOrEqual(TOKEN_BUDGET); + for (const snippet of result.results) { + expect(files.map((file) => file.path)).toContain(snippet.source); + expect(snippet.title).toBe( + "GDScript signals > Declaring and emitting signals", + ); + expect(snippet.content).toContain( + content.match(/```gdscript\n[\s\S]*?\n```/)?.[0], + ); + } + } finally { + budgetDb.close(); + } + }); +}); diff --git a/packages/context/src/search.test.ts b/packages/context/src/search.test.ts index 76f6aa4..199424a 100644 --- a/packages/context/src/search.test.ts +++ b/packages/context/src/search.test.ts @@ -141,4 +141,115 @@ describe("search", () => { expect(result.results).toHaveLength(1); }); + + it.each([ + '"ExecStart', + 'ExecStart"', + '"""ExecStart', + '"ExecStart"', + '"" ExecStart', + '"!!!" ExecStart', + "ExecStart=", + "(ExecStart*)", + ])("searches literal terms in %j", (topic) => { + insertChunk(db, { + docPath: "man/systemd.service.html", + docTitle: "Service", + sectionTitle: "Commands", + content: "ExecStart sets the command to execute when a service starts.", + tokens: 20, + }); + rebuildFtsIndex(db); + + expect(search(db, topic).results.map((r) => r.source)).toEqual([ + "man/systemd.service.html", + ]); + }); + + it.each([ + "AND", + "OR", + "NOT", + "NEAR", + ])("treats %s as a literal word, alone and between keywords", (word) => { + insertChunk(db, { + docPath: "docs/operators.md", + docTitle: "Operators", + sectionTitle: "Example", + content: `ExecStart service example containing the literal word ${word}.`, + tokens: 20, + }); + insertChunk(db, { + docPath: "docs/service.md", + docTitle: "Service", + sectionTitle: "Commands", + content: "ExecStart service example without the operator word.", + tokens: 20, + }); + rebuildFtsIndex(db); + + for (const topic of [word, `ExecStart ${word} service`]) { + expect(search(db, topic).results.map((r) => r.source)).toEqual([ + "docs/operators.md", + ]); + } + }); + + it.each([ + "", + " \t\n", + '"', + '""', + "= -- . () : * +", + '"!!!"', + "___", + ])("returns no results for a topic without words: %j", (topic) => { + rebuildFtsIndex(db); + expect(search(db, topic).results).toEqual([]); + }); + + it("requires adjacent words only inside paired double quotes", () => { + for (const [docPath, content] of [ + ["docs/phrase.md", "Rendering server components is useful."], + ["docs/keywords.md", "Rendering components on the server is useful."], + ] as const) { + insertChunk(db, { + docPath, + docTitle: "Rendering", + sectionTitle: "Overview", + content, + tokens: 20, + }); + } + rebuildFtsIndex(db); + + expect( + search(db, '"server components" rendering').results.map((r) => r.source), + ).toEqual(["docs/phrase.md"]); + for (const topic of [ + "server components rendering", + '"server components rendering', + ]) { + expect( + search(db, topic) + .results.map((r) => r.source) + .sort(), + ).toEqual(["docs/keywords.md", "docs/phrase.md"]); + } + }); + + it.each(["café", "日本語"])("preserves Unicode keywords: %s", (topic) => { + insertChunk(db, { + docPath: "docs/unicode.md", + docTitle: "Unicode", + sectionTitle: "Examples", + content: "Unicode examples include café and 日本語.", + tokens: 20, + }); + rebuildFtsIndex(db); + + expect(search(db, topic).results.map((r) => r.source)).toEqual([ + "docs/unicode.md", + ]); + }); }); diff --git a/packages/context/src/search.ts b/packages/context/src/search.ts index 9103122..01bb682 100644 --- a/packages/context/src/search.ts +++ b/packages/context/src/search.ts @@ -36,16 +36,20 @@ interface ChunkMatch { } /** - * Build an FTS5 query from user topic. - * - Cleans special characters (keeps alphanumeric, spaces, quotes) - * - Words are implicitly ANDed by FTS5 + * Quote literal terms so reserved words cannot become FTS5 operators. + * Only paired double quotes group a phrase; unmatched quotes are ignored. + * Punctuation separates words, and terms/phrases are implicitly ANDed. */ function buildQuery(topic: string): string { - return topic - .trim() - .replace(/[^\w\s"]/g, " ") - .replace(/\s+/g, " ") - .trim(); + const parts = topic + .replace(/[^\p{L}\p{N}\p{M}\s"]/gu, " ") + .match(/"[^"]*"|[^\s"]+/g); + + return (parts ?? []) + .map((part) => part.replaceAll('"', "").trim()) + .filter(Boolean) + .map((part) => `"${part}"`) + .join(" "); } function searchFts(db: DatabaseConnection, query: string): ChunkMatch[] {