diff --git a/.changeset/tidy-ravens-search.md b/.changeset/tidy-ravens-search.md new file mode 100644 index 0000000..30e6cb6 --- /dev/null +++ b/.changeset/tidy-ravens-search.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Handle search topics as literal keywords with optional quoted phrases, preventing malformed quotes and FTS operator words from causing SQLite errors. Preserve Unicode keywords and add local ingestion-to-retrieval regression fixtures. diff --git a/packages/context/README.md b/packages/context/README.md index f8c182e..08e762d 100644 --- a/packages/context/README.md +++ b/packages/context/README.md @@ -556,6 +556,17 @@ context query nextjs 'middleware authentication' # Returns the same JSON format as the MCP get_docs tool ``` +Topics are literal keywords, matched together (AND). Punctuation separates words, +so `ExecStart=`, `systemctl --user`, and `spring.main.banner-mode` can be used +directly. Words such as `AND`, `OR`, `NOT`, and `NEAR` are searched as text; +advanced FTS syntax is not supported. + +Use paired double quotes for a phrase, for example +`context query nextjs '"server components" rendering'`. Phrase words must appear +next to each other in order, with the usual search stemming. An unmatched double +quote is ignored and the remaining words are searched as keywords. Empty or +punctuation-only topics return no results. These rules also apply to MCP `get_docs`. + --- ## :gear: Architecture diff --git a/packages/context/src/fixtures/retrieval/README.md b/packages/context/src/fixtures/retrieval/README.md new file mode 100644 index 0000000..b6da4d0 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/README.md @@ -0,0 +1,18 @@ +# Retrieval fixtures + +These small, hand-written documentation samples exercise the real package builder +(parsing, chunking, deduplication, and FTS indexing) without network access. +They are regression examples, not copies of upstream documentation. + +`search.retrieval.test.ts` defines the query expectations. Each expected source +must appear among the first **three distinct documents** returned. Language +queries additionally require the intended document first: both signal documents +mention GDScript and C#, providing competing matches for the same API concept. +Assertions cover document and section attribution (Markdown title metadata and +HTML filename fallback), complete fenced examples, +and the search budget of 2,000 estimated content tokens (one token per four +characters, rounded up per snippet). + +A separate budget test builds numbered copies of the GDScript example. Unique +content keeps deduplication from collapsing them, so matching content exceeds +the budget and retrieval must select a subset while preserving examples. diff --git a/packages/context/src/fixtures/retrieval/csharp-signals.md b/packages/context/src/fixtures/retrieval/csharp-signals.md new file mode 100644 index 0000000..4852f69 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/csharp-signals.md @@ -0,0 +1,27 @@ +--- +title: C# signals +--- + +# C# signals + +## Declaring and emitting signals + +C# signals use a delegate marked with the Signal attribute. Connect a listener +with the generated event and notify it with EmitSignal. Unlike GDScript signals, +C# signal delegates must have names ending in EventHandler. + +```csharp +using Godot; + +public partial class Health : Node +{ + [Signal] + public delegate void HealthChangedEventHandler(int value); + + public override void _Ready() + { + HealthChanged += value => GD.Print(value); + EmitSignal(SignalName.HealthChanged, 80); + } +} +``` diff --git a/packages/context/src/fixtures/retrieval/gdscript-signals.md b/packages/context/src/fixtures/retrieval/gdscript-signals.md new file mode 100644 index 0000000..123ef49 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/gdscript-signals.md @@ -0,0 +1,24 @@ +--- +title: GDScript signals +--- + +# GDScript signals + +## Declaring and emitting signals + +GDScript signals let a node notify listeners when an event happens. Declare a +signal with the signal keyword, connect a callable, and emit it with the value +listeners need. Unlike C# events, GDScript uses the signal object's emit method. + +```gdscript +extends Node + +signal health_changed(value: int) + +func _ready() -> void: + health_changed.connect(_on_health_changed) + health_changed.emit(80) + +func _on_health_changed(value: int) -> void: + print(value) +``` diff --git a/packages/context/src/fixtures/retrieval/spring-configuration.md b/packages/context/src/fixtures/retrieval/spring-configuration.md new file mode 100644 index 0000000..017e66b --- /dev/null +++ b/packages/context/src/fixtures/retrieval/spring-configuration.md @@ -0,0 +1,16 @@ +--- +title: Spring Boot configuration +--- + +# Spring Boot configuration + +## Banner mode + +Set spring.main.banner-mode in application.properties to disable the startup +banner. The setting accepts off, console, or log. This example disables the +banner while choosing an HTTP port for the application. + +```properties +spring.main.banner-mode=off +server.port=8080 +``` diff --git a/packages/context/src/fixtures/retrieval/systemctl.html b/packages/context/src/fixtures/retrieval/systemctl.html new file mode 100644 index 0000000..877cfa4 --- /dev/null +++ b/packages/context/src/fixtures/retrieval/systemctl.html @@ -0,0 +1,15 @@ + + +
Use systemctl --user to manage services belonging to the current user. +After editing a user unit, reload the user manager, start the service, +and inspect its status.
+systemctl --user daemon-reload
+systemctl --user start example.service
+systemctl --user status example.service
+
+
+
diff --git a/packages/context/src/fixtures/retrieval/systemd.service.html b/packages/context/src/fixtures/retrieval/systemd.service.html
new file mode 100644
index 0000000..f9f728f
--- /dev/null
+++ b/packages/context/src/fixtures/retrieval/systemd.service.html
@@ -0,0 +1,19 @@
+
+
+ExecStart= specifies the command to execute when a service starts. +Place ExecStart in the Service section of a unit file. This example starts +an application after the network is available.
+[Unit]
+Description=Example application
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/example --serve
+Restart=on-failure
+
+
+
diff --git a/packages/context/src/guidance.ts b/packages/context/src/guidance.ts
index 99f8349..cb1c960 100644
--- a/packages/context/src/guidance.ts
+++ b/packages/context/src/guidance.ts
@@ -5,7 +5,7 @@ export const GET_DOCS_LIBRARY_DESCRIPTION =
"Installed library to search (name@version). If it is not installed, use search_packages, then download_package, then retry get_docs.";
export const GET_DOCS_TOPIC_DESCRIPTION =
- "Use a short API name, keyword, or phrase (for example: 'createServer', 'cors middleware'). Search terms are all matched together, so extra words will narrow but can also eliminate results.";
+ 'Use a short API name, keyword, or phrase (for example: createServer, cors middleware). Search terms are all matched together, so extra words will narrow but can also eliminate results. Punctuation separates words. Paired double quotes require a phrase ("server components"); unmatched quotes are ignored. FTS operator words are literal.';
export const SEARCH_PACKAGES_DESCRIPTION =
"Search for documentation packages available on the registry server. Use short package names like 'react', 'next', or 'fastapi'. If you find a match, call download_package, then retry get_docs. If the registry package is unavailable or insufficient, ask the user to run `context add` to build docs from source.";
diff --git a/packages/context/src/search.retrieval.test.ts b/packages/context/src/search.retrieval.test.ts
new file mode 100644
index 0000000..b7555bf
--- /dev/null
+++ b/packages/context/src/search.retrieval.test.ts
@@ -0,0 +1,141 @@
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { DatabaseConnection } from "./database.js";
+import { initDatabase, openDatabase } from "./database.js";
+import { buildPackage } from "./package-builder.js";
+import type { DocSnippet } from "./search.js";
+import { search } from "./search.js";
+
+const FIXTURE_DIR = new URL("./fixtures/retrieval/", import.meta.url);
+const FIXTURE_NAMES = [
+ "systemd.service.html",
+ "systemctl.html",
+ "spring-configuration.md",
+ "gdscript-signals.md",
+ "csharp-signals.md",
+];
+const TOP_K = 3;
+const TOKEN_BUDGET = 2000;
+
+function contentTokens(results: DocSnippet[]): number {
+ return results.reduce((sum, r) => sum + Math.ceil(r.content.length / 4), 0);
+}
+
+describe("retrieval through package ingestion", () => {
+ let testDir: string;
+ let db: DatabaseConnection;
+
+ beforeAll(async () => {
+ await initDatabase();
+ testDir = mkdtempSync(join(tmpdir(), "context-retrieval-"));
+ const packagePath = join(testDir, "retrieval.db");
+ const built = buildPackage(
+ packagePath,
+ FIXTURE_NAMES.map((name) => ({
+ path: `docs/${name}`,
+ content: readFileSync(new URL(name, FIXTURE_DIR), "utf8"),
+ })),
+ { name: "retrieval-fixtures", version: "1.0.0" },
+ );
+ expect(built.skippedFiles).toBe(0);
+ db = openDatabase(packagePath, { readonly: true });
+ });
+
+ afterAll(() => {
+ db?.close();
+ if (testDir) rmSync(testDir, { recursive: true, force: true });
+ });
+
+ it.each([
+ {
+ topic: "ExecStart=",
+ file: "systemd.service.html",
+ title: "systemd.service.html > ExecStart",
+ code: "ExecStart=/usr/bin/example --serve",
+ },
+ {
+ topic: "systemctl --user",
+ file: "systemctl.html",
+ title: "systemctl.html > User services",
+ code: "systemctl --user start example.service",
+ },
+ {
+ topic: "spring.main.banner-mode",
+ file: "spring-configuration.md",
+ title: "Spring Boot configuration > Banner mode",
+ code: "spring.main.banner-mode=off",
+ },
+ {
+ topic: "GDScript signals",
+ file: "gdscript-signals.md",
+ title: "GDScript signals > Declaring and emitting signals",
+ code: "health_changed.emit(80)",
+ },
+ {
+ topic: "C# signals",
+ file: "csharp-signals.md",
+ title: "C# signals > Declaring and emitting signals",
+ code: "EmitSignal(SignalName.HealthChanged, 80);",
+ },
+ ])("retrieves attributed documentation and code for $topic", (fixture) => {
+ const result = search(db, fixture.topic);
+ const sources = [...new Set(result.results.map((r) => r.source))];
+ expect(result.library).toBe("retrieval-fixtures@1.0.0");
+ expect(result.version).toBe("1.0.0");
+ expect(sources.slice(0, TOP_K)).toContain(`docs/${fixture.file}`);
+ expect(contentTokens(result.results)).toBeLessThanOrEqual(TOKEN_BUDGET);
+
+ const snippet = result.results.find((r) => r.title === fixture.title);
+ expect(snippet?.source).toBe(`docs/${fixture.file}`);
+ const codeBlocks = snippet?.content.match(/```[^\n]*\n[\s\S]*?\n```/g);
+ expect(codeBlocks?.some((block) => block.includes(fixture.code))).toBe(
+ true,
+ );
+ });
+
+ it.each([
+ ["GDScript signals", "gdscript-signals.md"],
+ ["C# signals", "csharp-signals.md"],
+ ])("ranks the intended language first for %s", (topic, file) => {
+ expect(search(db, topic).results[0]?.source).toBe(`docs/${file}`);
+ });
+
+ it("preserves examples when matching content exceeds the token budget", () => {
+ const content = readFileSync(
+ new URL("gdscript-signals.md", FIXTURE_DIR),
+ "utf8",
+ );
+ const files = Array.from({ length: 20 }, (_, i) => ({
+ path: `docs/example-${i}.md`,
+ content: `${content}\nThis is signal example ${i}.\n`,
+ }));
+ const packagePath = join(testDir, "budget.db");
+ const built = buildPackage(packagePath, files, {
+ name: "budget-fixtures",
+ version: "1.0.0",
+ });
+ expect(built.skippedFiles).toBe(0);
+ expect(built.totalTokens).toBeGreaterThan(TOKEN_BUDGET);
+
+ const budgetDb = openDatabase(packagePath, { readonly: true });
+ try {
+ const result = search(budgetDb, "GDScript signals");
+ expect(result.results.length).toBeGreaterThan(0);
+ expect(result.results.length).toBeLessThan(files.length);
+ expect(contentTokens(result.results)).toBeLessThanOrEqual(TOKEN_BUDGET);
+ for (const snippet of result.results) {
+ expect(files.map((file) => file.path)).toContain(snippet.source);
+ expect(snippet.title).toBe(
+ "GDScript signals > Declaring and emitting signals",
+ );
+ expect(snippet.content).toContain(
+ content.match(/```gdscript\n[\s\S]*?\n```/)?.[0],
+ );
+ }
+ } finally {
+ budgetDb.close();
+ }
+ });
+});
diff --git a/packages/context/src/search.test.ts b/packages/context/src/search.test.ts
index 76f6aa4..199424a 100644
--- a/packages/context/src/search.test.ts
+++ b/packages/context/src/search.test.ts
@@ -141,4 +141,115 @@ describe("search", () => {
expect(result.results).toHaveLength(1);
});
+
+ it.each([
+ '"ExecStart',
+ 'ExecStart"',
+ '"""ExecStart',
+ '"ExecStart"',
+ '"" ExecStart',
+ '"!!!" ExecStart',
+ "ExecStart=",
+ "(ExecStart*)",
+ ])("searches literal terms in %j", (topic) => {
+ insertChunk(db, {
+ docPath: "man/systemd.service.html",
+ docTitle: "Service",
+ sectionTitle: "Commands",
+ content: "ExecStart sets the command to execute when a service starts.",
+ tokens: 20,
+ });
+ rebuildFtsIndex(db);
+
+ expect(search(db, topic).results.map((r) => r.source)).toEqual([
+ "man/systemd.service.html",
+ ]);
+ });
+
+ it.each([
+ "AND",
+ "OR",
+ "NOT",
+ "NEAR",
+ ])("treats %s as a literal word, alone and between keywords", (word) => {
+ insertChunk(db, {
+ docPath: "docs/operators.md",
+ docTitle: "Operators",
+ sectionTitle: "Example",
+ content: `ExecStart service example containing the literal word ${word}.`,
+ tokens: 20,
+ });
+ insertChunk(db, {
+ docPath: "docs/service.md",
+ docTitle: "Service",
+ sectionTitle: "Commands",
+ content: "ExecStart service example without the operator word.",
+ tokens: 20,
+ });
+ rebuildFtsIndex(db);
+
+ for (const topic of [word, `ExecStart ${word} service`]) {
+ expect(search(db, topic).results.map((r) => r.source)).toEqual([
+ "docs/operators.md",
+ ]);
+ }
+ });
+
+ it.each([
+ "",
+ " \t\n",
+ '"',
+ '""',
+ "= -- . () : * +",
+ '"!!!"',
+ "___",
+ ])("returns no results for a topic without words: %j", (topic) => {
+ rebuildFtsIndex(db);
+ expect(search(db, topic).results).toEqual([]);
+ });
+
+ it("requires adjacent words only inside paired double quotes", () => {
+ for (const [docPath, content] of [
+ ["docs/phrase.md", "Rendering server components is useful."],
+ ["docs/keywords.md", "Rendering components on the server is useful."],
+ ] as const) {
+ insertChunk(db, {
+ docPath,
+ docTitle: "Rendering",
+ sectionTitle: "Overview",
+ content,
+ tokens: 20,
+ });
+ }
+ rebuildFtsIndex(db);
+
+ expect(
+ search(db, '"server components" rendering').results.map((r) => r.source),
+ ).toEqual(["docs/phrase.md"]);
+ for (const topic of [
+ "server components rendering",
+ '"server components rendering',
+ ]) {
+ expect(
+ search(db, topic)
+ .results.map((r) => r.source)
+ .sort(),
+ ).toEqual(["docs/keywords.md", "docs/phrase.md"]);
+ }
+ });
+
+ it.each(["café", "日本語"])("preserves Unicode keywords: %s", (topic) => {
+ insertChunk(db, {
+ docPath: "docs/unicode.md",
+ docTitle: "Unicode",
+ sectionTitle: "Examples",
+ content: "Unicode examples include café and 日本語.",
+ tokens: 20,
+ });
+ rebuildFtsIndex(db);
+
+ expect(search(db, topic).results.map((r) => r.source)).toEqual([
+ "docs/unicode.md",
+ ]);
+ });
});
diff --git a/packages/context/src/search.ts b/packages/context/src/search.ts
index 9103122..01bb682 100644
--- a/packages/context/src/search.ts
+++ b/packages/context/src/search.ts
@@ -36,16 +36,20 @@ interface ChunkMatch {
}
/**
- * Build an FTS5 query from user topic.
- * - Cleans special characters (keeps alphanumeric, spaces, quotes)
- * - Words are implicitly ANDed by FTS5
+ * Quote literal terms so reserved words cannot become FTS5 operators.
+ * Only paired double quotes group a phrase; unmatched quotes are ignored.
+ * Punctuation separates words, and terms/phrases are implicitly ANDed.
*/
function buildQuery(topic: string): string {
- return topic
- .trim()
- .replace(/[^\w\s"]/g, " ")
- .replace(/\s+/g, " ")
- .trim();
+ const parts = topic
+ .replace(/[^\p{L}\p{N}\p{M}\s"]/gu, " ")
+ .match(/"[^"]*"|[^\s"]+/g);
+
+ return (parts ?? [])
+ .map((part) => part.replaceAll('"', "").trim())
+ .filter(Boolean)
+ .map((part) => `"${part}"`)
+ .join(" ");
}
function searchFts(db: DatabaseConnection, query: string): ChunkMatch[] {