fix(context): handle literal search topics safely (#145)

.changeset/tidy-ravens-search.md

@@ -0,0 +1,5 @@
+---
+"@neuledge/context": patch
+---
+
+Handle search topics as literal keywords with optional quoted phrases, preventing malformed quotes and FTS operator words from causing SQLite errors. Preserve Unicode keywords and add local ingestion-to-retrieval regression fixtures.

packages/context/README.md

@@ -556,6 +556,17 @@ context query nextjs 'middleware authentication'
 # Returns the same JSON format as the MCP get_docs tool
 ```
 
+Topics are literal keywords, matched together (AND). Punctuation separates words,
+so `ExecStart=`, `systemctl --user`, and `spring.main.banner-mode` can be used
+directly. Words such as `AND`, `OR`, `NOT`, and `NEAR` are searched as text;
+advanced FTS syntax is not supported.
+
+Use paired double quotes for a phrase, for example
+`context query nextjs '"server components" rendering'`. Phrase words must appear
+next to each other in order, with the usual search stemming. An unmatched double
+quote is ignored and the remaining words are searched as keywords. Empty or
+punctuation-only topics return no results. These rules also apply to MCP `get_docs`.
+
 ---
 
 ## :gear: Architecture

packages/context/src/fixtures/retrieval/README.md

@@ -0,0 +1,18 @@
+# Retrieval fixtures
+
+These small, hand-written documentation samples exercise the real package builder
+(parsing, chunking, deduplication, and FTS indexing) without network access.
+They are regression examples, not copies of upstream documentation.
+
+`search.retrieval.test.ts` defines the query expectations. Each expected source
+must appear among the first **three distinct documents** returned. Language
+queries additionally require the intended document first: both signal documents
+mention GDScript and C#, providing competing matches for the same API concept.
+Assertions cover document and section attribution (Markdown title metadata and
+HTML filename fallback), complete fenced examples,
+and the search budget of 2,000 estimated content tokens (one token per four
+characters, rounded up per snippet).
+
+A separate budget test builds numbered copies of the GDScript example. Unique
+content keeps deduplication from collapsing them, so matching content exceeds
+the budget and retrieval must select a subset while preserving examples.

packages/context/src/fixtures/retrieval/csharp-signals.md

@@ -0,0 +1,27 @@
+---
+title: C# signals
+---
+
+# C# signals
+
+## Declaring and emitting signals
+
+C# signals use a delegate marked with the Signal attribute. Connect a listener
+with the generated event and notify it with EmitSignal. Unlike GDScript signals,
+C# signal delegates must have names ending in EventHandler.
+
+```csharp
+using Godot;
+
+public partial class Health : Node
+{
+    [Signal]
+    public delegate void HealthChangedEventHandler(int value);
+
+    public override void _Ready()
+    {
+        HealthChanged += value => GD.Print(value);
+        EmitSignal(SignalName.HealthChanged, 80);
+    }
+}
+```

packages/context/src/fixtures/retrieval/gdscript-signals.md

@@ -0,0 +1,24 @@
+---
+title: GDScript signals
+---
+
+# GDScript signals
+
+## Declaring and emitting signals
+
+GDScript signals let a node notify listeners when an event happens. Declare a
+signal with the signal keyword, connect a callable, and emit it with the value
+listeners need. Unlike C# events, GDScript uses the signal object's emit method.
+
+```gdscript
+extends Node
+
+signal health_changed(value: int)
+
+func _ready() -> void:
+    health_changed.connect(_on_health_changed)
+    health_changed.emit(80)
+
+func _on_health_changed(value: int) -> void:
+    print(value)
+```

packages/context/src/fixtures/retrieval/spring-configuration.md

@@ -0,0 +1,16 @@
+---
+title: Spring Boot configuration
+---
+
+# Spring Boot configuration
+
+## Banner mode
+
+Set spring.main.banner-mode in application.properties to disable the startup
+banner. The setting accepts off, console, or log. This example disables the
+banner while choosing an HTTP port for the application.
+
+```properties
+spring.main.banner-mode=off
+server.port=8080
+```

packages/context/src/fixtures/retrieval/systemctl.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+<head><title>systemctl</title></head>
+<body>
+<h1>systemctl</h1>
+<h2>User services</h2>
+<p>Use systemctl --user to manage services belonging to the current user.
+After editing a user unit, reload the user manager, start the service,
+and inspect its status.</p>
+<pre><code class="language-sh">systemctl --user daemon-reload
+systemctl --user start example.service
+systemctl --user status example.service
+</code></pre>
+</body>
+</html>

packages/context/src/fixtures/retrieval/systemd.service.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+<head><title>systemd.service</title></head>
+<body>
+<h1>systemd.service</h1>
+<h2>ExecStart</h2>
+<p>ExecStart= specifies the command to execute when a service starts.
+Place ExecStart in the Service section of a unit file. This example starts
+an application after the network is available.</p>
+<pre><code class="language-ini">[Unit]
+Description=Example application
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/example --serve
+Restart=on-failure
+</code></pre>
+</body>
+</html>

packages/context/src/guidance.ts

@@ -5,7 +5,7 @@ export const GET_DOCS_LIBRARY_DESCRIPTION =
   "Installed library to search (name@version). If it is not installed, use search_packages, then download_package, then retry get_docs.";
 
 export const GET_DOCS_TOPIC_DESCRIPTION =
-  "Use a short API name, keyword, or phrase (for example: 'createServer', 'cors middleware'). Search terms are all matched together, so extra words will narrow but can also eliminate results.";
+  'Use a short API name, keyword, or phrase (for example: createServer, cors middleware). Search terms are all matched together, so extra words will narrow but can also eliminate results. Punctuation separates words. Paired double quotes require a phrase ("server components"); unmatched quotes are ignored. FTS operator words are literal.';
 
 export const SEARCH_PACKAGES_DESCRIPTION =
   "Search for documentation packages available on the registry server. Use short package names like 'react', 'next', or 'fastapi'. If you find a match, call download_package, then retry get_docs. If the registry package is unavailable or insufficient, ask the user to run `context add` to build docs from source.";

packages/context/src/search.retrieval.test.ts

@@ -0,0 +1,141 @@
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { DatabaseConnection } from "./database.js";
+import { initDatabase, openDatabase } from "./database.js";
+import { buildPackage } from "./package-builder.js";
+import type { DocSnippet } from "./search.js";
+import { search } from "./search.js";
+
+const FIXTURE_DIR = new URL("./fixtures/retrieval/", import.meta.url);
+const FIXTURE_NAMES = [
+  "systemd.service.html",
+  "systemctl.html",
+  "spring-configuration.md",
+  "gdscript-signals.md",
+  "csharp-signals.md",
+];
+const TOP_K = 3;
+const TOKEN_BUDGET = 2000;
+
+function contentTokens(results: DocSnippet[]): number {
+  return results.reduce((sum, r) => sum + Math.ceil(r.content.length / 4), 0);
+}
+
+describe("retrieval through package ingestion", () => {
+  let testDir: string;
+  let db: DatabaseConnection;
+
+  beforeAll(async () => {
+    await initDatabase();
+    testDir = mkdtempSync(join(tmpdir(), "context-retrieval-"));
+    const packagePath = join(testDir, "retrieval.db");
+    const built = buildPackage(
+      packagePath,
+      FIXTURE_NAMES.map((name) => ({
+        path: `docs/${name}`,
+        content: readFileSync(new URL(name, FIXTURE_DIR), "utf8"),
+      })),
+      { name: "retrieval-fixtures", version: "1.0.0" },
+    );
+    expect(built.skippedFiles).toBe(0);
+    db = openDatabase(packagePath, { readonly: true });
+  });
+
+  afterAll(() => {
+    db?.close();
+    if (testDir) rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it.each([
+    {
+      topic: "ExecStart=",
+      file: "systemd.service.html",
+      title: "systemd.service.html > ExecStart",
+      code: "ExecStart=/usr/bin/example --serve",
+    },
+    {
+      topic: "systemctl --user",
+      file: "systemctl.html",
+      title: "systemctl.html > User services",
+      code: "systemctl --user start example.service",
+    },
+    {
+      topic: "spring.main.banner-mode",
+      file: "spring-configuration.md",
+      title: "Spring Boot configuration > Banner mode",
+      code: "spring.main.banner-mode=off",
+    },
+    {
+      topic: "GDScript signals",
+      file: "gdscript-signals.md",
+      title: "GDScript signals > Declaring and emitting signals",
+      code: "health_changed.emit(80)",
+    },
+    {
+      topic: "C# signals",
+      file: "csharp-signals.md",
+      title: "C# signals > Declaring and emitting signals",
+      code: "EmitSignal(SignalName.HealthChanged, 80);",
+    },
+  ])("retrieves attributed documentation and code for $topic", (fixture) => {
+    const result = search(db, fixture.topic);
+    const sources = [...new Set(result.results.map((r) => r.source))];
+    expect(result.library).toBe("retrieval-fixtures@1.0.0");
+    expect(result.version).toBe("1.0.0");
+    expect(sources.slice(0, TOP_K)).toContain(`docs/${fixture.file}`);
+    expect(contentTokens(result.results)).toBeLessThanOrEqual(TOKEN_BUDGET);
+
+    const snippet = result.results.find((r) => r.title === fixture.title);
+    expect(snippet?.source).toBe(`docs/${fixture.file}`);
+    const codeBlocks = snippet?.content.match(/```[^\n]*\n[\s\S]*?\n```/g);
+    expect(codeBlocks?.some((block) => block.includes(fixture.code))).toBe(
+      true,
+    );
+  });
+
+  it.each([
+    ["GDScript signals", "gdscript-signals.md"],
+    ["C# signals", "csharp-signals.md"],
+  ])("ranks the intended language first for %s", (topic, file) => {
+    expect(search(db, topic).results[0]?.source).toBe(`docs/${file}`);
+  });
+
+  it("preserves examples when matching content exceeds the token budget", () => {
+    const content = readFileSync(
+      new URL("gdscript-signals.md", FIXTURE_DIR),
+      "utf8",
+    );
+    const files = Array.from({ length: 20 }, (_, i) => ({
+      path: `docs/example-${i}.md`,
+      content: `${content}\nThis is signal example ${i}.\n`,
+    }));
+    const packagePath = join(testDir, "budget.db");
+    const built = buildPackage(packagePath, files, {
+      name: "budget-fixtures",
+      version: "1.0.0",
+    });
+    expect(built.skippedFiles).toBe(0);
+    expect(built.totalTokens).toBeGreaterThan(TOKEN_BUDGET);
+
+    const budgetDb = openDatabase(packagePath, { readonly: true });
+    try {
+      const result = search(budgetDb, "GDScript signals");
+      expect(result.results.length).toBeGreaterThan(0);
+      expect(result.results.length).toBeLessThan(files.length);
+      expect(contentTokens(result.results)).toBeLessThanOrEqual(TOKEN_BUDGET);
+      for (const snippet of result.results) {
+        expect(files.map((file) => file.path)).toContain(snippet.source);
+        expect(snippet.title).toBe(
+          "GDScript signals > Declaring and emitting signals",
+        );
+        expect(snippet.content).toContain(
+          content.match(/```gdscript\n[\s\S]*?\n```/)?.[0],
+        );
+      }
+    } finally {
+      budgetDb.close();
+    }
+  });
+});