diff --git a/.env.example b/.env.example index a9495aee55..0b51d6af74 100644 --- a/.env.example +++ b/.env.example @@ -94,6 +94,21 @@ # EMBEDDING_PROVIDER=local # local | openai | voyage | cohere | gemini | openrouter +# LOCAL_EMBEDDING_MODEL=Xenova/all-MiniLM-L6-v2 +# LOCAL_EMBEDDING_DIMENSIONS=384 # Set explicitly for a different model width +# LOCAL_EMBEDDING_POOLING=mean # mean | cls | first_token | last_token | eos +# LOCAL_EMBEDDING_DTYPE=q8 # Transformers.js dtype, e.g. q8 or fp32 +# Optional exact prefixes. For E5-style inputs, quote env-file values to retain spaces: +# LOCAL_EMBEDDING_QUERY_PREFIX="query: " +# LOCAL_EMBEDDING_DOCUMENT_PREFIX="passage: " +# LOCAL_EMBEDDING_CACHE_DIR= # Optional pipeline cache directory +# LOCAL_EMBEDDING_BATCH_SIZE= # Optional positive batch limit +# Model, dimensions, pooling, dtype, document-prefix or cache-path changes require new vectors. +# Unidentified older local indexes require an explicit rebuild too: back up the +# store, set AGENTMEMORY_DROP_STALE_INDEX=true for one restart, then unset it. +# This removes only derived vectors. Set AGENTMEMORY_VECTOR_BACKFILL=all +# to run the full backfill; otherwise it waits for explicit opt-in. + # VOYAGE_API_KEY=pa-... # Optimised for code embeddings # COHERE_API_KEY=... # General-purpose embeddings diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index c5b24a0d1e..d1e65e5691 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -51,7 +51,7 @@ jobs: engine_pin=$(node --import tsx --input-type=module -e 'import { III_PINNED_VERSION } from "./src/version.ts"; console.log(III_PINNED_VERSION)') engine_bin="$HOME/.cache/agentmemory-release-gate/iii-${engine_pin}-linux-$(node -p process.arch)" test -x "$engine_bin" - AGENTMEMORY_TEST_III="$engine_bin" AGENTMEMORY_REQUIRE_LIVE_TESTS=1 npx vitest run test/compose-supervision-integration.test.ts test/codex-live-daemon.test.ts test/compose-c1-integration.test.ts + AGENTMEMORY_TEST_III="$engine_bin" AGENTMEMORY_REQUIRE_LIVE_TESTS=1 npx vitest run test/compose-supervision-integration.test.ts test/codex-live-daemon.test.ts test/compose-cache-integration.test.ts test/compose-c1-integration.test.ts - run: npm run test:startup:packaged -- --skip-build --out "${{ runner.temp }}/startup-gate" - name: Test packaged vector recovery shell: bash diff --git a/README.md b/README.md index e12cba755a..c0834fb44c 100644 --- a/README.md +++ b/README.md @@ -1114,6 +1114,19 @@ Reasoning-class models (`o1`-style with `` blocks) can return empty `cont Local embeddings ship as an optional dependency but are not enabled by default. Set `EMBEDDING_PROVIDER=local` to opt into `Xenova/all-MiniLM-L6-v2` (384-dim). The first embedding request downloads the model; inference is on-device afterward. Without that setting or a remote embedding key, vectors stay disabled, `mem::search` uses BM25, and `smart-search` can still add existing graph matches. +Local Transformers.js models can be selected with `LOCAL_EMBEDDING_MODEL`, `LOCAL_EMBEDDING_DIMENSIONS`, `LOCAL_EMBEDDING_POOLING` and `LOCAL_EMBEDDING_DTYPE` (defaults: the model above, `384`, `mean`, `q8`). Optional `LOCAL_EMBEDDING_QUERY_PREFIX` and `LOCAL_EMBEDDING_DOCUMENT_PREFIX` preserve exact prefixes, including trailing spaces. `LOCAL_EMBEDDING_CACHE_DIR` selects the pipeline cache and `LOCAL_EMBEDDING_BATCH_SIZE` bounds extraction batches. Use settings and a Transformers.js export compatible with the selected model. The native CLI forwards process environment values literally, including `$` and `${...}`; relative cache paths resolve from the CLI directory. + +In `~/.agentmemory/.env`, quote prefixes that require trailing spaces. For E5-style inputs, for example: + +```dotenv +LOCAL_EMBEDDING_QUERY_PREFIX="query: " +LOCAL_EMBEDDING_DOCUMENT_PREFIX="passage: " +``` + +Unquoted env-file values are trimmed before the provider reads them. + +Startup records the local document-embedding configuration before generating vectors. Changing the model, dimensions, pooling, dtype, document prefix or configured cache directory refuses restoration of both checkpoints and pending vectors, even when dimensions match. Older local indexes without this identity also require an explicit rebuild: back up the store, set `AGENTMEMORY_DROP_STALE_INDEX=true` for one restart, then unset it. This removes derived vectors while retaining memories and observations. Set `AGENTMEMORY_VECTOR_BACKFILL=all` to run the full backfill; otherwise it waits for explicit opt-in. Query-prefix and batch-size changes do not require document vectors to be rebuilt. A cache-directory change conservatively requires rebuilding because it may select different artifacts under the same model ID. This configuration identity does not fingerprint weight files or pin mutable upstream model revisions. For Docker, use a cache path inside the container, such as `/data/models`, and mount or persist that location. + ### Cost-aware model selection Start keyless for capture and keyword recall. Enable `EMBEDDING_PROVIDER=local` for on-device semantic search after the initial model download. LLM-written compression requires a provider and `AGENTMEMORY_AUTO_COMPRESS=true`; it can make a request for each captured observation, so check your provider's current pricing before enabling it. diff --git a/deploy/coolify/docker-compose.yml b/deploy/coolify/docker-compose.yml index e35233996f..c305ad9ce0 100644 --- a/deploy/coolify/docker-compose.yml +++ b/deploy/coolify/docker-compose.yml @@ -15,6 +15,14 @@ services: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:3113,127.0.0.1:3113,[::1]:3113}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:3113,http://127.0.0.1:3113}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + LOCAL_EMBEDDING_MODEL: "${LOCAL_EMBEDDING_MODEL:-}" + LOCAL_EMBEDDING_DIMENSIONS: "${LOCAL_EMBEDDING_DIMENSIONS:-}" + LOCAL_EMBEDDING_POOLING: "${LOCAL_EMBEDDING_POOLING:-}" + LOCAL_EMBEDDING_DTYPE: "${LOCAL_EMBEDDING_DTYPE:-}" + LOCAL_EMBEDDING_QUERY_PREFIX: "${LOCAL_EMBEDDING_QUERY_PREFIX:-}" + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${LOCAL_EMBEDDING_DOCUMENT_PREFIX:-}" + LOCAL_EMBEDDING_CACHE_DIR: "${LOCAL_EMBEDDING_CACHE_DIR:-}" + LOCAL_EMBEDDING_BATCH_SIZE: "${LOCAL_EMBEDDING_BATCH_SIZE:-}" AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" diff --git a/deploy/runtime/worker-compose.yaml b/deploy/runtime/worker-compose.yaml index d3f8b91308..f7abf6e974 100644 --- a/deploy/runtime/worker-compose.yaml +++ b/deploy/runtime/worker-compose.yaml @@ -74,6 +74,14 @@ containers: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:3113,127.0.0.1:3113,[::1]:3113}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:3113,http://127.0.0.1:3113}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + LOCAL_EMBEDDING_MODEL: "${LOCAL_EMBEDDING_MODEL:-}" + LOCAL_EMBEDDING_DIMENSIONS: "${LOCAL_EMBEDDING_DIMENSIONS:-}" + LOCAL_EMBEDDING_POOLING: "${LOCAL_EMBEDDING_POOLING:-}" + LOCAL_EMBEDDING_DTYPE: "${LOCAL_EMBEDDING_DTYPE:-}" + LOCAL_EMBEDDING_QUERY_PREFIX: "${LOCAL_EMBEDDING_QUERY_PREFIX:-}" + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${LOCAL_EMBEDDING_DOCUMENT_PREFIX:-}" + LOCAL_EMBEDDING_CACHE_DIR: "${LOCAL_EMBEDDING_CACHE_DIR:-}" + LOCAL_EMBEDDING_BATCH_SIZE: "${LOCAL_EMBEDDING_BATCH_SIZE:-}" AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" diff --git a/docker-compose.yml b/docker-compose.yml index 0fd899e20e..67e78b9993 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,14 @@ services: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:${AGENTMEMORY_VIEWER_PORT:-3113},127.0.0.1:${AGENTMEMORY_VIEWER_PORT:-3113},[::1]:${AGENTMEMORY_VIEWER_PORT:-3113}}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:${AGENTMEMORY_VIEWER_PORT:-3113},http://127.0.0.1:${AGENTMEMORY_VIEWER_PORT:-3113}}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + LOCAL_EMBEDDING_MODEL: "${LOCAL_EMBEDDING_MODEL:-}" + LOCAL_EMBEDDING_DIMENSIONS: "${LOCAL_EMBEDDING_DIMENSIONS:-}" + LOCAL_EMBEDDING_POOLING: "${LOCAL_EMBEDDING_POOLING:-}" + LOCAL_EMBEDDING_DTYPE: "${LOCAL_EMBEDDING_DTYPE:-}" + LOCAL_EMBEDDING_QUERY_PREFIX: "${LOCAL_EMBEDDING_QUERY_PREFIX:-}" + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${LOCAL_EMBEDDING_DOCUMENT_PREFIX:-}" + LOCAL_EMBEDDING_CACHE_DIR: "${LOCAL_EMBEDDING_CACHE_DIR:-}" + LOCAL_EMBEDDING_BATCH_SIZE: "${LOCAL_EMBEDDING_BATCH_SIZE:-}" AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" diff --git a/scripts/docs-sync.state.json b/scripts/docs-sync.state.json index 7f6afbb042..e08552e6a5 100644 --- a/scripts/docs-sync.state.json +++ b/scripts/docs-sync.state.json @@ -26,7 +26,7 @@ "323" ], "sourceFiles": [ - "262" + "263" ], "loc": [ "62000" diff --git a/scripts/release-gate/startup.mjs b/scripts/release-gate/startup.mjs index e9bfe0f0c4..5062235ef6 100644 --- a/scripts/release-gate/startup.mjs +++ b/scripts/release-gate/startup.mjs @@ -414,8 +414,17 @@ async function main() { const binary = join(i.runtime, "bin/iii"); const launchPid = join(i.dir, "launched-compose.pid"); writeFileSync(binary, `#!${process.execPath}\nconst {writeFileSync}=require('node:fs');\nif(process.argv.includes('--version')) { console.log('iii ${installed.dependencies["iii-sdk"]}'); process.exit(0); }\nwriteFileSync(${JSON.stringify(launchPid)},String(process.pid));\nsetInterval(()=>{},1000);\n`); chmodSync(binary, 0o755); Object.assign(d, await expectFailure(i, [], { AGENTMEMORY_ENGINE_READY_TIMEOUT_MS: "300" }, { message: "The engine process started but the REST API never responded." })); - const pid = Number(readFileSync(launchPid, "utf8")); i.ownedPids.add(pid); await poll("owned Compose engine timeout cleanup", () => !alive(pid), 5_000); + const composePid = Number(i.cli.text.match(/Compose daemon started \(pid (\d+)\)/)?.[1]); + assert(Number.isSafeInteger(composePid) && composePid > 0, `No owned Compose daemon was started: ${i.cli.text}`); + i.ownedPids.add(composePid); + // The 300ms readiness budget can expire before the daemon launches iii on + // a busy runner. That is still a valid timeout; always check the daemon, + // and check its engine child too when the fixture reached its entrypoint. + const pid = existsSync(launchPid) ? Number(readFileSync(launchPid, "utf8")) : null; + if (pid !== null) { assert(Number.isSafeInteger(pid) && pid > 0); i.ownedPids.add(pid); } + await poll("owned Compose engine timeout cleanup", () => !alive(composePid) && (pid === null || !alive(pid)), 5_000); assert(!existsSync(join(i.runtime, "engine-state.json"))); + d.engineEntrypointReached = pid !== null; d.transport = "version-compatible iii fixture accepting compose but never opening the engine port; actual installed CLI"; }); await scenario("cancel-owned-startup", async (i, d) => { diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 21957a44c1..1c8f8abae5 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -1,6 +1,7 @@ import { chmodSync, existsSync, readFileSync, mkdirSync, renameSync, writeFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve, win32 } from "node:path"; import { parse, stringify } from "yaml"; +import { parse as parseDotenv } from "dotenv"; import { MAX_STARTUP_TIMEOUT_MS } from "./readiness.js"; import { getStateSaveIntervalMs } from "../config.js"; @@ -52,6 +53,9 @@ export function composeProjectPaths(dataDir: string): { } const AGENTMEMORY_WORKER_ENV_ALLOWLIST = new Set([ + "LOCAL_EMBEDDING_MODEL", "LOCAL_EMBEDDING_DIMENSIONS", "LOCAL_EMBEDDING_POOLING", + "LOCAL_EMBEDDING_QUERY_PREFIX", "LOCAL_EMBEDDING_DOCUMENT_PREFIX", + "LOCAL_EMBEDDING_CACHE_DIR", "LOCAL_EMBEDDING_BATCH_SIZE", "LOCAL_EMBEDDING_DTYPE", "EMBEDDING_PROVIDER", "AGENTMEMORY_LLM_PROVIDER", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_MODEL", "OPENAI_EMBEDDING_API_KEY", "OPENAI_EMBEDDING_BASE_URL", "OPENAI_EMBEDDING_MODEL", "OPENAI_EMBEDDING_DIMENSIONS", "ANTHROPIC_API_KEY", @@ -365,6 +369,13 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { if (filteredEnvFile) lines.push(` env_file: [${yamlString(filteredEnvFile)}]`); const explicitWorkerEnvironment = Object.entries(options.workerEnvironment ?? {}) .filter(([key, value]) => value !== undefined && AGENTMEMORY_WORKER_ENV_ALLOWLIST.has(key) && !COMPOSE_RESERVED_ENV.has(key)) + .map(([key, value]) => { + const path = windows ? win32 : { isAbsolute, resolve }; + const forwardedValue = key === "LOCAL_EMBEDDING_CACHE_DIR" && value && !path.isAbsolute(value) + ? path.resolve(options.cwd ?? process.cwd(), value) + : value; + return [key, forwardedValue?.replaceAll("$", () => "$$")] as const; + }) .sort(([left], [right]) => left.localeCompare(right)); lines.push( " environment:", @@ -412,10 +423,15 @@ export function writeWorkerCompose(options: ComposeProjectOptions): { file: stri chmodSync(join(projectDir, "iii.worker.yaml"), 0o600); } } - const text = renderWorkerCompose(options); + const envFileContents = options.envFile ? readFileSync(options.envFile, "utf8") : ""; + const cacheDir = options.workerEnvironment?.LOCAL_EMBEDDING_CACHE_DIR ?? + (options.envFile ? parseDotenv(envFileContents).LOCAL_EMBEDDING_CACHE_DIR : undefined); + const text = renderWorkerCompose(cacheDir === undefined ? options : { + ...options, workerEnvironment: { ...options.workerEnvironment, LOCAL_EMBEDDING_CACHE_DIR: cacheDir }, + }); if (options.envFile) { const filteredPath = join(paths.directory, ".agentmemory-filtered.env"); - const entries = readFileSync(options.envFile, "utf8").split(/\r?\n/).filter((line) => { + const entries = envFileContents.split(/\r?\n/).filter((line) => { const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/); return !match || !COMPOSE_RESERVED_ENV.has(match[1]!); }); diff --git a/src/functions/search.ts b/src/functions/search.ts index 201ae53d84..bb0f8a0458 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -124,7 +124,7 @@ export async function rankMemoryIds( let semantic: Array<{ obsId: string }> = [] if (vectorIndex && vectorIndex.size > 0 && currentEmbeddingProvider) { try { - const embedding = await currentEmbeddingProvider.embed(query) + const embedding = await (currentEmbeddingProvider.embedQuery?.(query) ?? currentEmbeddingProvider.embed(query)) semantic = vectorIndex .search(embedding, fetchLimit, "memory", (id) => getSearchIndex().layerOf(id)) } catch (err) { diff --git a/src/index.ts b/src/index.ts index cda1df0641..e6299f35a6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,7 @@ import { VectorIndex } from "./state/vector-index.js"; import type { SearchLayer } from "./state/search-layer.js"; import { HybridSearch } from "./state/hybrid-search.js"; import { IndexPersistence } from "./state/index-persistence.js"; +import { ensureEmbeddingIdentity } from "./state/embedding-identity.js"; import { SHUTDOWN_FLUSH_TIMEOUT_MS, SHUTDOWN_HARD_EXIT_MS, runStartupCleanup, settleWithin } from "./shutdown.js"; import { registerPrivacyFunction, withWriteScrubbing } from "./functions/privacy.js"; import { withRegistrationDefaults } from "./runtime/registration.js"; @@ -274,6 +275,7 @@ async function main() { bootWarn(`Waiting for the state worker to register state::get and state::list (${error})`), }); assertStateReady(stateReadiness); + await ensureEmbeddingIdentity(kv, embeddingProvider, isDropStaleIndexEnabled()); if (stateReadiness.attempts > 1) { bootLog( `State worker ready after ${stateReadiness.waitedMs}ms (${stateReadiness.attempts} attempts)`, diff --git a/src/providers/embedding/index.ts b/src/providers/embedding/index.ts index 58f48a9fe8..8799a988ec 100644 --- a/src/providers/embedding/index.ts +++ b/src/providers/embedding/index.ts @@ -72,6 +72,9 @@ export function withDimensionGuard(provider: EmbeddingProvider): EmbeddingProvid out.forEach((v, i) => check(v, `embedBatch[${i}]`)); return out; }; + if (provider.embedQuery) { + wrapped.embedQuery = async (t) => check(await provider.embedQuery!(t), "embedQuery"); + } if (provider.embedImage) { wrapped.embedImage = async (s: string) => check(await provider.embedImage!(s), "embedImage"); diff --git a/src/providers/embedding/local.ts b/src/providers/embedding/local.ts index 7646815df6..9459de4aaf 100644 --- a/src/providers/embedding/local.ts +++ b/src/providers/embedding/local.ts @@ -1,47 +1,104 @@ import type { EmbeddingProvider } from "../../types.js"; +import type { DataType } from "@huggingface/transformers"; +import { resolve } from "node:path"; type FeatureExtractor = ( texts: string[], options: { pooling: string; normalize: boolean }, ) => Promise<{ tolist: () => number[][] }>; +function positiveInteger(name: string, fallback: number): number { + const raw = process.env[name]; + if (!raw?.trim()) return fallback; + const value = Number(raw); + if (!/^\d+$/.test(raw) || !Number.isSafeInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + export class LocalEmbeddingProvider implements EmbeddingProvider { - readonly name = "local"; - readonly dimensions = 384; - private extractor: FeatureExtractor | null = null; + readonly model = process.env.LOCAL_EMBEDDING_MODEL?.trim() || "Xenova/all-MiniLM-L6-v2"; + readonly name = this.model === "Xenova/all-MiniLM-L6-v2" ? "local" : `local:${this.model}`; + readonly dimensions = positiveInteger("LOCAL_EMBEDDING_DIMENSIONS", 384); + readonly pooling = process.env.LOCAL_EMBEDDING_POOLING?.trim() || "mean"; + readonly dtype = (process.env.LOCAL_EMBEDDING_DTYPE?.trim() || "q8") as DataType; + readonly cacheDir = process.env.LOCAL_EMBEDDING_CACHE_DIR ? resolve(process.env.LOCAL_EMBEDDING_CACHE_DIR) : undefined; + private readonly batchSize = positiveInteger("LOCAL_EMBEDDING_BATCH_SIZE", Number.MAX_SAFE_INTEGER); + private readonly queryPrefix = process.env.LOCAL_EMBEDDING_QUERY_PREFIX || ""; + private readonly documentPrefix = process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX || ""; + private extractor: Promise | null = null; + + /** A different cache may contain different artifacts under the same model identifier. */ + get indexIdentity(): string { + return JSON.stringify(["local-v2", this.model, this.dimensions, this.pooling, this.dtype, this.documentPrefix, this.cacheDir ?? null]); + } + + constructor() { + if (!["mean", "cls", "first_token", "last_token", "eos"].includes(this.pooling)) { + throw new Error("LOCAL_EMBEDDING_POOLING must be mean, cls, first_token, last_token, or eos"); + } + if (!["auto", "fp32", "fp16", "q8", "int8", "uint8", "q4", "bnb4", "q4f16", "q2", "q2f16", "q1", "q1f16"].includes(this.dtype)) { + throw new Error("LOCAL_EMBEDDING_DTYPE must be a supported Transformers.js dtype"); + } + } async embed(text: string): Promise { const [result] = await this.embedBatch([text]); return result; } + async embedQuery(text: string): Promise { + const [result] = await this.extract([this.queryPrefix + text]); + return result; + } + async embedBatch(texts: string[]): Promise { + return this.extract(texts.map((text) => this.documentPrefix + text)); + } + + private async extract(texts: string[]): Promise { + if (!texts.length) return []; const extractor = await this.getExtractor(); - const output = await extractor(texts, { - pooling: "mean", - normalize: true, - }); - return output.tolist().map((v) => new Float32Array(v)); + const vectors: Float32Array[] = []; + for (let offset = 0; offset < texts.length; offset += this.batchSize) { + const output = await extractor(texts.slice(offset, offset + this.batchSize), { + pooling: this.pooling, + normalize: true, + }); + const rows = output.tolist(); + if (rows.length !== Math.min(this.batchSize, texts.length - offset) || + rows.some((row) => row.length !== this.dimensions || row.some((n) => !Number.isFinite(n)))) { + throw new Error(`Local embedding output does not match configured dimensions (${this.dimensions})`); + } + vectors.push(...rows.map((row) => new Float32Array(row))); + } + return vectors; } - private async getExtractor() { - if (this.extractor) return this.extractor; + private getExtractor(): Promise { + if (!this.extractor) { + this.extractor = this.loadExtractor().catch((error) => { + this.extractor = null; + throw error; + }); + } + return this.extractor; + } + + private async loadExtractor(): Promise { let transformers: typeof import("@huggingface/transformers"); try { transformers = await import("@huggingface/transformers"); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ERR_MODULE_NOT_FOUND") { - throw new Error( - "Install @huggingface/transformers for local embeddings: npm install @huggingface/transformers", - ); + throw new Error("Install @huggingface/transformers for local embeddings: npm install @huggingface/transformers"); } throw err; } - this.extractor = (await transformers.pipeline( - "feature-extraction", - "Xenova/all-MiniLM-L6-v2", - { dtype: "q8" }, - )) as FeatureExtractor; - return this.extractor; + return await transformers.pipeline("feature-extraction", this.model, { + dtype: this.dtype, + ...(this.cacheDir ? { cache_dir: this.cacheDir } : {}), + }) as FeatureExtractor; } } diff --git a/src/state/embedding-identity.ts b/src/state/embedding-identity.ts new file mode 100644 index 0000000000..ae5a81a915 --- /dev/null +++ b/src/state/embedding-identity.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; +import type { EmbeddingProvider } from "../types.js"; +import type { StateKV } from "./kv.js"; +import { KV } from "./schema.js"; +import { safeAudit } from "../functions/audit.js"; + +const IDENTITY_KEY = "vectors:embedding-identity"; +const VECTOR_KEYS = ["vectors", "vectors:manifest", "vectors:meta"]; +const VECTOR_SCOPE_PREFIXES = [`${KV.bm25Index}:vec:`, `${KV.bm25Index}:vectors:`]; +const DELETE_CONCURRENCY = 32; + +async function deleteKeys(kv: StateKV, scope: string, keys: string[]): Promise { + for (let offset = 0; offset < keys.length; offset += DELETE_CONCURRENCY) { + const results = await Promise.allSettled(keys.slice(offset, offset + DELETE_CONCURRENCY).map((key) => kv.delete(scope, key))); + const failed = results.find((result) => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; + } +} + +/** Refuse unverified vectors before startup can migrate checkpoints or replay pending writes. */ +export async function ensureEmbeddingIdentity( + kv: StateKV, + provider: EmbeddingProvider | null, + dropStale: boolean, +): Promise { + if (!provider) return; + const identity = createHash("sha256") + .update(provider.indexIdentity ?? JSON.stringify([provider.name, provider.dimensions])) + .digest("hex"); + const saved = await kv.get<{ v: number; identity: string }>(KV.bm25Index, IDENTITY_KEY); + if (!saved && !provider.indexIdentity) return; + if (saved?.v === 1 && saved.identity === identity) return; + + const groups = await kv.listGroups(); + const vectorScopes = groups.filter((scope) => VECTOR_SCOPE_PREFIXES.some((prefix) => scope.startsWith(prefix))); + const snapshots = await Promise.all(VECTOR_KEYS.map((key) => kv.get(KV.bm25Index, key))); + const pending = await kv.listKeys(KV.vectorPendingLog); + const hasVectors = vectorScopes.length > 0 || snapshots.some((value) => value != null) || pending.length > 0; + if (hasVectors && !dropStale) { + throw new Error( + "Persisted vector embedding configuration differs or is unknown. Refusing to restore vectors or replay the pending log. " + + "Switch back to the original configuration, or back up the store and set AGENTMEMORY_DROP_STALE_INDEX=true once to discard derived vectors, and AGENTMEMORY_VECTOR_BACKFILL=all to run the full backfill. " + + "Unidentified indexes from earlier versions also require this explicit rebuild.", + ); + } + if (hasVectors) { + await kv.set(KV.bm25Index, "vectors:backfill", { v: 2, since: null }); + for (const scope of vectorScopes) { + await deleteKeys(kv, scope, await kv.listKeys(scope)); + } + await deleteKeys(kv, KV.vectorPendingLog, pending); + await deleteKeys(kv, KV.bm25Index, VECTOR_KEYS); + await safeAudit(kv, "index_persist", "mem::embedding-identity", [], { + action: "rebuild", vectorScopes: vectorScopes.length, pendingEntries: pending.length, + }); + } + await kv.set(KV.bm25Index, IDENTITY_KEY, { v: 1, identity }); +} diff --git a/src/state/hybrid-search.ts b/src/state/hybrid-search.ts index cdd13e5e83..dc559e96f4 100644 --- a/src/state/hybrid-search.ts +++ b/src/state/hybrid-search.ts @@ -118,7 +118,7 @@ export class HybridSearch { try { if (request) { if (!request.embedding || request.embedding.query !== query) { - request.embedding = { query, value: this.embeddingProvider.embed(query) }; + request.embedding = { query, value: (this.embeddingProvider.embedQuery?.(query) ?? this.embeddingProvider.embed(query)) }; } const embedding = request.embedding; try { @@ -128,7 +128,7 @@ export class HybridSearch { throw error; } } else { - queryEmbedding = await this.embeddingProvider.embed(query); + queryEmbedding = await (this.embeddingProvider.embedQuery?.(query) ?? this.embeddingProvider.embed(query)); } vectorResults = this.vector.search(queryEmbedding, poolLimit, targetLayer, (id) => this.bm25.layerOf(id)); } catch { diff --git a/src/types.ts b/src/types.ts index d4061c1d58..4c1dabbd12 100644 --- a/src/types.ts +++ b/src/types.ts @@ -318,7 +318,9 @@ export interface MemorySlot { export interface EmbeddingProvider { name: string; dimensions: number; + indexIdentity?: string; embed(text: string): Promise; + embedQuery?(text: string): Promise; embedBatch(texts: string[]): Promise; embedImage?(src: string): Promise; } diff --git a/test/cli-lifecycle-safety.test.ts b/test/cli-lifecycle-safety.test.ts index 3992b2c5f0..28dc4b1f3a 100644 --- a/test/cli-lifecycle-safety.test.ts +++ b/test/cli-lifecycle-safety.test.ts @@ -153,7 +153,8 @@ describe("native removal shutdown", () => { expect(existsSync(result.enginePidfile)).toBe(true); expect(existsSync(result.engineState)).toBe(true); expect(existsSync(result.privateBin)).toBe(true); - }); + // Starting the isolated CLI can exceed the default budget on busy runners. + }, 30_000); it("refuses an instance-scoped global uninstall before deleting shared assets", () => { const { result, privateBin } = runInstanceRemove(); diff --git a/test/compose-cache-integration.test.ts b/test/compose-cache-integration.test.ts new file mode 100644 index 0000000000..e991c71c18 --- /dev/null +++ b/test/compose-cache-integration.test.ts @@ -0,0 +1,106 @@ +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { setTimeout as sleep } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import { composeNamespace, writeWorkerCompose } from "../src/cli/compose-project.js"; +import { III_PINNED_VERSION } from "../src/version.js"; +import { prepareProbe, probeLifetime, stopProbe } from "./helpers/compose-probe-lifetime.js"; + +const engine = process.env.AGENTMEMORY_TEST_III; +const version = engine ? spawnSync(engine, ["--version"], { encoding: "utf8", timeout: 3000 }) : null; +const available = version?.status === 0 && version.stdout.trim() === III_PINNED_VERSION; +const skipped = !available || process.platform === "win32" || process.env.AGENTMEMORY_SKIP_LIVE_TESTS === "1"; +if (process.env.AGENTMEMORY_REQUIRE_LIVE_TESTS === "1" && skipped) { + throw new Error(`Required Compose cache path test needs iii ${III_PINNED_VERSION} and live tests enabled on macOS or Linux`); +} + +async function freePort(): Promise { + const server = createServer(); + await new Promise((accept, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", accept); + }); + const port = (server.address() as { port: number }).port; + await new Promise((accept) => server.close(() => accept())); + return port; +} + + +describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { + it.each(["env-file", "process"])("preserves literal cache paths from %s through Compose interpolation", async (source) => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-compose-cache-")); + let child: ChildProcess | undefined; + try { + prepareProbe(root); + const pkg = join(root, "worker"); + const snapshot = join(root, "received.json"); + mkdirSync(join(pkg, "dist"), { recursive: true }); + const value = "model-cache/${HOME}/$MODELS/$$/models"; + const cwd = join(root, "caller$directory"); + const expected = { + cache: join(cwd, value), + document: source === "env-file" ? "passage: ${HOME} $$ " : "passage: ${AGENTMEMORY_TEST_UNDEFINED} $$ ", + query: "query: ${HOME} $$ ", + endpoint: "https://literal.example/${AGENTMEMORY_TEST_UNDEFINED}/$$", + }; + const envFile = join(root, "worker.env"); + writeFileSync(envFile, `export LOCAL_EMBEDDING_CACHE_DIR='${source === "env-file" ? value : "ignored-cache"}'\n`); + const sdk = pathToFileURL(createRequire(import.meta.url).resolve("iii-sdk")).href; + writeFileSync(join(pkg, "dist/index.mjs"), ` + import { existsSync, renameSync, writeFileSync } from "node:fs"; + ${probeLifetime(root)} + const { default: sdkModule } = await import(${JSON.stringify(sdk)}); + const { registerWorker } = sdkModule; + const worker = registerWorker(process.env.III_URL, { workerName: "agentmemory" }); + const received = { cache: process.env.LOCAL_EMBEDDING_CACHE_DIR, + document: process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX, query: process.env.LOCAL_EMBEDDING_QUERY_PREFIX, + endpoint: process.env.OPENAI_BASE_URL }; + worker.registerFunction("test::env", async () => received); + writeFileSync(${JSON.stringify(`${snapshot}.tmp`)}, JSON.stringify(received)); + renameSync(${JSON.stringify(`${snapshot}.tmp`)}, ${JSON.stringify(snapshot)}); + `); + const ports = { restPort: await freePort(), viewerPort: await freePort(), enginePort: await freePort() }; + const project = writeWorkerCompose({ + ports, projectRoot: pkg, dataDir: join(root, "data"), homeDir: join(root, "home"), + nodeBin: process.execPath, cwd, envFile, + workerEnvironment: { + ...(source === "process" ? { LOCAL_EMBEDDING_CACHE_DIR: value } : {}), + LOCAL_EMBEDDING_DOCUMENT_PREFIX: expected.document, + LOCAL_EMBEDDING_QUERY_PREFIX: expected.query, + OPENAI_BASE_URL: expected.endpoint, + }, + }); + const header = project.text.slice(0, project.text.indexOf("containers:\n")); + const app = project.text.slice(project.text.indexOf(" agentmemory:\n")) + .replace(" start_after: [state, http]", " start_after: []"); + writeFileSync(project.file, `${header}containers:\n${app}`); + child = spawn(engine!, ["compose", "--namespace", composeNamespace(ports.restPort), "--up", "--file", project.file], { + env: { PATH: process.env.PATH, HOME: join(root, "home"), USERPROFILE: join(root, "home"), + III_COMPOSE_STATE_DIR: project.stateDir, III_TELEMETRY_ENABLED: "false", III_BUNDLE_WORKERS_DISABLED: "1", RUST_LOG: "error" }, + detached: true, stdio: ["ignore", "ignore", "pipe"], + }); + let logs = ""; + child.stderr?.on("data", (chunk) => { logs = `${logs}${chunk}`.slice(-16000); }); + const deadline = Date.now() + 20000; + while (Date.now() < deadline) { + try { + expect(JSON.parse(readFileSync(snapshot, "utf8"))).toEqual(expected); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (child.exitCode !== null || child.signalCode !== null) throw new Error(`Compose exited before the worker received its value: ${logs}`); + await sleep(50); + } + throw new Error(`Worker environment snapshot timed out: ${logs}`); + } finally { + try { await stopProbe(child, root); } + finally { rmSync(root, { recursive: true, force: true }); } + } + }, 30000); +}); diff --git a/test/compose-probe-cleanup.test.ts b/test/compose-probe-cleanup.test.ts new file mode 100644 index 0000000000..4da4ca2882 --- /dev/null +++ b/test/compose-probe-cleanup.test.ts @@ -0,0 +1,73 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { EventEmitter } from "node:events"; +import { prepareProbe, probeLifetime, stopProbe } from "./helpers/compose-probe-lifetime.js"; + +it("stops the owned probe when its Compose parent has already exited", async () => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-probe-cleanup-")); + let parent: ChildProcess | undefined; + let probe: ChildProcess | undefined; + try { + prepareProbe(root); + parent = spawn(process.execPath, ["-e", "process.exit(0)"], { stdio: "ignore" }); + await new Promise((accept) => parent!.once("exit", () => accept())); + probe = spawn(process.execPath, ["--input-type=module", "-e", ` + import { existsSync, writeFileSync } from "node:fs"; + ${probeLifetime(root)} + `], { stdio: "ignore" }); + const deadline = Date.now() + 5000; + while (!existsSync(join(root, "probe-pid"))) { + if (Date.now() >= deadline) throw new Error("Test probe did not start"); + await sleep(20); + } + await stopProbe(parent, root); + expect(parent.exitCode).toBe(0); + expect(probe.exitCode).toBe(0); + } finally { + try { await stopProbe(parent, root); } + finally { + if (probe && probe.exitCode === null && probe.signalCode === null) { + const exited = new Promise((accept) => probe!.once("exit", () => accept())); + probe.kill("SIGKILL"); + await exited; + } + rmSync(root, { recursive: true, force: true }); + } + } +}, 15000); + + +it.each(["parent", "probe", "both"])("preserves cleanup diagnostics: %s", async (failure) => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-probe-errors-")); + const killFailure = Object.assign(new Error("process-group kill denied"), { code: "EPERM" }); + const child = Object.assign(new EventEmitter(), { + exitCode: null, signalCode: null, pid: 123456, kill: vi.fn(() => true), + }) as unknown as ChildProcess; + const kill = vi.spyOn(process, "kill").mockImplementation((pid) => { + if (pid < 0) throw killFailure; + if (failure === "parent") throw Object.assign(new Error("probe already exited"), { code: "ESRCH" }); + return true; + }); + try { + prepareProbe(root); + writeFileSync(join(root, "probe-pid"), "123456"); + let caught: unknown; + try { await stopProbe(failure === "probe" ? undefined : child, root, 1); } + catch (error) { caught = error; } + if (failure === "parent") expect(caught).toBe(killFailure); + else if (failure === "probe") expect(caught).toHaveProperty("message", "Test probe did not exit after its lifetime ended"); + else { + expect(caught).toBeInstanceOf(AggregateError); + expect((caught as AggregateError).errors).toEqual([ + killFailure, expect.objectContaining({ message: "Test probe did not exit after its lifetime ended" }), + ]); + } + } finally { + kill.mockRestore(); + rmSync(root, { recursive: true, force: true }); + } +}, 15000); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index 0a7047eddf..dcb922ad49 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -24,6 +24,82 @@ const options = (overrides: Record = {}) => ({ }); describe("Worker Compose project", () => { + it("forwards all local embedding settings to the application worker", () => { + const settings = { + LOCAL_EMBEDDING_MODEL: "test/model", LOCAL_EMBEDDING_DIMENSIONS: "3", + LOCAL_EMBEDDING_POOLING: "cls", LOCAL_EMBEDDING_QUERY_PREFIX: "query: ", + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "passage: ", LOCAL_EMBEDDING_CACHE_DIR: join(root, "model-cache"), + LOCAL_EMBEDDING_BATCH_SIZE: "2", LOCAL_EMBEDDING_DTYPE: "fp32", + }; + const app = parse(renderWorkerCompose(options({ workerEnvironment: settings }))).containers.agentmemory; + expect(app.environment).toMatchObject(settings); + }); + + it("resolves a relative model cache against the CLI directory and escapes line breaks", () => { + const cwd = join(root, "project"); + const value = "model-cache\n雪"; + const text = renderWorkerCompose(options({ cwd, workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: value } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(cwd, value)); + expect(text).toContain("model-cache\\n雪"); + }); + + it("preserves literal process environment values after cache path normalization", () => { + const text = renderWorkerCompose(options({ cwd: join(root, "caller$directory"), workerEnvironment: { + LOCAL_EMBEDDING_CACHE_DIR: "${HOME}/$MODELS/$$/cache", + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${DOCUMENT_PREFIX}", + LOCAL_EMBEDDING_QUERY_PREFIX: "query: ${HOME} $$ ", + LOCAL_EMBEDDING_MODEL: "models/${MODEL}", + OPENAI_BASE_URL: "${ENDPOINT}", + } })); + const env = parse(text).containers.agentmemory.environment; + expect(env.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(root, "caller$$directory", "$${HOME}/$$MODELS/$$$$/cache")); + expect(env.LOCAL_EMBEDDING_DOCUMENT_PREFIX).toBe("$${DOCUMENT_PREFIX}"); + expect(env.LOCAL_EMBEDDING_QUERY_PREFIX).toBe("query: $${HOME} $$$$ "); + expect(env.LOCAL_EMBEDDING_MODEL).toBe("models/$${MODEL}"); + expect(env.OPENAI_BASE_URL).toBe("$${ENDPOINT}"); + }); + + it("escapes literal dollars in absolute Windows cache paths", () => { + const text = renderWorkerCompose(options({ windows: true, workerEnvironment: { + LOCAL_EMBEDDING_CACHE_DIR: "C:\\$models\\${HOME}\\cache", + } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe("C:\\$$models\\$${HOME}\\cache"); + }); + + it("resolves Windows cache paths against the CLI directory", () => { + const text = renderWorkerCompose(options({ windows: true, cwd: "C:\\project", nodeBin: "C:\\node\\node.exe", workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "model-cache" } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe("C:\\project\\model-cache"); + }); + + it("preserves an absolute cache path when forwarding to the worker", () => { + const value = join(root, "cache"); + const text = renderWorkerCompose(options({ cwd: join(root, "other"), workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: value } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(value); + }); + + it.each(["LOCAL_EMBEDDING_CACHE_DIR=model-cache", "export LOCAL_EMBEDDING_CACHE_DIR='model-cache with spaces'"])( + "resolves an effective cache value from the env file: %s", (line) => { + mkdirSync(root, { recursive: true }); + const envFile = join(root, "model.env"); + writeFileSync(envFile, `${line}\nUNRELATED_SETTING=kept-in-file\n`); + const cwd = join(root, "project"); + const result = writeWorkerCompose(options({ envFile, cwd, workerEnvironment: {} })); + const env = parse(result.text).containers.agentmemory.environment; + const relative = line.startsWith("export ") ? "model-cache with spaces" : "model-cache"; + expect(env.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(cwd, relative)); + expect(env).not.toHaveProperty("UNRELATED_SETTING"); + }, + ); + + it("keeps process-environment cache precedence over the env file", () => { + mkdirSync(root, { recursive: true }); + const envFile = join(root, "model.env"); + writeFileSync(envFile, "export LOCAL_EMBEDDING_CACHE_DIR=file-cache\n"); + const cwd = join(root, "project"); + const result = writeWorkerCompose(options({ envFile, cwd, workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "process-cache" } })); + expect(parse(result.text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(cwd, "process-cache")); + }); + it.each(Array.from({ length: 32 }, (_, offset) => 0x80 + offset))( "escapes C1 character U+%s in single-line and multiline scalars", (code) => { for (const value of [`first${String.fromCharCode(code)}second`, `first\n${String.fromCharCode(code)}second`]) { diff --git a/test/connect-pi.test.ts b/test/connect-pi.test.ts index d38ba7109b..3b09e61e39 100644 --- a/test/connect-pi.test.ts +++ b/test/connect-pi.test.ts @@ -135,5 +135,6 @@ describe("Pi package archive", () => { expect(readFileSync(join(root, "LICENSE"), "utf8")).toBe(readFileSync("LICENSE", "utf8")); expect(JSON.parse(readFileSync("package.json", "utf8")).files).toContain("integrations/pi/"); } finally { rmSync(dir, { recursive: true, force: true }); } - }); + // This checks archive contents, not npm/tar wall-clock performance. + }, 30_000); }); diff --git a/test/deploy-runtime.test.ts b/test/deploy-runtime.test.ts index d3468a45cf..99f931dcea 100644 --- a/test/deploy-runtime.test.ts +++ b/test/deploy-runtime.test.ts @@ -18,6 +18,17 @@ function fixture() { afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); describe("deployment entrypoint", () => { + it("forwards every local model setting through Docker and runtime Compose", () => { + const keys = ["MODEL", "DIMENSIONS", "POOLING", "DTYPE", "QUERY_PREFIX", "DOCUMENT_PREFIX", "CACHE_DIR", "BATCH_SIZE"]; + for (const file of ["docker-compose.yml", "deploy/coolify/docker-compose.yml", "deploy/runtime/worker-compose.yaml"]) { + const config = parse(readFileSync(file, "utf8")); + const env = file.endsWith("worker-compose.yaml") ? config.containers.agentmemory.environment : config.services.agentmemory.environment; + for (const key of keys) { + const name = `LOCAL_EMBEDDING_${key}`; + expect(env[name]).toBe(`\${${name}:-}`); + } + } + }); it.each(["2000", "500"])("generates file and Redis configurations with save interval %s", (interval) => { const root = fixture(); const program = entrypoint.split("node <<'NODE'\n")[1].split("\nNODE")[0] diff --git a/test/embedding-identity.test.ts b/test/embedding-identity.test.ts new file mode 100644 index 0000000000..7988356c38 --- /dev/null +++ b/test/embedding-identity.test.ts @@ -0,0 +1,146 @@ +import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; +import { ensureEmbeddingIdentity } from "../src/state/embedding-identity.js"; +import { IndexPersistence } from "../src/state/index-persistence.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { LocalEmbeddingProvider } from "../src/providers/embedding/local.js"; +import { withDimensionGuard } from "../src/providers/embedding/index.js"; +import { KV } from "../src/state/schema.js"; + +function mockKV() { + const store = new Map>(); + return { + store, + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, value: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, value); + return value; + }, + delete: async (scope: string, key: string) => { store.get(scope)?.delete(key); }, + list: async (scope: string): Promise => [...(store.get(scope)?.values() ?? [])] as T[], + listKeys: async (scope: string) => [...(store.get(scope)?.keys() ?? [])], + listGroups: async () => [...store].filter(([, rows]) => rows.size > 0).map(([scope]) => scope), + }; +} + +beforeEach(() => { + for (const key of ["MODEL", "DIMENSIONS", "POOLING", "DTYPE", "BATCH_SIZE", "QUERY_PREFIX", "DOCUMENT_PREFIX", "CACHE_DIR"]) { + vi.stubEnv(`LOCAL_EMBEDDING_${key}`, undefined); + } +}); +afterEach(() => vi.unstubAllEnvs()); + +describe("persisted embedding identity", () => { + it("restores a checkpoint and pending vectors when configuration matches", async () => { + const kv = mockKV(); + const p = withDimensionGuard(new LocalEmbeddingProvider()); + await ensureEmbeddingIdentity(kv as never, p, false); + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector); + vector.add("obs_saved", "ses", new Float32Array(384).fill(1)); + await persistence.save(); + vector.add("obs_pending", "ses", new Float32Array(384).fill(1)); + await persistence.flushPendingLog(); + persistence.stop(); + await ensureEmbeddingIdentity(kv as never, p, false); + const restored = new VectorIndex(); + const next = new IndexPersistence(kv as never, restored); + restored.restoreFrom((await next.load()).vector!); + await next.replayPendingLog(384); + next.stop(); + expect(restored.has("obs_saved")).toBe(true); + expect(restored.has("obs_pending")).toBe(true); + }); + + it.each([["MODEL", "other/model"], ["POOLING", "cls"], ["DTYPE", "fp32"], ["DOCUMENT_PREFIX", "passage: "], ["CACHE_DIR", "other-cache"]])( + "refuses same-dimension vectors after changing %s without deleting them", async (key, value) => { + const kv = mockKV(); + await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false); + await kv.set(KV.bm25Index, "vectors:meta", { v: 3, bucketCount: 0 }); + vi.stubEnv(`LOCAL_EMBEDDING_${key}`, value); + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).rejects.toThrow("Refusing to restore"); + expect(await kv.get(KV.bm25Index, "vectors:meta")).not.toBeNull(); + }, + ); + + it("does not rebuild document vectors for query-prefix or batch changes", async () => { + const kv = mockKV(); + await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false); + await kv.set(KV.bm25Index, "vectors:meta", { v: 3 }); + vi.stubEnv("LOCAL_EMBEDDING_QUERY_PREFIX", "query: "); + vi.stubEnv("LOCAL_EMBEDDING_BATCH_SIZE", "2"); + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).resolves.toBeUndefined(); + }); + + it.each(["vectors", "vectors:manifest", "vectors:meta", "pending"])("rejects unidentified %s", async (key) => { + const kv = mockKV(); + await kv.set(key === "pending" ? KV.vectorPendingLog : KV.bm25Index, key, {}); + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).rejects.toThrow("unknown"); + }); + + it("explicitly drops only derived vectors and requests a full backfill before adopting the identity", async () => { + const kv = mockKV(); + for (const key of ["vectors", "vectors:manifest", "vectors:meta"]) await kv.set(KV.bm25Index, key, {}); + await kv.set(`${KV.bm25Index}:vec:0000`, "obs", {}); + await kv.set(`${KV.bm25Index}:vectors:legacy:00000`, "data", "legacy"); + await kv.set(KV.vectorPendingLog, "obs", {}); + await kv.set(KV.memories, "mem_keep", "memory"); + await kv.set(KV.observations("ses"), "obs_keep", "observation"); + await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true); + expect(await kv.listKeys(KV.vectorPendingLog)).toEqual([]); + expect(await kv.listKeys(`${KV.bm25Index}:vec:0000`)).toEqual([]); + expect(await kv.get(KV.bm25Index, "vectors:meta")).toBeNull(); + expect(await kv.get(KV.bm25Index, "vectors:backfill")).toEqual({ v: 2, since: null }); + expect(await kv.get(KV.memories, "mem_keep")).toBe("memory"); + expect(await kv.get(KV.observations("ses"), "obs_keep")).toBe("observation"); + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).resolves.toBeUndefined(); + }); + + it("leaves identity unclaimed when deletion fails so a restart cannot load partially stale vectors", async () => { + const kv = mockKV(); + await kv.set(KV.vectorPendingLog, "obs", {}); + kv.delete = vi.fn().mockRejectedValue(new Error("disk unavailable")); + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true)).rejects.toThrow("disk unavailable"); + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).rejects.toThrow("unknown"); + }); + + it("overlaps independent deletions with bounded concurrency and adopts identity only after they finish", async () => { + const kv = mockKV(); + for (let i = 0; i < 70; i++) await kv.set(KV.vectorPendingLog, `obs_${i}`, {}); + const remove = kv.delete; + let active = 0; + let peak = 0; + kv.delete = async (scope, key) => { + active++; peak = Math.max(peak, active); + expect(await kv.get(KV.bm25Index, "vectors:embedding-identity")).toBeNull(); + await new Promise((resolve) => setTimeout(resolve, 1)); + await remove(scope, key); + active--; + }; + await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true); + expect(peak).toBeGreaterThan(1); + expect(peak).toBeLessThanOrEqual(32); + expect(active).toBe(0); + expect(await kv.listKeys(KV.vectorPendingLog)).toEqual([]); + expect(await kv.get(KV.bm25Index, "vectors:embedding-identity")).not.toBeNull(); + }); + + it("settles an in-flight deletion batch before surfacing an error", async () => { + const kv = mockKV(); + for (let i = 0; i < 40; i++) await kv.set(KV.vectorPendingLog, `obs_${i}`, {}); + const remove = kv.delete; + let active = 0; + kv.delete = async (scope, key) => { + active++; + try { + if (key === "obs_0") throw new Error("delete failed"); + await new Promise((resolve) => setTimeout(resolve, 1)); + await remove(scope, key); + } finally { active--; } + }; + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true)).rejects.toThrow("delete failed"); + expect(active).toBe(0); + expect(await kv.get(KV.bm25Index, "vectors:embedding-identity")).toBeNull(); + expect(await kv.get(KV.vectorPendingLog, "obs_39")).not.toBeNull(); + }); +}); diff --git a/test/graph.test.ts b/test/graph.test.ts index 1556a59023..23d1d695da 100644 --- a/test/graph.test.ts +++ b/test/graph.test.ts @@ -256,7 +256,8 @@ describe("Graph Functions", () => { expect(unbounded.limit).toBe(500); expect(unbounded.offset).toBe(0); expect(new Set(unbounded.nodes.map(node => node.id)).size).toBe(500); - }); + // Seeding and snapshotting 1200 nodes is a functional check, not a 5s benchmark. + }, 30_000); it("honors limit and offset for paged graph-query traversal", async () => { for (let i = 0; i < 50; i++) { diff --git a/test/helpers/compose-probe-lifetime.ts b/test/helpers/compose-probe-lifetime.ts new file mode 100644 index 0000000000..5675c6c364 --- /dev/null +++ b/test/helpers/compose-probe-lifetime.ts @@ -0,0 +1,60 @@ +import type { ChildProcess } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +export function prepareProbe(root: string): void { + writeFileSync(join(root, "probe-active"), "active"); +} + +export function probeLifetime(root: string): string { + // The probe must stop even when Compose exits without a working reaper. + return ` + setInterval(() => { + if (!existsSync(${JSON.stringify(join(root, "probe-active"))})) process.exit(0); + }, 50); + writeFileSync(${JSON.stringify(join(root, "probe-pid"))}, String(process.pid)); + if (!existsSync(${JSON.stringify(join(root, "probe-active"))})) process.exit(0); + `; +} + +function running(pid: number): boolean { + try { process.kill(pid, 0); return true; } + catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } +} + +export async function stopProbe(child: ChildProcess | undefined, root: string, timeoutMs = 5000): Promise { + const errors: unknown[] = []; + try { rmSync(join(root, "probe-active"), { force: true }); } + catch (error) { errors.push(error); } + try { + if (child && child.exitCode === null && child.signalCode === null) { + const exited = new Promise((accept) => child.once("exit", () => accept())); + child.kill("SIGINT"); + const ended = await Promise.race([exited.then(() => true), sleep(timeoutMs).then(() => false)]); + if (!ended && child.pid) { + try { process.kill(-child.pid, "SIGKILL"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } + await exited; + } + } + } catch (error) { errors.push(error); } + try { + let pid: number | undefined; + try { pid = Number(readFileSync(join(root, "probe-pid"), "utf8")); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (pid !== undefined) { + if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid test probe pid"); + const deadline = Date.now() + timeoutMs; + while (running(pid)) { + if (Date.now() >= deadline) throw new Error("Test probe did not exit after its lifetime ended"); + await sleep(50); + } + } + } catch (error) { errors.push(error); } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, "Compose and probe cleanup failed"); +} diff --git a/test/hybrid-search.test.ts b/test/hybrid-search.test.ts index 2467c37359..044de266fd 100644 --- a/test/hybrid-search.test.ts +++ b/test/hybrid-search.test.ts @@ -112,6 +112,28 @@ describe("HybridSearch", () => { expect(results.map((r) => r.observation.id)).toEqual(["obs_valid"]); }); + it("uses query embeddings for direct, cached and expanded searches", async () => { + const obs = makeObs(); + bm25.add(obs); + await kv.set(KV.observations(obs.sessionId), obs.id, obs); + const vector = new VectorIndex(); + vector.add(obs.id, obs.sessionId, new Float32Array([1, 0])); + const embed = vi.fn(async () => new Float32Array([0, 1])); + const embedQuery = vi.fn(async (_text: string) => new Float32Array([1, 0])); + const provider = { name: "test", dimensions: 2, embed, embedQuery, embedBatch: vi.fn() }; + const hybrid = new HybridSearch(bm25, vector, provider, kv as never, 0.4, 0.6, 0); + expect(await hybrid.search("auth")).toHaveLength(1); + const request = {}; + await hybrid.search("auth", 20, "all", undefined, request); + await hybrid.search("auth", 40, "all", undefined, request); + expect(embedQuery).toHaveBeenCalledTimes(2); + await hybrid.searchWithExpansion("auth", 20, { + reformulations: ["authentication"], temporalConcretizations: [], entityExtractions: [], + }); + expect(embedQuery.mock.calls.map((args) => args[0])).toEqual(["auth", "auth", "auth", "authentication"]); + expect(embed).not.toHaveBeenCalled(); + }); + it("reuses query embeddings only within a recall request", async () => { const obs = makeObs(); bm25.add(obs); diff --git a/test/local-embedding-config.test.ts b/test/local-embedding-config.test.ts new file mode 100644 index 0000000000..3153f64218 --- /dev/null +++ b/test/local-embedding-config.test.ts @@ -0,0 +1,96 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { LocalEmbeddingProvider } from "../src/providers/embedding/local.js"; +import { withDimensionGuard } from "../src/providers/embedding/index.js"; +import { resolve } from "node:path"; +const mock = vi.hoisted(() => ({ extractor: vi.fn(), pipeline: vi.fn() })); +vi.mock("@huggingface/transformers", () => ({ pipeline: mock.pipeline })); +beforeEach(() => { + for (const key of ["MODEL", "DIMENSIONS", "POOLING", "DTYPE", "BATCH_SIZE", "QUERY_PREFIX", "DOCUMENT_PREFIX", "CACHE_DIR"]) { + vi.stubEnv(`LOCAL_EMBEDDING_${key}`, undefined); + } +}); +afterEach(() => { vi.unstubAllEnvs(); vi.clearAllMocks(); }); +function configure() { + vi.stubEnv("LOCAL_EMBEDDING_MODEL", "test/chinese"); + vi.stubEnv("LOCAL_EMBEDDING_DIMENSIONS", "3"); + vi.stubEnv("LOCAL_EMBEDDING_POOLING", "cls"); + mock.extractor.mockImplementation(async (texts: string[]) => ({ tolist: () => texts.map(() => [1, 0, 0]) })); + mock.pipeline.mockResolvedValue(mock.extractor); +} +describe("local model configuration", () => { + it("keeps the official model, dimensions and pooling by default", () => { + const p = new LocalEmbeddingProvider(); + expect([p.name, p.model, p.dimensions, p.pooling]).toEqual(["local", "Xenova/all-MiniLM-L6-v2", 384, "mean"]); + }); + it("separates query and document prefixes, including through the guard", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_QUERY_PREFIX", "query: "); vi.stubEnv("LOCAL_EMBEDDING_DOCUMENT_PREFIX", "passage: "); + const p = withDimensionGuard(new LocalEmbeddingProvider()); + await p.embedQuery!("屏幕偏黄"); await p.embedBatch(["色彩配置"]); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8" }); + expect(mock.extractor.mock.calls.map((c) => c[0])).toEqual([["query: 屏幕偏黄"], ["passage: 色彩配置"]]); + expect(p.dimensions).toBe(3); + }); + it("bounds batches and shares initialization across concurrent calls", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_BATCH_SIZE", "2"); + const p = new LocalEmbeddingProvider(); + await Promise.all([p.embedBatch(["a", "b", "c"]), p.embedQuery("q")]); + expect(mock.pipeline).toHaveBeenCalledTimes(1); + expect(mock.extractor.mock.calls.every((c) => c[0].length <= 2)).toBe(true); + }); + it("rejects invalid dimensions and pooling before loading a model", () => { + vi.stubEnv("LOCAL_EMBEDDING_DIMENSIONS", "3oops"); + expect(() => new LocalEmbeddingProvider()).toThrow("positive integer"); + vi.stubEnv("LOCAL_EMBEDDING_DIMENSIONS", "3"); vi.stubEnv("LOCAL_EMBEDDING_POOLING", "none"); + expect(() => new LocalEmbeddingProvider()).toThrow("POOLING"); + }); + it("refuses a wrong-sized or non-finite embedding", async () => { + configure(); mock.extractor.mockResolvedValue({ tolist: () => [[1, 2]] }); + await expect(new LocalEmbeddingProvider().embed("test")).rejects.toThrow("dimensions"); + mock.extractor.mockResolvedValue({ tolist: () => [[1, NaN, 0]] }); + await expect(new LocalEmbeddingProvider().embedQuery("test")).rejects.toThrow("dimensions"); + }); + it("passes the configured cache directory to the pipeline", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "model-cache"); + await new LocalEmbeddingProvider().embed("test"); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8", cache_dir: resolve("model-cache") }); + }); + it("retries loading after a pipeline initialization failure", async () => { + configure(); mock.pipeline.mockRejectedValueOnce(new Error("model unavailable")); + const provider = new LocalEmbeddingProvider(); + await expect(provider.embed("test")).rejects.toThrow("model unavailable"); + expect(await provider.embed("test")).toHaveLength(3); + expect(mock.pipeline).toHaveBeenCalledTimes(2); + }); + it("passes the selected dtype with the cache directory", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_DTYPE", "fp32"); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "model-cache"); + await new LocalEmbeddingProvider().embed("test"); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "fp32", cache_dir: resolve("model-cache") }); + }); + it("loads from the same cache location used to establish its identity", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "first-cache"); + const provider = new LocalEmbeddingProvider(); + const identity = provider.indexIdentity; + vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "second-cache"); + expect(provider.indexIdentity).toBe(identity); + await provider.embed("test"); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8", cache_dir: resolve("first-cache") }); + }); + it("rejects an unsupported dtype before loading", () => { + vi.stubEnv("LOCAL_EMBEDDING_DTYPE", "bogus"); + expect(() => new LocalEmbeddingProvider()).toThrow("DTYPE"); + }); + it("checks query dimensions through the wrapper", async () => { + const provider = withDimensionGuard({ + name: "test", dimensions: 3, + embed: async () => new Float32Array(3), + embedQuery: async () => new Float32Array(2), + embedBatch: async () => [], + }); + await expect(provider.embedQuery!("test")).rejects.toThrow("dimension mismatch"); + }); + it.each(["0", "-1", "1.5", "9007199254740992"])("rejects invalid batch size %s", (value) => { + vi.stubEnv("LOCAL_EMBEDDING_BATCH_SIZE", value); + expect(() => new LocalEmbeddingProvider()).toThrow("positive integer"); + }); + +}); diff --git a/test/local-embedding-provider.test.ts b/test/local-embedding-provider.test.ts index 69ce049d8b..849b0fbc16 100644 --- a/test/local-embedding-provider.test.ts +++ b/test/local-embedding-provider.test.ts @@ -1,6 +1,13 @@ -import { describe, it, expect, vi, afterEach } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +beforeEach(() => { + for (const key of ["MODEL", "DIMENSIONS", "POOLING", "DTYPE", "BATCH_SIZE", "QUERY_PREFIX", "DOCUMENT_PREFIX", "CACHE_DIR"]) { + vi.stubEnv(`LOCAL_EMBEDDING_${key}`, undefined); + } +}); afterEach(() => { + vi.unstubAllEnvs(); vi.doUnmock("@huggingface/transformers"); vi.resetModules(); }); @@ -21,7 +28,7 @@ describe("LocalEmbeddingProvider (package unavailable)", () => { describe("LocalEmbeddingProvider (with loaded pipeline)", () => { function mockSuccessModule() { const extractor = vi.fn(async (texts: string[]) => ({ - tolist: () => texts.map(() => [0.1, 0.2, 0.3]), + tolist: () => texts.map(() => Array(384).fill(0.1)), })); const pipeline = vi.fn(() => Promise.resolve(extractor)); vi.doMock("@huggingface/transformers", () => ({ pipeline })); @@ -46,7 +53,7 @@ describe("LocalEmbeddingProvider (with loaded pipeline)", () => { normalize: true, }); expect(vec).toBeInstanceOf(Float32Array); - expect(vec).toEqual(new Float32Array([0.1, 0.2, 0.3])); + expect(vec).toEqual(new Float32Array(Array(384).fill(0.1))); }); it("embedBatch returns one Float32Array per input text", async () => { diff --git a/test/observation-source.test.ts b/test/observation-source.test.ts index 79d2018de1..b23bbc4ba7 100644 --- a/test/observation-source.test.ts +++ b/test/observation-source.test.ts @@ -277,7 +277,8 @@ describe("retained observation source", () => { expect(await fresh.call("mem::import", { exportData: exported })).toMatchObject({ success: true, observations: 1000, sourceOmitted: 1000 }); expect(await fresh.kv.get(KV.observations(template.sessionId), first[0].id)).toEqual(first[0]); expect(payloadByteLength({ result: await fresh.kv.list(KV.observations(template.sessionId)) })).toBeLessThan(SAFE_PAYLOAD_BYTES); - }); + // Two thousand source-heavy imports can exceed the default budget on busy CI runners. + }, 30_000); it("rejects an oversized replacement before deleting or writing any stored records", async () => { const r = rig(); @@ -317,7 +318,8 @@ describe("retained observation source", () => { expect(rows).toHaveLength(1200); expect(rows.reduce((sum, row) => sum + (row.source ? payloadByteLength(row.source) + 10 : 0), 0)).toBeLessThanOrEqual(SESSION_SOURCE_MAX_BYTES); expect(payloadByteLength({ result: rows })).toBeLessThan(SAFE_PAYLOAD_BYTES); - }); + // Both source-heavy batches must complete before checking the shared budget. + }, 30_000); it("refuses a replay result that expands beyond the transport frame", async () => { const r = rig();