diff --git a/.changeset/quiet-memories-upsert.md b/.changeset/quiet-memories-upsert.md new file mode 100644 index 000000000..127611a28 --- /dev/null +++ b/.changeset/quiet-memories-upsert.md @@ -0,0 +1,5 @@ +--- +'@agentskit/memory': patch +--- + +Fix memory backend edge cases: preserve Redis vector bytes, prefer stable Vectra IDs where supported, protect adapter-owned metadata fields, keep file and localStorage fallback writes atomic, make SQLite KV eviction transactional when the opener supports transactions, preserve same-event-loop localStorage writes and expiration purges, bound vector KV TTL recall to `k` and reject foreign collection rows, aggregate erasure failures, and retry failed lazy initialization after best-effort cleanup of failed internally owned resources. Mongo Atlas uses ordered ID upserts when supported by the injected collection, replacing canonical fields while preserving unrelated host fields. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d2cdc11..0bc115f45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,6 @@ jobs: fi - name: Build all packages - if: steps.package-scope.outputs.run_all == 'true' run: pnpm exec turbo run build --filter='./packages/*' --concurrency=2 - name: Build affected packages and dependents diff --git a/packages/memory/README.md b/packages/memory/README.md index ac31db379..60922b708 100644 --- a/packages/memory/README.md +++ b/packages/memory/README.md @@ -116,6 +116,21 @@ persistence succeeds. - `milvusVectorStore` — Milvus/Zilliz REST API. - `mongoAtlasVectorStore` — MongoDB Atlas Vector Search with an injected collection. +Mongo Atlas writes use ordered `_id` upserts when the collection provides +`bulkWrite` or `updateOne`. Repeating an ID replaces the adapter-owned +`content`, embedding, and `metadata` fields while preserving unrelated fields +on the Mongo document; duplicate IDs in one batch resolve to the last document +in input order. Older insert-only shims keep their insert behavior and cannot +promise repeated-ID replacement. + +The Vectra file adapter uses stable native IDs when the installed Vectra +version provides `upsertItem`. Older versions use insert-only writes, so +legacy random-ID duplicates are not automatically removed. Adapter-owned +`_id` and `content` metadata values take precedence over caller metadata with +those names. Core `VectorSearchOptions.filter` is not yet translated by every +remote vector adapter; consult the adapter implementation before relying on a +filter to separate results. + Vector adapters follow ADR 0003: a result is included only when its score is strictly greater than `threshold`. Remote HTTP adapters accept `timeoutMs` and `maxResponseBytes`, defaulting to 15 seconds and 2 MiB; both require positive @@ -123,7 +138,20 @@ safe integers. Injected `fetch` and caller `signal` are also supported. Timeout and response-limit failures use `AK_MEMORY_REMOTE_HTTP`. File KV writes are atomic and serialized across instances in one process. They are not a multi-process coordination primitive; use SQLite, Redis, or another external -store when several processes write the same file. +store when several processes write the same file. SQLite KV writes include +eviction in a transaction when the configured SQLite opener exposes native +transactions; simpler injected openers retain their non-transactional path. +File rename publication is atomic on the same filesystem, but writes are not +fsync-backed and the in-process queue does not coordinate separate processes. +The localStorage backend uses synchronous storage read-modify-write and +expiration purges for same-event-loop calls, but does not coordinate writes +across browser tabs. + +Vector KV TTL hides expired entries from `get` and `recall` but does not +physically delete their vector rows. TTL recall uses bounded over-fetch and +returns at most `k`; it cannot guarantee that it finds every live row if more +than the over-fetch window is occupied by expired or foreign-collection +results. `maxMessages` eviction is not enforced by this vector KV adapter. Same 3-method `VectorStore` contract — swap without touching agent code. diff --git a/packages/memory/src/file-vector.ts b/packages/memory/src/file-vector.ts index dd144cbf8..b8cf5721a 100644 --- a/packages/memory/src/file-vector.ts +++ b/packages/memory/src/file-vector.ts @@ -29,6 +29,7 @@ interface VectraIndex { endUpdate(): Promise cancelUpdate(): void insertItem(item: { vector: number[]; metadata: Record }): Promise + upsertItem?(item: { id: string; vector: number[]; metadata: Record }): Promise // vectra >=0.15 inserts a BM25 `query` string between vector and topK; pass "" for pure vector search. queryItems(vector: number[], query: string, topK: number): Promise } }>> listItems(): Promise }>>; @@ -59,10 +60,12 @@ function createVectraStore(dirPath: string): VectorStore { await idx.beginUpdate() try { for (const doc of docs) { - await idx.insertItem({ - vector: doc.vector, - metadata: { _id: doc.id, ...doc.metadata }, - }) + const metadata = { ...doc.metadata, _id: doc.id } + if (idx.upsertItem) { + await idx.upsertItem({ id: doc.id, vector: doc.vector, metadata }) + } else { + await idx.insertItem({ vector: doc.vector, metadata }) + } } await idx.endUpdate() } catch (err) { @@ -113,7 +116,7 @@ export function fileVectorMemory(config: FileVectorMemoryConfig): VectorMemory { await store.upsert(docs.map(doc => ({ id: doc.id, vector: doc.embedding, - metadata: { content: doc.content, ...doc.metadata }, + metadata: { ...doc.metadata, content: doc.content }, }))) }, async search(embedding, options) { diff --git a/packages/memory/src/forget.ts b/packages/memory/src/forget.ts index d65351b7d..2bbfa3ab9 100644 --- a/packages/memory/src/forget.ts +++ b/packages/memory/src/forget.ts @@ -48,6 +48,15 @@ export interface ForgetSubjectResult { evidenceHash: string } +function failedReport(backend: string, reason: string): ForgetReport { + return { + backend, + deletedCount: 0, + at: new Date().toISOString(), + failures: [{ id: '*', reason }], + } +} + function isForgettable(value: unknown): value is ForgettableMemory { return ( !!value && @@ -86,7 +95,11 @@ export async function forgetSubject( : 'unknown') continue } - reports.push(await memory.forgetSubject(subjectId)) + try { + reports.push(await memory.forgetSubject(subjectId)) + } catch (err) { + reports.push(failedReport(memory.__agentskitBackend, err instanceof Error ? err.message : String(err))) + } } const totalDeleted = reports.reduce((sum, r) => sum + r.deletedCount, 0) const evidenceHash = await hash( @@ -120,8 +133,14 @@ export function makeForgettable( return Object.assign(memory, { __agentskitBackend: options.backend, forgetSubject: async (subjectId: string): Promise => { - const ids = await options.listIds(subjectId) const failures: Array<{ id: string; reason: string }> = [] + let ids: string[] + try { + ids = await options.listIds(subjectId) + } catch (err) { + const reason = err instanceof Error ? err.message : String(err) + return failedReport(options.backend, reason) + } try { await options.deleteIds(ids) } catch (err) { diff --git a/packages/memory/src/kv-store-basic.ts b/packages/memory/src/kv-store-basic.ts index c4c82f082..79163cdfc 100644 --- a/packages/memory/src/kv-store-basic.ts +++ b/packages/memory/src/kv-store-basic.ts @@ -27,6 +27,32 @@ const enqueueFileWrite = (path: string, task: () => Promise): Promise => { + await mkdir(dirname(path), { recursive: true }) + const tempPath = join(dirname(path), `.${basename(path)}.${randomBytes(8).toString('hex')}.tmp`) + try { + await writeFile(tempPath, contents, { encoding: 'utf8', mode: 0o600 }) + await rename(tempPath, path) + } finally { + await unlink(tempPath).catch(() => {}) + } +} + +const removeExpiredFileEntry = ( + path: string, + key: string, + ttlSeconds: number | undefined, + load: () => Promise>, + persist: (map: Map) => Promise, +): Promise => enqueueFileWrite(path, async () => { + const map = await load() + const entry = map.get(key) + if (entry && isExpired(entry, ttlSeconds, Date.now())) { + map.delete(key) + await persist(map) + } +}) + /** Creates an in-memory key-value store with optional TTL and size limits. * @param config Backend and retention settings. * @returns A key-value store backed by a process-local map. @@ -35,20 +61,19 @@ const enqueueFileWrite = (path: string, task: () => Promise): Promise { validateKvRetention(config) const store = new Map() - const now = () => Date.now() return { id: 'in-memory', async get(key) { const entry = store.get(key) if (!entry) return undefined - if (isExpired(entry, config.ttlSeconds, now())) { + if (isExpired(entry, config.ttlSeconds, Date.now())) { store.delete(key) return undefined } return entry.value }, async set(key, value) { - store.set(key, { value, insertedAt: now() }) + store.set(key, { value, insertedAt: Date.now() }) enforceMaxMessages(store, config.maxMessages) }, } @@ -72,14 +97,7 @@ export const createFileStore = (config: FileKvConfig): AgentskitMemoryStore => { } const persist = async (map: Map): Promise => { - await mkdir(dirname(path), { recursive: true }) - const tempPath = join(dirname(path), `.${basename(path)}.${randomBytes(8).toString('hex')}.tmp`) - try { - await writeFile(tempPath, JSON.stringify(Object.fromEntries(map), null, 2), { encoding: 'utf8', mode: 0o600 }) - await rename(tempPath, path) - } finally { - try { await unlink(tempPath) } catch { /* rename already published it */ } - } + await writeFileAtomically(path, JSON.stringify(Object.fromEntries(map), null, 2)) } return { @@ -89,14 +107,7 @@ export const createFileStore = (config: FileKvConfig): AgentskitMemoryStore => { const entry = map.get(key) if (!entry) return undefined if (isExpired(entry, config.ttlSeconds, Date.now())) { - await enqueueFileWrite(path, async () => { - const latest = await load() - const current = latest.get(key) - if (current && isExpired(current, config.ttlSeconds, Date.now())) { - latest.delete(key) - await persist(latest) - } - }) + await removeExpiredFileEntry(path, key, config.ttlSeconds, load, persist) return undefined } return entry.value @@ -151,38 +162,27 @@ export const createLocalStorageStore = ({ } } - const load = async (): Promise> => - storage ? mapFromJson(storage.getItem(key)) : loadFromFile() - + const loadSync = (): Map => mapFromJson(storage!.getItem(key)) const persist = async (map: Map): Promise => { - const raw = JSON.stringify(Object.fromEntries(map), null, 2) - if (storage) { - storage.setItem(key, raw) - return - } - await mkdir(dirname(filePath), { recursive: true }) - await writeFile(filePath, raw, { encoding: 'utf8', mode: 0o600 }) + await writeFileAtomically(filePath, JSON.stringify(Object.fromEntries(map), null, 2)) + } + + const persistSync = (map: Map): void => { + storage!.setItem(key, JSON.stringify(Object.fromEntries(map), null, 2)) } return { id: storage ? `localstorage:${key}` : `localstorage-file:${filePath}:${key}`, async get(itemKey) { - const map = await load() + const map = storage ? loadSync() : await loadFromFile() const entry = map.get(itemKey) if (!entry) return undefined if (isExpired(entry, config.ttlSeconds, Date.now())) { if (storage) { map.delete(itemKey) - await persist(map) + persistSync(map) } else { - await enqueueFileWrite(filePath, async () => { - const latest = await loadFromFile() - const current = latest.get(itemKey) - if (current && isExpired(current, config.ttlSeconds, Date.now())) { - latest.delete(itemKey) - await persist(latest) - } - }) + await removeExpiredFileEntry(filePath, itemKey, config.ttlSeconds, loadFromFile, persist) } return undefined } @@ -190,10 +190,10 @@ export const createLocalStorageStore = ({ }, async set(itemKey, value) { if (storage) { - const map = await load() + const map = loadSync() map.set(itemKey, { value, insertedAt: Date.now() }) enforceMaxMessages(map, config.maxMessages) - await persist(map) + persistSync(map) return } await enqueueFileWrite(filePath, async () => { diff --git a/packages/memory/src/kv-store-sqlite.ts b/packages/memory/src/kv-store-sqlite.ts index 19ec4eef7..f7d0d479c 100644 --- a/packages/memory/src/kv-store-sqlite.ts +++ b/packages/memory/src/kv-store-sqlite.ts @@ -51,6 +51,14 @@ export const createSqliteStore = ({ config, open }: CreateSqliteStoreOpts): Agen } } + const write = db.transaction ? db.transaction((key: string, value: string, now: number) => { + setStmt.run(key, value, now) + enforce() + }) : (key: string, value: string, now: number): void => { + setStmt.run(key, value, now) + enforce() + } + return { id: `sqlite:${config.path}`, async get(key) { @@ -63,8 +71,7 @@ export const createSqliteStore = ({ config, open }: CreateSqliteStoreOpts): Agen return JSON.parse(row.value) as unknown }, async set(key, value) { - setStmt.run(key, JSON.stringify(value), Date.now()) - enforce() + write(key, JSON.stringify(value), Date.now()) }, } } diff --git a/packages/memory/src/kv-store-types.ts b/packages/memory/src/kv-store-types.ts index b3e3fb8df..e5b9ddc49 100644 --- a/packages/memory/src/kv-store-types.ts +++ b/packages/memory/src/kv-store-types.ts @@ -20,8 +20,7 @@ export interface KvEntry { } export const isExpired = (entry: KvEntry, ttlSeconds: number | undefined, now: number): boolean => { - if (ttlSeconds === undefined) return false - return now - entry.insertedAt > ttlSeconds * 1000 + return ttlSeconds !== undefined && now - entry.insertedAt > ttlSeconds * 1000 } export const enforceMaxMessages = (map: Map, maxMessages: number | undefined): void => { @@ -115,6 +114,7 @@ export interface SqliteStmt { export interface SqliteLike { exec(sql: string): void prepare(sql: string): SqliteStmt + transaction? unknown>(fn: T): T } /** Opens a SQLite database at the given path. */ diff --git a/packages/memory/src/kv-store-vector.ts b/packages/memory/src/kv-store-vector.ts index 6ea9666de..cc8257e86 100644 --- a/packages/memory/src/kv-store-vector.ts +++ b/packages/memory/src/kv-store-vector.ts @@ -30,6 +30,11 @@ export const createVectorStore = ({ }: CreateVectorStoreOpts): AgentskitMemoryStore & { recall(query: string, k?: number): Promise } => { validateKvRetention(config) const collection = config.collection + const isLive = (metadata: Record, now: number) => { + const insertedAt = metadata['__insertedAt'] + return metadata['__collection'] === collection && + (typeof insertedAt !== 'number' || !isExpired({ value: undefined, insertedAt }, config.ttlSeconds, now)) + } const embedOne = async (text: string): Promise => { const [vec] = await embedder.embed([text]) @@ -46,15 +51,12 @@ export const createVectorStore = ({ id: `vector:${config.provider}:${collection}`, async get(key) { const vec = await embedOne(key) - const hits = await vectorStore.query(vec, 1, { __collection: collection, __key: key }) - const hit = hits[0] - if (hit === undefined) return undefined - if (hit.metadata['__key'] !== key) return undefined - const insertedAt = hit.metadata['__insertedAt'] - if (typeof insertedAt === 'number' && isExpired({ value: undefined, insertedAt }, config.ttlSeconds, Date.now())) { - return undefined - } - return hit.metadata['__value'] + const hits = await vectorStore.query(vec, config.ttlSeconds === undefined ? 1 : 100, { __collection: collection, __key: key }) + const now = Date.now() + const hit = hits.find(candidate => { + return candidate.metadata['__key'] === key && isLive(candidate.metadata, now) + }) + return hit?.metadata['__value'] }, async set(key, value) { const vec = await embedOne(key) @@ -67,16 +69,15 @@ export const createVectorStore = ({ ]) }, async recall(query, k = 5) { + if (k <= 0) return [] const vec = await embedOne(query) - const hits = await vectorStore.query(vec, k, { __collection: collection }) + const hits = await vectorStore.query(vec, k + (config.ttlSeconds === undefined ? 0 : 100), { __collection: collection }) const now = Date.now() const results: unknown[] = [] for (const hit of hits) { - const insertedAt = hit.metadata['__insertedAt'] - if (typeof insertedAt === 'number' && isExpired({ value: undefined, insertedAt }, config.ttlSeconds, now)) { - continue - } + if (!isLive(hit.metadata, now)) continue results.push(hit.metadata['__value']) + if (results.length >= k) break } return results }, diff --git a/packages/memory/src/redis-chat.ts b/packages/memory/src/redis-chat.ts index 2359ee915..14318ca0b 100644 --- a/packages/memory/src/redis-chat.ts +++ b/packages/memory/src/redis-chat.ts @@ -37,7 +37,12 @@ export function redisChatMemory(config: RedisChatMemoryConfig): ChatMemory { const getClient = (): Promise => { if (config.client) return Promise.resolve(config.client) - if (!clientPromise) clientPromise = createRedisClientAdapter(config.url) + if (!clientPromise) { + clientPromise = createRedisClientAdapter(config.url).catch(error => { + clientPromise = null + throw error + }) + } return clientPromise } diff --git a/packages/memory/src/redis-client.ts b/packages/memory/src/redis-client.ts index 1a4f24003..da23333c3 100644 --- a/packages/memory/src/redis-client.ts +++ b/packages/memory/src/redis-client.ts @@ -39,11 +39,16 @@ export async function createRedisClientAdapter(url: string): Promise 0) await client.del(keys) }, async keys(pattern) { - return await client.keys(pattern) + return client.keys(pattern) }, async disconnect() { // node-redis v6 renamed the graceful close: client.disconnect() (v5) -> client.close(). await client.close() }, async call(command, ...args) { - return await client.sendCommand([command, ...args.map(String)]) + return client.sendCommand([command, ...args.map(arg => + Buffer.isBuffer(arg) ? arg : String(arg), + )]) }, } } diff --git a/packages/memory/src/redis-vector.ts b/packages/memory/src/redis-vector.ts index abf31b40c..1369aaad9 100644 --- a/packages/memory/src/redis-vector.ts +++ b/packages/memory/src/redis-vector.ts @@ -31,7 +31,12 @@ export function redisVectorMemory(config: RedisVectorMemoryConfig): VectorMemory const getClient = (): Promise => { if (config.client) return Promise.resolve(config.client) - if (!clientPromise) clientPromise = createRedisClientAdapter(config.url) + if (!clientPromise) { + clientPromise = createRedisClientAdapter(config.url).catch(error => { + clientPromise = null + throw error + }) + } return clientPromise } diff --git a/packages/memory/src/sqlite.ts b/packages/memory/src/sqlite.ts index b3dce0fc3..0e9ec7dc9 100644 --- a/packages/memory/src/sqlite.ts +++ b/packages/memory/src/sqlite.ts @@ -31,6 +31,7 @@ interface SqliteDb { run(...args: unknown[]): void get(...args: unknown[]): Record | undefined } + close?(): void } async function openDatabase(path: string): Promise { @@ -59,13 +60,21 @@ export function sqliteChatMemory(config: SqliteChatMemoryConfig): ChatMemory { const getDb = (): Promise => { if (!dbPromise) { dbPromise = openDatabase(config.path).then(db => { - db.prepare(` - CREATE TABLE IF NOT EXISTS conversations ( - id TEXT PRIMARY KEY, - messages TEXT NOT NULL - ) - `).run() - return db + try { + db.prepare(` + CREATE TABLE IF NOT EXISTS conversations ( + id TEXT PRIMARY KEY, + messages TEXT NOT NULL + ) + `).run() + return db + } catch (error) { + try { db.close?.() } catch { /* preserve the initialization error */ } + throw error + } + }).catch(error => { + dbPromise = null + throw error }) } return dbPromise diff --git a/packages/memory/src/turso.ts b/packages/memory/src/turso.ts index c7aa39931..4a3b6c6a5 100644 --- a/packages/memory/src/turso.ts +++ b/packages/memory/src/turso.ts @@ -22,6 +22,7 @@ export interface TursoChatMemoryConfig { interface LibsqlClient { execute(args: { sql: string; args?: unknown[] }): Promise<{ rows: Array> }> + close?(): void | Promise } interface LibsqlModule { @@ -42,7 +43,10 @@ async function loadSdk(): Promise { hint: 'tursoChatMemory uses the optional peer "@libsql/client".', }) } - })() + })().catch(error => { + cachedSdk = null + throw error + }) } return cachedSdk } @@ -72,11 +76,19 @@ export function tursoChatMemory(config: TursoChatMemoryConfig): ChatMemory { clientPromise = (async () => { const sdk = await loadSdk() const client = sdk.createClient({ url: config.url, authToken: config.authToken }) - await client.execute({ - sql: 'CREATE TABLE IF NOT EXISTS conversations (id TEXT PRIMARY KEY, messages TEXT NOT NULL)', - }) - return client - })() + try { + await client.execute({ + sql: 'CREATE TABLE IF NOT EXISTS conversations (id TEXT PRIMARY KEY, messages TEXT NOT NULL)', + }) + return client + } catch (error) { + try { await client.close?.() } catch { /* preserve the initialization error */ } + throw error + } + })().catch(error => { + clientPromise = null + throw error + }) } return clientPromise } diff --git a/packages/memory/src/vector/mongo-atlas.ts b/packages/memory/src/vector/mongo-atlas.ts index 036aad6dd..ae0316698 100644 --- a/packages/memory/src/vector/mongo-atlas.ts +++ b/packages/memory/src/vector/mongo-atlas.ts @@ -10,6 +10,8 @@ import type { RetrievedDocument, VectorDocument, VectorMemory } from '@agentskit export interface MongoCollectionLike { insertMany(docs: Array>, options?: unknown): Promise + updateOne?(filter: Record, update: Record, options?: unknown): Promise + bulkWrite?(operations: Array>, options?: unknown): Promise deleteMany(filter: Record): Promise aggregate>(pipeline: Array>): { toArray(): Promise @@ -35,16 +37,38 @@ export interface MongoAtlasVectorConfig { export function mongoAtlasVectorStore(config: MongoAtlasVectorConfig): VectorMemory { const defaultTopK = Math.max(1, config.topK ?? 10) const vectorField = config.vectorField ?? 'embedding' + const fields = (doc: VectorDocument) => ({ + content: doc.content, + [vectorField]: doc.embedding, + metadata: doc.metadata ?? {}, + }) return { async store(docs: VectorDocument[]) { if (docs.length === 0) return - await config.collection.insertMany(docs.map(d => ({ - _id: d.id, - content: d.content, - [vectorField]: d.embedding, - metadata: d.metadata ?? {}, - }))) + if (config.collection.bulkWrite) { + await config.collection.bulkWrite(docs.map(d => ({ + updateOne: { + filter: { _id: d.id }, + update: { + $set: fields(d), + }, + upsert: true, + }, + })), { ordered: true }) + } else { + if (config.collection.updateOne) { + for (const d of docs) { + await config.collection.updateOne({ _id: d.id }, { $set: fields(d) }, { upsert: true }) + } + } else { + // Compatibility for older injected collection shims without updateOne. + await config.collection.insertMany(docs.map(d => ({ + _id: d.id, + ...fields(d), + }))) + } + } }, async search(embedding: number[], options = {}): Promise { diff --git a/packages/memory/tests/file-vector.test.ts b/packages/memory/tests/file-vector.test.ts index 135032b79..843ef2639 100644 --- a/packages/memory/tests/file-vector.test.ts +++ b/packages/memory/tests/file-vector.test.ts @@ -20,6 +20,20 @@ const doc2: VectorDocument = { } describe('fileVectorMemory with custom VectorStore', () => { + it('keeps canonical id/content when metadata collides with reserved fields', async () => { + let stored: VectorStoreDocument[] = [] + const customStore: VectorStore = { + async upsert(docs) { stored = docs }, + async query() { return stored.map(d => ({ id: d.id, score: 1, metadata: d.metadata })) }, + async delete(ids) { stored = stored.filter(d => !ids.includes(d.id)) }, + } + const mem = fileVectorMemory({ path: join(tmpdir(), 'unused'), store: customStore }) + await mem.store([{ ...doc1, metadata: { _id: 'spoof', content: 'spoofed' } }]) + const result = await mem.search(doc1.embedding) + expect(result[0]).toMatchObject({ id: doc1.id, content: doc1.content }) + await mem.delete!([doc1.id]) + expect(stored).toHaveLength(0) + }) it('delegates to custom store', async () => { const stored: VectorStoreDocument[] = [] const customStore: VectorStore = { @@ -33,7 +47,7 @@ describe('fileVectorMemory with custom VectorStore', () => { }, } - const mem = fileVectorMemory({ path: '/tmp/unused', store: customStore }) + const mem = fileVectorMemory({ path: join(tmpdir(), 'unused'), store: customStore }) await mem.store([doc1]) expect(stored).toHaveLength(1) @@ -62,7 +76,7 @@ describe('fileVectorMemory with custom VectorStore', () => { async delete() {}, } - const mem = fileVectorMemory({ path: '/tmp/unused', store: customStore }) + const mem = fileVectorMemory({ path: join(tmpdir(), 'unused'), store: customStore }) const results = await mem.search([0.1], { topK: 2 }) expect(results).toHaveLength(2) }) @@ -79,7 +93,7 @@ describe('fileVectorMemory with custom VectorStore', () => { async delete() {}, } - const mem = fileVectorMemory({ path: '/tmp/unused', store: customStore }) + const mem = fileVectorMemory({ path: join(tmpdir(), 'unused'), store: customStore }) const results = await mem.search([0.1], { threshold: 0.5 }) expect(results).toHaveLength(1) expect(results[0].id).toBe('a') @@ -122,4 +136,29 @@ describe('fileVectorMemory with vectra', () => { const ids = results.map(r => r.id) expect(ids).not.toContain('doc-1') }, VECTRA_TIMEOUT) + + it('upserts one persistent item per public id', async () => { + const first = fileVectorMemory({ path: dirPath }) + await first.store([{ ...doc1, content: 'old' }]) + await first.store([{ ...doc1, content: 'new' }]) + const reopened = fileVectorMemory({ path: dirPath }) + const results = await reopened.search(doc1.embedding, { topK: 10 }) + expect(results.filter(result => result.id === doc1.id)).toHaveLength(1) + expect(results.find(result => result.id === doc1.id)?.content).toBe('new') + }, VECTRA_TIMEOUT) + + it('keeps legacy random-ID duplicates while adding the stable-ID row', async () => { + const { LocalIndex } = await import('vectra') + const index = new LocalIndex(dirPath) + await index.createIndex() + await index.insertItem({ vector: doc1.embedding, metadata: { _id: doc1.id, content: 'legacy one' } }) + await index.insertItem({ vector: doc1.embedding, metadata: { _id: doc1.id, content: 'legacy two' } }) + + const memory = fileVectorMemory({ path: dirPath }) + await memory.store([{ ...doc1, content: 'canonical' }]) + const reopened = fileVectorMemory({ path: dirPath }) + const duplicates = (await reopened.search(doc1.embedding, { topK: 10 })).filter(result => result.id === doc1.id) + expect(duplicates).toHaveLength(3) + expect(duplicates.map(result => result.content)).toContain('canonical') + }, VECTRA_TIMEOUT) }) diff --git a/packages/memory/tests/forget.test.ts b/packages/memory/tests/forget.test.ts index 2f571e303..322e28c3e 100644 --- a/packages/memory/tests/forget.test.ts +++ b/packages/memory/tests/forget.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' import { forgetSubject, makeForgettable } from '../src/forget' +import { fileVectorMemory } from '../src/file-vector' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' describe('forgetSubject', () => { it('runs forgetSubject on every forgettable backend', async () => { @@ -52,4 +56,66 @@ describe('forgetSubject', () => { expect(result.incomplete).toBe(true) expect(result.skippedBackends).toEqual(['unknown']) }) + + it('continues after one backend rejects and reports incomplete progress', async () => { + let healthyCalled = false + const failing = { __agentskitBackend: 'failed', forgetSubject: async () => { throw new Error('offline') } } + const healthy = makeForgettable({}, { + backend: 'healthy', listIds: async () => ['id-1'], deleteIds: async () => { healthyCalled = true }, + }) + const result = await forgetSubject([failing, healthy], 'subject') + expect(healthyCalled).toBe(true) + expect(result.incomplete).toBe(true) + expect(result.reports).toHaveLength(2) + expect(result.reports[0]?.failures?.[0]).toMatchObject({ id: '*', reason: 'offline' }) + expect(result.totalDeleted).toBe(1) + }) + + it('reports ID enumeration failure without claiming deletions', async () => { + const memory = makeForgettable({}, { + backend: 'enumeration', listIds: async () => { throw new Error('list failed') }, deleteIds: async () => {}, + }) + const result = await forgetSubject([memory], 'subject') + expect(result.incomplete).toBe(true) + expect(result.totalDeleted).toBe(0) + expect(result.reports[0]?.failures?.[0]).toMatchObject({ id: '*', reason: 'list failed' }) + }) + + it('continues after a failing backend and physically erases indexed subject rows from local Vectra', async () => { + const directory = await mkdtemp(join(tmpdir(), 'agentskit-forget-vectra-')) + try { + const memory = fileVectorMemory({ path: directory }) + await memory.store([ + { id: 'subject-a-1', content: 'A one', embedding: [1, 0, 0], metadata: { subjectId: 'A' } }, + { id: 'subject-a-2', content: 'A two', embedding: [0, 1, 0], metadata: { subjectId: 'A' } }, + { id: 'subject-b-1', content: 'B one', embedding: [0, 0, 1], metadata: { subjectId: 'B' } }, + ]) + const failing = { __agentskitBackend: 'first', forgetSubject: async () => { throw new Error('offline') } } + const { LocalIndex } = await import('vectra') + const index = new LocalIndex(directory) + const forgettable = makeForgettable(memory, { + backend: 'vectra-local', + listIds: async subjectId => (await index.listItems()) + .filter(item => item.metadata.subjectId === subjectId) + .map(item => String(item.metadata._id ?? item.id)), + deleteIds: async ids => { await memory.delete!(ids) }, + }) + + const result = await forgetSubject([failing, forgettable], 'A') + expect(result.incomplete).toBe(true) + expect(result.totalDeleted).toBe(2) + expect(result.reports[0]?.failures?.[0]).toMatchObject({ id: '*', reason: 'offline' }) + + const reopened = fileVectorMemory({ path: directory }) + const nativeIndex = new LocalIndex(directory) + const persisted = await nativeIndex.listItems() + const ids = persisted.map(item => String(item.metadata._id ?? item.id)) + expect(ids).not.toContain('subject-a-1') + expect(ids).not.toContain('subject-a-2') + expect(ids).toContain('subject-b-1') + expect((await reopened.search([0, 0, 1], { topK: 50 })).map(row => row.id)).toEqual(['subject-b-1']) + } finally { + await rm(directory, { recursive: true, force: true }) + } + }, 30_000) }) diff --git a/packages/memory/tests/kv-store-basic.test.ts b/packages/memory/tests/kv-store-basic.test.ts index 799c122f3..cee0dc644 100644 --- a/packages/memory/tests/kv-store-basic.test.ts +++ b/packages/memory/tests/kv-store-basic.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { rm } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { @@ -45,7 +45,43 @@ describe('createInMemoryStore', () => { describe('createFileStore', () => { const paths: string[] = [] afterEach(async () => { - await Promise.all(paths.splice(0).map((p) => rm(p, { force: true }))) + await Promise.all(paths.splice(0).map((p) => rm(p, { force: true, recursive: true }))) + }) + + it('does not replace a corrupt regular file or leave a temp file on failed writes', async () => { + const directory = await mkdtemp(join(tmpdir(), 'ak-kv-')) + const path = join(directory, 'memory.json') + paths.push(directory) + const original = '{"broken":' + await writeFile(path, original, 'utf8') + const store = createFileStore({ backend: 'file', path }) + await expect(store.get('broken')).rejects.toThrow() + await expect(store.set('new', 'value')).rejects.toThrow() + expect(await readFile(path, 'utf8')).toBe(original) + expect(await readdir(directory)).toEqual(['memory.json']) + }) + + it('preserves valid sibling entries when atomic file replacement fails', async () => { + const directory = await mkdtemp(join(tmpdir(), 'ak-kv-')) + const path = join(directory, 'memory.json') + paths.push(directory) + const original = JSON.stringify({ keep: { value: 'unrelated', insertedAt: 1 } }) + await writeFile(path, original, 'utf8') + vi.resetModules() + vi.doMock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { ...actual, rename: vi.fn(async () => { throw new Error('injected rename failure') }) } + }) + try { + const { createFileStore: create } = await import('../src/kv-store-basic') + const store = create({ backend: 'file', path }) + await expect(store.set('new', 'value')).rejects.toThrow('injected rename failure') + expect(await readFile(path, 'utf8')).toBe(original) + expect(await readdir(directory)).toEqual(['memory.json']) + } finally { + vi.doUnmock('node:fs/promises') + vi.resetModules() + } }) it('persists across instances', async () => { @@ -69,6 +105,21 @@ describe('createFileStore', () => { expect(await b.get('b')).toBe(2) }) + it('keeps an existing destination intact and removes the temp file when atomic rename fails', async () => { + const directory = await mkdtemp(join(tmpdir(), 'ak-kv-rename-')) + const path = join(directory, 'memory.json') + paths.push(directory) + await mkdir(path, { recursive: true }) + await writeFile(join(path, 'keep.txt'), 'existing destination', 'utf8') + + const store = createFileStore({ backend: 'file', path }) + await expect(store.set('new', 'value')).rejects.toThrow() + + expect(await readFile(join(path, 'keep.txt'), 'utf8')).toBe('existing destination') + expect(await readdir(directory)).toEqual(['memory.json']) + expect(await readdir(path)).toEqual(['keep.txt']) + }) + it('rejects invalid retention settings', () => { expect(() => createInMemoryStore({ backend: 'in-memory', maxMessages: 0 })).toThrow(/maxMessages/) expect(() => createInMemoryStore({ backend: 'in-memory', ttlSeconds: Number.NaN })).toThrow(/ttlSeconds/) @@ -102,7 +153,7 @@ describe('createFileStore', () => { describe('createLocalStorageStore', () => { const paths: string[] = [] afterEach(async () => { - await Promise.all(paths.splice(0).map((p) => rm(p, { force: true }))) + await Promise.all(paths.splice(0).map((p) => rm(p, { force: true, recursive: true }))) }) it('persists through an injected storage', async () => { @@ -152,4 +203,42 @@ describe('createLocalStorageStore', () => { vi.useRealTimers() } }) + + it('preserves distinct concurrent writes to synchronous storage', async () => { + const backing = new Map() + const storage: LocalStorageLike = { + getItem: k => backing.get(k) ?? null, + setItem: (k, v) => { backing.set(k, v) }, + } + const a = createLocalStorageStore({ config: { backend: 'localstorage', key: 'shared' }, storage }) + const b = createLocalStorageStore({ config: { backend: 'localstorage', key: 'shared' }, storage }) + await Promise.all(Array.from({ length: 100 }, (_, i) => (i % 2 ? a : b).set(`k${i}`, i))) + await Promise.all(Array.from({ length: 100 }, async (_, i) => { + expect(await a.get(`k${i}`)).toBe(i) + })) + }) + + it('does not lose a new localStorage write when another instance purges an expired key', async () => { + vi.useFakeTimers() + try { + const backing = new Map() + const storage: LocalStorageLike = { + getItem: k => backing.get(k) ?? null, + setItem: (k, v) => { backing.set(k, v) }, + } + const a = createLocalStorageStore({ config: { backend: 'localstorage', key: 'shared-expiry', ttlSeconds: 1 }, storage }) + const b = createLocalStorageStore({ config: { backend: 'localstorage', key: 'shared-expiry', ttlSeconds: 1 }, storage }) + await a.set('expired', 'old') + vi.advanceTimersByTime(2000) + + const expiredRead = a.get('expired') + const concurrentWrite = b.set('new', 'live') + await Promise.all([expiredRead, concurrentWrite]) + + expect(await a.get('expired')).toBeUndefined() + expect(await a.get('new')).toBe('live') + } finally { + vi.useRealTimers() + } + }) }) diff --git a/packages/memory/tests/kv-store-localstorage-file.test.ts b/packages/memory/tests/kv-store-localstorage-file.test.ts new file mode 100644 index 000000000..0f610dd87 --- /dev/null +++ b/packages/memory/tests/kv-store-localstorage-file.test.ts @@ -0,0 +1,73 @@ +import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' + +afterEach(() => { vi.unstubAllGlobals() }) + +describe('localStorage file fallback atomic persistence', () => { + it('fails closed on corrupt JSON without replacing bytes or leaving a temp file', async () => { + const directory = await mkdtemp(join(tmpdir(), 'agentskit-localstorage-corrupt-')) + const filePath = join(directory, 'memory.json') + const original = '{"broken":' + await writeFile(filePath, original, 'utf8') + try { + const { createLocalStorageStore } = await import('../src/kv-store-basic') + const store = createLocalStorageStore({ + config: { backend: 'localstorage', key: 'legacy' }, storage: undefined, filePath, + }) + await expect(store.get('broken')).rejects.toThrow() + await expect(store.set('new', 'value')).rejects.toThrow() + expect(await readFile(filePath, 'utf8')).toBe(original) + expect(await readdir(directory)).toEqual(['memory.json']) + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) + + it('fails closed on corrupt native localStorage JSON without replacing bytes', async () => { + const original = '{"broken":' + const values = new Map([['legacy', original]]) + const storage = { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => { values.set(key, value) }, + } + const { createLocalStorageStore } = await import('../src/kv-store-basic') + const store = createLocalStorageStore({ config: { backend: 'localstorage', key: 'legacy' }, storage }) + await expect(store.get('broken')).rejects.toThrow() + await expect(store.set('new', 'value')).rejects.toThrow() + expect(values.get('legacy')).toBe(original) + }) + + it('preserves original bytes and removes its temp file when rename fails', async () => { + const directory = await mkdtemp(join(tmpdir(), 'agentskit-localstorage-fallback-')) + const filePath = join(directory, 'memory.json') + const original = '{"legacy":{"value":"keep","insertedAt":1}}' + await writeFile(filePath, original, 'utf8') + vi.stubGlobal('localStorage', undefined) + vi.resetModules() + vi.doMock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + rename: vi.fn(async () => { throw new Error('injected fallback rename failure') }), + } + }) + + try { + const { createLocalStorageStore } = await import('../src/kv-store-basic') + const store = createLocalStorageStore({ + config: { backend: 'localstorage', key: 'legacy' }, + storage: undefined, + filePath, + }) + await expect(store.set('new', 'value')).rejects.toThrow('injected fallback rename failure') + expect(await readFile(filePath, 'utf8')).toBe(original) + expect(await readdir(directory)).toEqual(['memory.json']) + } finally { + vi.doUnmock('node:fs/promises') + vi.resetModules() + await rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/memory/tests/kv-store-sqlite.test.ts b/packages/memory/tests/kv-store-sqlite.test.ts index fd2d7c3ce..e9bd4a104 100644 --- a/packages/memory/tests/kv-store-sqlite.test.ts +++ b/packages/memory/tests/kv-store-sqlite.test.ts @@ -1,5 +1,9 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type Database from 'better-sqlite3' import { describe, expect, it } from 'vitest' -import { createSqliteStore, type SqliteLike, type SqliteStmt } from '../src/index' +import { createSqliteStore, tryDefaultSqliteOpener, type SqliteLike, type SqliteStmt } from '../src/index' // Minimal in-memory fake of the better-sqlite3 surface the store uses. const fakeSqlite = (): SqliteLike => { @@ -38,4 +42,35 @@ describe('createSqliteStore', () => { expect(await s.get('a')).toBeUndefined() expect(await s.get('b')).toBe(2) }) + + it('rolls back eviction failure through the real lazy better-sqlite3 opener and native transaction', async () => { + const directory = await mkdtemp(join(tmpdir(), 'agentskit-sqlite-atomic-')) + const path = join(directory, 'memory.sqlite') + let db: Database.Database | undefined + try { + const nativeOpen = await tryDefaultSqliteOpener() + expect(nativeOpen).toBeDefined() + const open = (databasePath: string): SqliteLike => { + db = nativeOpen!(databasePath) as Database.Database + return db + } + expect(typeof open(path).transaction).toBe('function') + db!.close() + db = undefined + const store = createSqliteStore({ config: { backend: 'sqlite', path, maxMessages: 1 }, open }) + await store.set('a', 1) + db!.exec(`CREATE TRIGGER reject_memory_eviction BEFORE DELETE ON memory + BEGIN SELECT RAISE(ABORT, 'injected eviction failure'); END;`) + + await expect(store.set('b', 2)).rejects.toThrow('injected eviction failure') + expect(db!.prepare('SELECT key, value FROM memory ORDER BY key').all()).toEqual([ + { key: 'a', value: '1' }, + ]) + expect(await store.get('a')).toBe(1) + expect(await store.get('b')).toBeUndefined() + } finally { + db?.close() + await rm(directory, { recursive: true, force: true }) + } + }) }) diff --git a/packages/memory/tests/kv-store-vector.test.ts b/packages/memory/tests/kv-store-vector.test.ts index d3dc6b5db..bd7df0041 100644 --- a/packages/memory/tests/kv-store-vector.test.ts +++ b/packages/memory/tests/kv-store-vector.test.ts @@ -22,6 +22,20 @@ const fakeVector = (): MemoryVectorStoreLike => { } } +const fakeVectorWithRows = (rows: { chunkId: string; metadata: Record }[]): MemoryVectorStoreLike => ({ + upsert: async input => { + for (const row of input) { + const i = rows.findIndex(existing => existing.chunkId === row.chunkId) + const next = { chunkId: row.chunkId, metadata: row.metadata } + if (i < 0) rows.push(next) + else rows[i] = next + } + }, + query: async (_vec, k, filter) => rows.filter(row => + Object.entries(filter ?? {}).every(([field, value]) => row.metadata[field] === value), + ).slice(0, k).map(row => ({ chunkId: row.chunkId, score: 1, metadata: row.metadata })), +}) + const embedder: MemoryEmbedderLike = { embed: async (texts) => texts.map(() => [0.1, 0.2, 0.3]) } describe('createVectorStore', () => { @@ -41,4 +55,63 @@ describe('createVectorStore', () => { const out = await s.recall('anything', 5) expect(out).toEqual(expect.arrayContaining([1, 2])) }) + + it('get skips expired nearest duplicates to find a live exact key', async () => { + const rows = [ + { chunkId: 'c:topic', metadata: { __collection: 'c', __key: 'topic', __value: 'expired', __insertedAt: 1 } }, + { chunkId: 'legacy:topic', metadata: { __collection: 'c', __key: 'topic', __value: 'live', __insertedAt: Date.now() } }, + ] + const store = createVectorStore({ + config: { ...config, ttlSeconds: 1 }, vectorStore: fakeVectorWithRows(rows), embedder, + }) + expect(await store.get('topic')).toBe('live') + }) + + it('recall over-fetches a bounded amount to skip expired hits', async () => { + const rows = Array.from({ length: 5 }, (_, i) => ({ + chunkId: `c:${i}`, + metadata: { __collection: 'c', __key: `${i}`, __value: i, __insertedAt: i < 3 ? 1 : Date.now() }, + })) + const vector = fakeVectorWithRows(rows) + let queried = 0 + const wrapped: MemoryVectorStoreLike = { + ...vector, + query: async (vec, k, filter) => { queried = k; return vector.query(vec, k, filter) }, + } + const store = createVectorStore({ + config: { ...config, ttlSeconds: 1 }, vectorStore: wrapped, embedder, + }) + expect(await store.recall('anything', 2)).toEqual([3, 4]) + expect(queried).toBe(102) + }) + + it('rejects cross-collection rows returned by a provider during TTL over-fetch', async () => { + const rows = [ + { chunkId: 'other:topic', metadata: { __collection: 'other', __key: 'topic', __value: 'foreign', __insertedAt: Date.now() } }, + { chunkId: 'c:topic', metadata: { __collection: 'c', __key: 'topic', __value: 'local', __insertedAt: Date.now() } }, + ] + const vector = fakeVectorWithRows(rows) + const providerIgnoringFilter: MemoryVectorStoreLike = { + ...vector, + query: async (_vec, k) => rows.slice(0, k).map(row => ({ chunkId: row.chunkId, score: 1, metadata: row.metadata })), + } + const store = createVectorStore({ + config: { ...config, ttlSeconds: 60 }, vectorStore: providerIgnoringFilter, embedder, + }) + + expect(await store.get('topic')).toBe('local') + expect(await store.recall('anything', 5)).toEqual(['local']) + }) + + it('never returns more than k live values after TTL over-fetch and treats k=0 as empty', async () => { + const rows = Array.from({ length: 8 }, (_, i) => ({ + chunkId: `c:${i}`, + metadata: { __collection: 'c', __key: `${i}`, __value: i, __insertedAt: Date.now() }, + })) + const store = createVectorStore({ + config: { ...config, ttlSeconds: 60 }, vectorStore: fakeVectorWithRows(rows), embedder, + }) + expect(await store.recall('anything', 2)).toEqual([0, 1]) + expect(await store.recall('anything', 0)).toEqual([]) + }) }) diff --git a/packages/memory/tests/memory-batch-2.test.ts b/packages/memory/tests/memory-batch-2.test.ts index 328226ac7..1c1d3ea0d 100644 --- a/packages/memory/tests/memory-batch-2.test.ts +++ b/packages/memory/tests/memory-batch-2.test.ts @@ -78,6 +78,34 @@ describe('milvusVectorStore', () => { }) describe('mongoAtlasVectorStore', () => { + it('replaces by id through ordered upsert bulk writes and keeps the last duplicate in a batch', async () => { + let operations: Array> = [] + let options: unknown + const collection = { + insertMany: vi.fn(), + bulkWrite: vi.fn(async (ops: Array>, opts: unknown) => { operations = ops; options = opts }), + deleteMany: vi.fn(async () => undefined), + aggregate: vi.fn(), + } + const store = mongoAtlasVectorStore({ collection, indexName: 'idx' }) + const first = { id: 'a1', content: 'first', embedding: [0.2], metadata: { version: 1 } } + const last = { id: 'a1', content: 'last', embedding: [0.5], metadata: { source: 'test' } } + await store.store([first, last]) + expect(operations).toHaveLength(2) + expect(operations[0]?.updateOne).toMatchObject({ + filter: { _id: 'a1' }, + upsert: true, + update: { $set: { content: 'first', embedding: [0.2], metadata: { version: 1 } } }, + }) + expect(operations[1]?.updateOne).toMatchObject({ + filter: { _id: 'a1' }, + upsert: true, + update: { $set: { content: 'last', embedding: [0.5], metadata: { source: 'test' } } }, + }) + expect(options).toEqual({ ordered: true }) + expect(collection.insertMany).not.toHaveBeenCalled() + }) + it('store inserts with _id + embedding field', async () => { const inserted: Array> = [] const collection = { @@ -91,6 +119,29 @@ describe('mongoAtlasVectorStore', () => { expect((inserted[0] as { embedding: unknown }).embedding).toEqual([0.1]) }) + it('uses ordered updateOne upserts when bulkWrite is unavailable and preserves host fields', async () => { + const rows = new Map>([['a1', { _id: 'a1', hostField: 'keep' }]]) + const calls: string[] = [] + const collection = { + insertMany: vi.fn(), + updateOne: vi.fn(async (filter: Record, update: Record) => { + const id = filter._id as string + calls.push(String((update.$set as Record).content)) + rows.set(id, { ...rows.get(id), ...(update.$set as Record) }) + }), + deleteMany: vi.fn(async () => undefined), + aggregate: vi.fn(), + } + const store = mongoAtlasVectorStore({ collection, indexName: 'idx' }) + await store.store([ + { id: 'a1', content: 'old', embedding: [0.1] }, + { id: 'a1', content: 'new', embedding: [0.9] }, + ]) + expect(calls).toEqual(['old', 'new']) + expect(rows.get('a1')).toMatchObject({ hostField: 'keep', content: 'new', embedding: [0.9] }) + expect(collection.insertMany).not.toHaveBeenCalled() + }) + it('search builds a $vectorSearch pipeline', async () => { const captured: Array>> = [] const collection = { diff --git a/packages/memory/tests/redis-chat-extra.test.ts b/packages/memory/tests/redis-chat-extra.test.ts index 6bdcea537..8c291ac51 100644 --- a/packages/memory/tests/redis-chat-extra.test.ts +++ b/packages/memory/tests/redis-chat-extra.test.ts @@ -51,4 +51,33 @@ describe('redisChatMemory — extra branches', () => { const result = await mem.load() expect(result).toEqual([]) }) + + it('retries lazy client creation after connect rejects', async () => { + const failedClient = { + connect: vi.fn(async () => { throw new Error('temporary connect failure') }), + destroy: vi.fn(), + } + const healthyClient = { + connect: vi.fn(async () => undefined), + get: vi.fn(async () => null), + set: vi.fn(async () => undefined), + del: vi.fn(async () => undefined), + keys: vi.fn(async () => []), + close: vi.fn(async () => undefined), + destroy: vi.fn(), + sendCommand: vi.fn(async () => null), + } + const createClient = vi.fn() + .mockReturnValueOnce(failedClient) + .mockReturnValueOnce(healthyClient) + vi.doMock('redis', () => ({ createClient })) + + const { redisChatMemory } = await import('../src/redis-chat') + const memory = redisChatMemory({ url: 'redis://localhost:6379' }) + await expect(memory.load()).rejects.toThrow('temporary connect failure') + await expect(memory.load()).resolves.toEqual([]) + expect(createClient).toHaveBeenCalledTimes(2) + expect(failedClient.destroy).toHaveBeenCalledTimes(1) + expect(healthyClient.destroy).not.toHaveBeenCalled() + }) }) diff --git a/packages/memory/tests/redis-client.test.ts b/packages/memory/tests/redis-client.test.ts index f72f96127..344c06813 100644 --- a/packages/memory/tests/redis-client.test.ts +++ b/packages/memory/tests/redis-client.test.ts @@ -19,7 +19,7 @@ describe('createRedisClientAdapter', () => { }) it('creates adapter methods when redis is available', async () => { - const commands: string[] = [] + const commands: Array> = [] const store = new Map() const fakeClient = { @@ -32,8 +32,8 @@ describe('createRedisClientAdapter', () => { return [...store.keys()].filter(k => k.startsWith(prefix)) }), close: vi.fn().mockResolvedValue(undefined), - sendCommand: vi.fn(async (args: string[]) => { - commands.push(args[0]!) + sendCommand: vi.fn(async (args: Array) => { + commands.push(args) return 'OK' }), } @@ -74,6 +74,9 @@ describe('createRedisClientAdapter', () => { const result = await adapter.call('PING') expect(fakeClient.sendCommand).toHaveBeenCalled() expect(result).toBe('OK') + const vector = Buffer.from([0, 1, 2, 255]) + await adapter.call('HSET', 'key', 'embedding', vector) + expect(commands.at(-1)?.at(-1)).toEqual(vector) }) it('del with empty array is a no-op', async () => { @@ -94,4 +97,52 @@ describe('createRedisClientAdapter', () => { await adapter.del([]) expect(fakeClient.del).not.toHaveBeenCalled() }) + + it('preserves the connection error when destroying an owned failed client also throws', async () => { + const connectionError = new Error('connect failed') + const fakeClient = { + connect: vi.fn().mockRejectedValue(connectionError), + destroy: vi.fn(() => { throw new Error('destroy failed') }), + } + vi.doMock('redis', () => ({ createClient: vi.fn(() => fakeClient) })) + const { createRedisClientAdapter: create } = await import('../src/redis-client') + + await expect(create('redis://localhost')).rejects.toBe(connectionError) + expect(fakeClient.destroy).toHaveBeenCalledOnce() + }) + + it('preserves Float32 vector bytes through the default Redis adapter for store and search', async () => { + const calls: Array<{ args: Array }> = [] + const fakeClient = { + connect: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + sendCommand: vi.fn(async (args: Array) => { + calls.push({ args }) + return args[0] === 'FT.SEARCH' ? [0] : 'OK' + }), + destroy: vi.fn(), + } + vi.doMock('redis', () => ({ createClient: vi.fn(() => fakeClient) })) + const { redisVectorMemory } = await import('../src/redis-vector') + const memory = redisVectorMemory({ url: 'redis://synthetic.invalid', dimensions: 3 }) + const expected = Buffer.alloc(12) + ;[0.1, 0.2, 0.3].forEach((value, index) => expected.writeFloatLE(value, index * 4)) + + await memory.store([{ id: 'vector-1', content: 'synthetic', embedding: [0.1, 0.2, 0.3] }]) + await memory.search([0.1, 0.2, 0.3]) + + const hset = calls.find(({ args }) => args[0] === 'HSET')?.args + const search = calls.find(({ args }) => args[0] === 'FT.SEARCH')?.args + const storedEmbedding = hset?.at(-1) + expect(Buffer.isBuffer(storedEmbedding)).toBe(true) + expect((storedEmbedding as Buffer).byteLength).toBe(12) + expect(storedEmbedding).toEqual(expected) + const searchVectorIndex = search?.indexOf('vec') ?? -1 + expect(searchVectorIndex).toBeGreaterThanOrEqual(0) + const searchedEmbedding = search?.[searchVectorIndex + 1] + expect(Buffer.isBuffer(searchedEmbedding)).toBe(true) + expect((searchedEmbedding as Buffer).byteLength).toBe(12) + expect(searchedEmbedding).toEqual(expected) + expect(fakeClient.destroy).not.toHaveBeenCalled() + }) }) diff --git a/packages/memory/tests/sqlite.test.ts b/packages/memory/tests/sqlite.test.ts index 977a2674a..24e59b288 100644 --- a/packages/memory/tests/sqlite.test.ts +++ b/packages/memory/tests/sqlite.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { sqliteChatMemory } from '../src/sqlite' import type { Message } from '@agentskit/core' import { join } from 'node:path' @@ -22,6 +22,8 @@ describe('sqliteChatMemory', () => { afterEach(async () => { try { await unlink(dbPath) } catch {} + vi.doUnmock('better-sqlite3') + vi.resetModules() }) it('returns empty array when no messages saved', async () => { @@ -29,6 +31,52 @@ describe('sqliteChatMemory', () => { expect(await mem.load()).toEqual([]) }) + it('retries database initialization after the first open fails', async () => { + let constructorCalls = 0 + class RetryDatabase { + constructor() { + constructorCalls++ + if (constructorCalls === 1) throw new Error('temporary open failure') + } + prepare() { + return { run: vi.fn(), get: vi.fn(() => undefined) } + } + } + vi.doMock('better-sqlite3', () => ({ default: RetryDatabase })) + const { sqliteChatMemory: createMemory } = await import('../src/sqlite') + const memory = createMemory({ path: dbPath }) + + await expect(memory.load()).rejects.toThrow() + await expect(memory.load()).resolves.toEqual([]) + expect(constructorCalls).toBe(2) + vi.doUnmock('better-sqlite3') + }) + + it('closes an owned database when schema initialization fails and preserves that error', async () => { + let constructorCalls = 0 + const close = vi.fn(() => { throw new Error('cleanup failure') }) + class RetryDatabase { + constructor() { constructorCalls++ } + prepare(sql: string) { + return { + run: vi.fn(() => { + if (constructorCalls === 1 && sql.includes('CREATE TABLE')) throw new Error('schema failure') + }), + get: vi.fn(() => undefined), + } + } + close() { close() } + } + vi.doMock('better-sqlite3', () => ({ default: RetryDatabase })) + const { sqliteChatMemory: createMemory } = await import('../src/sqlite') + const memory = createMemory({ path: dbPath }) + + await expect(memory.load()).rejects.toThrow('schema failure') + await expect(memory.load()).resolves.toEqual([]) + expect(close).toHaveBeenCalledTimes(1) + expect(constructorCalls).toBe(2) + }) + it('save then load round-trips with date serialization', async () => { const mem = sqliteChatMemory({ path: dbPath }) await mem.save([sampleMessage]) diff --git a/packages/memory/tests/turso-full.test.ts b/packages/memory/tests/turso-full.test.ts index 59f4ee63b..0539f3178 100644 --- a/packages/memory/tests/turso-full.test.ts +++ b/packages/memory/tests/turso-full.test.ts @@ -41,6 +41,31 @@ function makeFakeClient() { } describe('tursoChatMemory (injected fake client)', () => { + it('retries client initialization after schema creation rejects', async () => { + let createCalls = 0 + const firstClose = vi.fn(async () => { throw new Error('cleanup failure') }) + const failedClient = { + execute: vi.fn(async () => { throw new Error('temporary schema failure') }), + close: firstClose, + } + const healthyClient = { execute: vi.fn(async () => ({ rows: [] })), close: vi.fn(async () => undefined) } + const fakeLibsql = { + createClient: vi.fn(() => { + createCalls++ + return createCalls === 1 ? failedClient : healthyClient + }), + } + vi.doMock('@libsql/client', () => fakeLibsql) + + const { tursoChatMemory } = await import('../src/turso') + const memory = tursoChatMemory({ url: 'file::memory:' }) + await expect(memory.load()).rejects.toThrow('temporary schema failure') + await expect(memory.load()).resolves.toEqual([]) + expect(createCalls).toBe(2) + expect(firstClose).toHaveBeenCalledTimes(1) + expect(healthyClient.close).not.toHaveBeenCalled() + }) + it('load returns empty array when no messages stored', async () => { const fakeClient = makeFakeClient() const fakeLibsql = { createClient: vi.fn(() => fakeClient) } diff --git a/readme-standard-v1.json b/readme-standard-v1.json index 1c22c7b4c..f9221ee09 100644 --- a/readme-standard-v1.json +++ b/readme-standard-v1.json @@ -987,7 +987,7 @@ "docs/STABILITY.md", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:e93bb599c99d5faf5c8b6f15137b1407933bd9c1e56650fa7313b6abaa677ad8" + "sourceHash": "sha256:752ba43c1f3597ba227abc62ad6d4ef93fc32a32e367f6fddb0b79f296be49ec" }, "exceptions": [] },