From 214208867e9c58660e9f4bd3d3ecddbb637c122e Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Sun, 8 Nov 2026 11:15:45 +0800 Subject: [PATCH 1/9] feat(embedding): configure local models and query prefixes Signed-off-by: zhichao-poper --- AGENTS.md | 4 +- README.md | 4 +- READMEs/README.ar-SA.md | 4 +- READMEs/README.bn-BD.md | 4 +- READMEs/README.cs-CZ.md | 4 +- READMEs/README.da-DK.md | 4 +- READMEs/README.de-DE.md | 4 +- READMEs/README.el-GR.md | 4 +- READMEs/README.es-ES.md | 4 +- READMEs/README.fi-FI.md | 4 +- READMEs/README.fr-FR.md | 4 +- READMEs/README.he-IL.md | 4 +- READMEs/README.hi-IN.md | 4 +- READMEs/README.hu-HU.md | 4 +- READMEs/README.id-ID.md | 4 +- READMEs/README.it-IT.md | 4 +- READMEs/README.ja-JP.md | 4 +- READMEs/README.ko-KR.md | 4 +- READMEs/README.nb-NO.md | 4 +- READMEs/README.nl-NL.md | 4 +- READMEs/README.pl-PL.md | 4 +- READMEs/README.pt-BR.md | 4 +- READMEs/README.pt-PT.md | 4 +- READMEs/README.ro-RO.md | 4 +- READMEs/README.ru-RU.md | 4 +- READMEs/README.sv-SE.md | 4 +- READMEs/README.th-TH.md | 4 +- READMEs/README.tl-PH.md | 4 +- READMEs/README.tr-TR.md | 4 +- READMEs/README.uk-UA.md | 4 +- READMEs/README.ur-PK.md | 4 +- READMEs/README.vi-VN.md | 4 +- READMEs/README.zh-CN.md | 4 +- READMEs/README.zh-TW.md | 4 +- assets/tags/light/stat-tests.svg | 4 +- assets/tags/stat-tests.svg | 4 +- scripts/docs-sync.state.json | 2 +- src/cli/compose-project.ts | 3 + src/functions/search.ts | 2 +- src/providers/embedding/index.ts | 3 + src/providers/embedding/local.ts | 83 +++++++++++++++++++++------ src/state/hybrid-search.ts | 4 +- src/types.ts | 1 + test/compose-project.test.ts | 11 ++++ test/hybrid-search.test.ts | 22 +++++++ test/local-embedding-config.test.ts | 72 +++++++++++++++++++++++ test/local-embedding-provider.test.ts | 4 +- 47 files changed, 254 insertions(+), 97 deletions(-) create mode 100644 test/local-embedding-config.test.ts diff --git a/AGENTS.md b/AGENTS.md index 8b2287d7c5..626ff4de24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,7 +101,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Testing -- Before every commit run, in order: `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm test` (4,400+ tests). Commit anything the generators changed. CI runs the same build, skills check and tests. +- Before every commit run, in order: `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm test` (4,500+ tests). Commit anything the generators changed. CI runs the same build, skills check and tests. - Before any npm publish: `npm run release:gate` must pass. It installs the packed tarballs into a clean home and runs capture, recovery, restart, export/import, MCP and status checks against the installed CLI (see CONTRIBUTING.md, Release process) - Mock pattern: `vi.mock("iii-sdk")` with mock `sdk.trigger`, `kv.get/set/list` - Test files go in `test/` with `.test.ts` extension @@ -114,4 +114,4 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). - 6 MCP resources, 3 MCP prompts - 12 hooks, 17 skills - 322+ iii functions -- 4,400+ tests +- 4,500+ tests diff --git a/README.md b/README.md index 3686943fc1..e60c37d257 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1307,7 +1307,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.ar-SA.md b/READMEs/README.ar-SA.md index 17f9286e93..43d2c4d432 100644 --- a/READMEs/README.ar-SA.md +++ b/READMEs/README.ar-SA.md @@ -71,7 +71,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1242,7 +1242,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.bn-BD.md b/READMEs/README.bn-BD.md index 08c252833a..8310128629 100644 --- a/READMEs/README.bn-BD.md +++ b/READMEs/README.bn-BD.md @@ -69,7 +69,7 @@ 54 MCP টুল 12 অটো হুক 0 এক্সটার্নাল ডিবি - 4,400+ টেস্ট পাস + 4,500+ টেস্ট পাস

@@ -1255,7 +1255,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.cs-CZ.md b/READMEs/README.cs-CZ.md index 821c2742b9..5a6c82efe5 100644 --- a/READMEs/README.cs-CZ.md +++ b/READMEs/README.cs-CZ.md @@ -69,7 +69,7 @@ 54 MCP nástrojů 12 automatických hooks 0 externích databází - 4,400+ testů úspěšně proběhlo + 4,500+ testů úspěšně proběhlo

@@ -1241,7 +1241,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.da-DK.md b/READMEs/README.da-DK.md index f11435c2d8..4a2d0e1000 100644 --- a/READMEs/README.da-DK.md +++ b/READMEs/README.da-DK.md @@ -69,7 +69,7 @@ 54 MCP-værktøjer 12 automatiske hooks 0 eksterne databaser - 4,400+ beståede tests + 4,500+ beståede tests

@@ -1249,7 +1249,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.de-DE.md b/READMEs/README.de-DE.md index cbaca6af3e..40064c7371 100644 --- a/READMEs/README.de-DE.md +++ b/READMEs/README.de-DE.md @@ -69,7 +69,7 @@ 54 MCP-Tools 12 Auto-Hooks 0 externe DBs - 4,400+ bestandene Tests + 4,500+ bestandene Tests

@@ -1247,7 +1247,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.el-GR.md b/READMEs/README.el-GR.md index 435ae03670..63d3a6fe5d 100644 --- a/READMEs/README.el-GR.md +++ b/READMEs/README.el-GR.md @@ -69,7 +69,7 @@ 54 εργαλεία MCP 12 αυτόματα hooks 0 εξωτερικές βάσεις δεδομένων - 4,400+ επιτυχημένες δοκιμές + 4,500+ επιτυχημένες δοκιμές

@@ -1241,7 +1241,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.es-ES.md b/READMEs/README.es-ES.md index 25a414bff8..798c26c413 100644 --- a/READMEs/README.es-ES.md +++ b/READMEs/README.es-ES.md @@ -69,7 +69,7 @@ 54 tools MCP 12 hooks automáticos 0 bases de datos externas - 4,400+ tests superados + 4,500+ tests superados

@@ -1243,7 +1243,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.fi-FI.md b/READMEs/README.fi-FI.md index c608d1bc9c..1a7520e672 100644 --- a/READMEs/README.fi-FI.md +++ b/READMEs/README.fi-FI.md @@ -69,7 +69,7 @@ 54 MCP-työkalua 12 automaattista hookia 0 ulkoista tietokantaa - 4,400+ testiä läpäisty + 4,500+ testiä läpäisty

@@ -1241,7 +1241,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.fr-FR.md b/READMEs/README.fr-FR.md index 7284da37e8..42bba1dc7d 100644 --- a/READMEs/README.fr-FR.md +++ b/READMEs/README.fr-FR.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1250,7 +1250,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.he-IL.md b/READMEs/README.he-IL.md index d8f06e0b17..0b116fa100 100644 --- a/READMEs/README.he-IL.md +++ b/READMEs/README.he-IL.md @@ -71,7 +71,7 @@ 54 כלי MCP 12 hooks אוטומטיים 0 מסדי נתונים חיצוניים - 4,400+ בדיקות עוברות + 4,500+ בדיקות עוברות

@@ -1242,7 +1242,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.hi-IN.md b/READMEs/README.hi-IN.md index 84f168106a..c628d732be 100644 --- a/READMEs/README.hi-IN.md +++ b/READMEs/README.hi-IN.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1256,7 +1256,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.hu-HU.md b/READMEs/README.hu-HU.md index fd1d89534f..6afe735b63 100644 --- a/READMEs/README.hu-HU.md +++ b/READMEs/README.hu-HU.md @@ -69,7 +69,7 @@ 54 MCP eszköz 12 automatikus hook 0 külső adatbázis - 4,400+ sikeres teszt + 4,500+ sikeres teszt

@@ -1239,7 +1239,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.id-ID.md b/READMEs/README.id-ID.md index 025497f109..85d729f1ec 100644 --- a/READMEs/README.id-ID.md +++ b/READMEs/README.id-ID.md @@ -69,7 +69,7 @@ 54 tool MCP 12 hook otomatis 0 DB eksternal - 4,400+ pengujian lolos + 4,500+ pengujian lolos

@@ -1256,7 +1256,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.it-IT.md b/READMEs/README.it-IT.md index 2ca4bacd8c..d473421353 100644 --- a/READMEs/README.it-IT.md +++ b/READMEs/README.it-IT.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1249,7 +1249,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.ja-JP.md b/READMEs/README.ja-JP.md index 18d1747608..253ea84286 100644 --- a/READMEs/README.ja-JP.md +++ b/READMEs/README.ja-JP.md @@ -69,7 +69,7 @@ 54 MCP ツール 12 自動 hooks 外部 DB 0 個 - 4,400+ 件のテストが成功 + 4,500+ 件のテストが成功

@@ -1255,7 +1255,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.ko-KR.md b/READMEs/README.ko-KR.md index 54b35e4b52..25091180c3 100644 --- a/READMEs/README.ko-KR.md +++ b/READMEs/README.ko-KR.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1254,7 +1254,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.nb-NO.md b/READMEs/README.nb-NO.md index f404abe30a..fe6b9e189f 100644 --- a/READMEs/README.nb-NO.md +++ b/READMEs/README.nb-NO.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1247,7 +1247,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.nl-NL.md b/READMEs/README.nl-NL.md index 4f2a723fa5..d83f316edf 100644 --- a/READMEs/README.nl-NL.md +++ b/READMEs/README.nl-NL.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1243,7 +1243,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.pl-PL.md b/READMEs/README.pl-PL.md index b6215cbe2c..6c5a94a121 100644 --- a/READMEs/README.pl-PL.md +++ b/READMEs/README.pl-PL.md @@ -69,7 +69,7 @@ 54 narzędzia MCP 12 automatycznych hooków 0 zewnętrznych baz danych - 4,400+ zaliczonych testów + 4,500+ zaliczonych testów

@@ -1239,7 +1239,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.pt-BR.md b/READMEs/README.pt-BR.md index 2a3020d51c..785e39df45 100644 --- a/READMEs/README.pt-BR.md +++ b/READMEs/README.pt-BR.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1247,7 +1247,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.pt-PT.md b/READMEs/README.pt-PT.md index 7d5dd04a05..c13dd11522 100644 --- a/READMEs/README.pt-PT.md +++ b/READMEs/README.pt-PT.md @@ -69,7 +69,7 @@ 54 ferramentas MCP 12 hooks automáticos 0 bases de dados externas - 4,400+ testes aprovados + 4,500+ testes aprovados

@@ -1243,7 +1243,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.ro-RO.md b/READMEs/README.ro-RO.md index c1495e92e9..2333786508 100644 --- a/READMEs/README.ro-RO.md +++ b/READMEs/README.ro-RO.md @@ -69,7 +69,7 @@ 54 de instrumente MCP 12 hook-uri automate 0 baze de date externe - 4,400+ teste trecute + 4,500+ teste trecute

@@ -1235,7 +1235,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.ru-RU.md b/READMEs/README.ru-RU.md index 18f84998a8..2155e0c98a 100644 --- a/READMEs/README.ru-RU.md +++ b/READMEs/README.ru-RU.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1241,7 +1241,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.sv-SE.md b/READMEs/README.sv-SE.md index 1443692691..4381b16ac6 100644 --- a/READMEs/README.sv-SE.md +++ b/READMEs/README.sv-SE.md @@ -69,7 +69,7 @@ 54 MCP-verktyg 12 automatiska hooks 0 externa databaser - 4,400+ godkända tester + 4,500+ godkända tester

@@ -1249,7 +1249,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.th-TH.md b/READMEs/README.th-TH.md index dd435eac3f..47fe863712 100644 --- a/READMEs/README.th-TH.md +++ b/READMEs/README.th-TH.md @@ -69,7 +69,7 @@ 54 เครื่องมือ MCP 12 hook อัตโนมัติ ไม่ต้องพึ่ง external DB เลย - ผ่านการทดสอบ 4,400+ รายการ + ผ่านการทดสอบ 4,500+ รายการ

@@ -1242,7 +1242,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.tl-PH.md b/READMEs/README.tl-PH.md index d10ae9448d..4c3c0788be 100644 --- a/READMEs/README.tl-PH.md +++ b/READMEs/README.tl-PH.md @@ -69,7 +69,7 @@ 54 MCP tool 12 awtomatikong hook 0 panlabas na DB - 4,400+ pumapasang test + 4,500+ pumapasang test

@@ -1251,7 +1251,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.tr-TR.md b/READMEs/README.tr-TR.md index a722398737..74a7497fe1 100644 --- a/READMEs/README.tr-TR.md +++ b/READMEs/README.tr-TR.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1241,7 +1241,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.uk-UA.md b/READMEs/README.uk-UA.md index 6f03ca9aaa..a7dcc8236a 100644 --- a/READMEs/README.uk-UA.md +++ b/READMEs/README.uk-UA.md @@ -69,7 +69,7 @@ 54 інструменти MCP 12 автоматичних хуків 0 зовнішніх БД - 4,400+ тестів пройдено + 4,500+ тестів пройдено

@@ -1242,7 +1242,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.ur-PK.md b/READMEs/README.ur-PK.md index b81fcb3ce4..8527ed85cf 100644 --- a/READMEs/README.ur-PK.md +++ b/READMEs/README.ur-PK.md @@ -71,7 +71,7 @@ 54 MCP ٹولز 12 خودکار ہکس 0 خارجی ڈیٹابیسز - 4,400+ ٹیسٹس پاس ہو رہے ہیں + 4,500+ ٹیسٹس پاس ہو رہے ہیں

@@ -1246,7 +1246,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.vi-VN.md b/READMEs/README.vi-VN.md index 5e157b445e..f00703fa98 100644 --- a/READMEs/README.vi-VN.md +++ b/READMEs/README.vi-VN.md @@ -69,7 +69,7 @@ 54 MCP tools 12 auto hooks 0 external DBs - 4,400+ tests passing + 4,500+ tests passing

@@ -1251,7 +1251,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.zh-CN.md b/READMEs/README.zh-CN.md index dca9c73d72..13233660b0 100644 --- a/READMEs/README.zh-CN.md +++ b/READMEs/README.zh-CN.md @@ -69,7 +69,7 @@ 54 个 MCP 工具 12 个自动 hooks 0 个外部数据库 - 4,400+ 项测试通过 + 4,500+ 项测试通过

@@ -1252,7 +1252,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/READMEs/README.zh-TW.md b/READMEs/README.zh-TW.md index eb7a1ddb4a..0041886d6c 100644 --- a/READMEs/README.zh-TW.md +++ b/READMEs/README.zh-TW.md @@ -69,7 +69,7 @@ 54 個 MCP 工具 12 個自動 hooks 0 個外部資料庫 - 4,400+ 測試通過 + 4,500+ 測試通過

@@ -1244,7 +1244,7 @@ curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Authorization: ```bash npm run dev # Hot reload npm run build # Production build -npm test # 4,400+ tests +npm test # 4,500+ tests npm run test:integration # API tests (requires running services) ``` diff --git a/assets/tags/light/stat-tests.svg b/assets/tags/light/stat-tests.svg index 26a2ed1add..3439518513 100644 --- a/assets/tags/light/stat-tests.svg +++ b/assets/tags/light/stat-tests.svg @@ -1,5 +1,5 @@ - + - 4400+ + 4500+ TESTS PASSING diff --git a/assets/tags/stat-tests.svg b/assets/tags/stat-tests.svg index 6b3ab1051e..9352019436 100644 --- a/assets/tags/stat-tests.svg +++ b/assets/tags/stat-tests.svg @@ -1,5 +1,5 @@ - + - 4400+ + 4500+ TESTS PASSING diff --git a/scripts/docs-sync.state.json b/scripts/docs-sync.state.json index 3ed32d2441..c62d278fdf 100644 --- a/scripts/docs-sync.state.json +++ b/scripts/docs-sync.state.json @@ -20,7 +20,7 @@ "12" ], "tests": [ - "4400+" + "4500+" ], "functions": [ "323" diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 289828f117..d101f23e28 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -52,6 +52,9 @@ export function composeProjectPaths(dataDir: string): { } const AGENTMEMORY_WORKER_ENV_ALLOWLIST = new Set([ + "LOCAL_EMBEDDING_MODEL", "LOCAL_EMBEDDING_DIMENSIONS", "LOCAL_EMBEDDING_POOLING", + "LOCAL_EMBEDDING_QUERY_PREFIX", "LOCAL_EMBEDDING_DOCUMENT_PREFIX", + "LOCAL_EMBEDDING_CACHE_DIR", "LOCAL_EMBEDDING_BATCH_SIZE", "EMBEDDING_PROVIDER", "AGENTMEMORY_LLM_PROVIDER", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_MODEL", "OPENAI_EMBEDDING_API_KEY", "OPENAI_EMBEDDING_BASE_URL", "OPENAI_EMBEDDING_MODEL", "OPENAI_EMBEDDING_DIMENSIONS", "ANTHROPIC_API_KEY", diff --git a/src/functions/search.ts b/src/functions/search.ts index fefd6b9d11..0b296e22ad 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -123,7 +123,7 @@ export async function rankMemoryIds( let semantic: Array<{ obsId: string }> = [] if (vectorIndex && vectorIndex.size > 0 && currentEmbeddingProvider) { try { - const embedding = await currentEmbeddingProvider.embed(query) + const embedding = await (currentEmbeddingProvider.embedQuery?.(query) ?? currentEmbeddingProvider.embed(query)) semantic = vectorIndex .search(embedding, fetchLimit, "memory", (id) => getSearchIndex().layerOf(id)) } catch (err) { diff --git a/src/providers/embedding/index.ts b/src/providers/embedding/index.ts index 58f48a9fe8..8799a988ec 100644 --- a/src/providers/embedding/index.ts +++ b/src/providers/embedding/index.ts @@ -72,6 +72,9 @@ export function withDimensionGuard(provider: EmbeddingProvider): EmbeddingProvid out.forEach((v, i) => check(v, `embedBatch[${i}]`)); return out; }; + if (provider.embedQuery) { + wrapped.embedQuery = async (t) => check(await provider.embedQuery!(t), "embedQuery"); + } if (provider.embedImage) { wrapped.embedImage = async (s: string) => check(await provider.embedImage!(s), "embedImage"); diff --git a/src/providers/embedding/local.ts b/src/providers/embedding/local.ts index 7646815df6..2cbc054ed6 100644 --- a/src/providers/embedding/local.ts +++ b/src/providers/embedding/local.ts @@ -5,43 +5,88 @@ type FeatureExtractor = ( options: { pooling: string; normalize: boolean }, ) => Promise<{ tolist: () => number[][] }>; +function positiveInteger(name: string, fallback: number): number { + const raw = process.env[name]; + if (!raw?.trim()) return fallback; + const value = Number(raw); + if (!/^\d+$/.test(raw) || !Number.isSafeInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + export class LocalEmbeddingProvider implements EmbeddingProvider { - readonly name = "local"; - readonly dimensions = 384; - private extractor: FeatureExtractor | null = null; + readonly model = process.env.LOCAL_EMBEDDING_MODEL?.trim() || "Xenova/all-MiniLM-L6-v2"; + readonly name = this.model === "Xenova/all-MiniLM-L6-v2" ? "local" : `local:${this.model}`; + readonly dimensions = positiveInteger("LOCAL_EMBEDDING_DIMENSIONS", 384); + readonly pooling = process.env.LOCAL_EMBEDDING_POOLING?.trim() || "mean"; + private readonly batchSize = positiveInteger("LOCAL_EMBEDDING_BATCH_SIZE", Number.MAX_SAFE_INTEGER); + private readonly queryPrefix = process.env.LOCAL_EMBEDDING_QUERY_PREFIX || ""; + private readonly documentPrefix = process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX || ""; + private extractor: Promise | null = null; + + constructor() { + if (!["mean", "cls", "first_token", "last_token", "eos"].includes(this.pooling)) { + throw new Error("LOCAL_EMBEDDING_POOLING must be mean, cls, first_token, last_token, or eos"); + } + } async embed(text: string): Promise { const [result] = await this.embedBatch([text]); return result; } + async embedQuery(text: string): Promise { + const [result] = await this.extract([this.queryPrefix + text]); + return result; + } + async embedBatch(texts: string[]): Promise { + return this.extract(texts.map((text) => this.documentPrefix + text)); + } + + private async extract(texts: string[]): Promise { + if (!texts.length) return []; const extractor = await this.getExtractor(); - const output = await extractor(texts, { - pooling: "mean", - normalize: true, - }); - return output.tolist().map((v) => new Float32Array(v)); + const vectors: Float32Array[] = []; + for (let offset = 0; offset < texts.length; offset += this.batchSize) { + const output = await extractor(texts.slice(offset, offset + this.batchSize), { + pooling: this.pooling, + normalize: true, + }); + const rows = output.tolist(); + if (rows.length !== Math.min(this.batchSize, texts.length - offset) || + rows.some((row) => row.length !== this.dimensions || row.some((n) => !Number.isFinite(n)))) { + throw new Error(`Local embedding output does not match configured dimensions (${this.dimensions})`); + } + vectors.push(...rows.map((row) => new Float32Array(row))); + } + return vectors; } - private async getExtractor() { - if (this.extractor) return this.extractor; + private getExtractor(): Promise { + if (!this.extractor) { + this.extractor = this.loadExtractor().catch((error) => { + this.extractor = null; + throw error; + }); + } + return this.extractor; + } + + private async loadExtractor(): Promise { let transformers: typeof import("@huggingface/transformers"); try { transformers = await import("@huggingface/transformers"); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ERR_MODULE_NOT_FOUND") { - throw new Error( - "Install @huggingface/transformers for local embeddings: npm install @huggingface/transformers", - ); + throw new Error("Install @huggingface/transformers for local embeddings: npm install @huggingface/transformers"); } throw err; } - this.extractor = (await transformers.pipeline( - "feature-extraction", - "Xenova/all-MiniLM-L6-v2", - { dtype: "q8" }, - )) as FeatureExtractor; - return this.extractor; + return await transformers.pipeline("feature-extraction", this.model, { + dtype: "q8", + ...(process.env.LOCAL_EMBEDDING_CACHE_DIR ? { cache_dir: process.env.LOCAL_EMBEDDING_CACHE_DIR } : {}), + }) as FeatureExtractor; } } diff --git a/src/state/hybrid-search.ts b/src/state/hybrid-search.ts index caeb8f03b8..5e5db6048f 100644 --- a/src/state/hybrid-search.ts +++ b/src/state/hybrid-search.ts @@ -116,7 +116,7 @@ export class HybridSearch { try { if (request) { if (!request.embedding || request.embedding.query !== query) { - request.embedding = { query, value: this.embeddingProvider.embed(query) }; + request.embedding = { query, value: (this.embeddingProvider.embedQuery?.(query) ?? this.embeddingProvider.embed(query)) }; } const embedding = request.embedding; try { @@ -126,7 +126,7 @@ export class HybridSearch { throw error; } } else { - queryEmbedding = await this.embeddingProvider.embed(query); + queryEmbedding = await (this.embeddingProvider.embedQuery?.(query) ?? this.embeddingProvider.embed(query)); } vectorResults = this.vector.search(queryEmbedding, retrievalLimit * 2, targetLayer, (id) => this.bm25.layerOf(id)); } catch { diff --git a/src/types.ts b/src/types.ts index 5ab42970f2..3908cc3030 100644 --- a/src/types.ts +++ b/src/types.ts @@ -317,6 +317,7 @@ export interface EmbeddingProvider { name: string; dimensions: number; embed(text: string): Promise; + embedQuery?(text: string): Promise; embedBatch(texts: string[]): Promise; embedImage?(src: string): Promise; } diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index 621a246667..78be31468b 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -24,6 +24,17 @@ const options = (overrides: Record = {}) => ({ }); describe("Worker Compose project", () => { + it("forwards all local embedding settings to the application worker", () => { + const settings = { + LOCAL_EMBEDDING_MODEL: "test/model", LOCAL_EMBEDDING_DIMENSIONS: "3", + LOCAL_EMBEDDING_POOLING: "cls", LOCAL_EMBEDDING_QUERY_PREFIX: "query: ", + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "passage: ", LOCAL_EMBEDDING_CACHE_DIR: "model-cache", + LOCAL_EMBEDDING_BATCH_SIZE: "2", + }; + const app = parse(renderWorkerCompose(options({ workerEnvironment: settings }))).containers.agentmemory; + expect(app.environment).toMatchObject(settings); + }); + it("pins registry workers and configures the isolated engine, state and HTTP", () => { const text = renderWorkerCompose(options()); expect(text).toContain("namespace: agentmemory-3511"); diff --git a/test/hybrid-search.test.ts b/test/hybrid-search.test.ts index fde7aa2deb..0165e1449f 100644 --- a/test/hybrid-search.test.ts +++ b/test/hybrid-search.test.ts @@ -54,6 +54,28 @@ describe("HybridSearch", () => { kv = mockKV(); }); + it("uses query embeddings for direct, cached and expanded searches", async () => { + const obs = makeObs(); + bm25.add(obs); + await kv.set(KV.observations(obs.sessionId), obs.id, obs); + const vector = new VectorIndex(); + vector.add(obs.id, obs.sessionId, new Float32Array([1, 0])); + const embed = vi.fn(async () => new Float32Array([0, 1])); + const embedQuery = vi.fn(async (_text: string) => new Float32Array([1, 0])); + const provider = { name: "test", dimensions: 2, embed, embedQuery, embedBatch: vi.fn() }; + const hybrid = new HybridSearch(bm25, vector, provider, kv as never, 0.4, 0.6, 0); + expect(await hybrid.search("auth")).toHaveLength(1); + const request = {}; + await hybrid.search("auth", 20, "all", undefined, request); + await hybrid.search("auth", 40, "all", undefined, request); + expect(embedQuery).toHaveBeenCalledTimes(2); + await hybrid.searchWithExpansion("auth", 20, { + reformulations: ["authentication"], temporalConcretizations: [], entityExtractions: [], + }); + expect(embedQuery.mock.calls.map((args) => args[0])).toEqual(["auth", "auth", "auth", "authentication"]); + expect(embed).not.toHaveBeenCalled(); + }); + it("reuses query embeddings only within a recall request", async () => { const obs = makeObs(); bm25.add(obs); diff --git a/test/local-embedding-config.test.ts b/test/local-embedding-config.test.ts new file mode 100644 index 0000000000..110a571ff1 --- /dev/null +++ b/test/local-embedding-config.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { LocalEmbeddingProvider } from "../src/providers/embedding/local.js"; +import { withDimensionGuard } from "../src/providers/embedding/index.js"; +const mock = vi.hoisted(() => ({ extractor: vi.fn(), pipeline: vi.fn() })); +vi.mock("@huggingface/transformers", () => ({ pipeline: mock.pipeline })); +afterEach(() => { vi.unstubAllEnvs(); vi.clearAllMocks(); }); +function configure() { + vi.stubEnv("LOCAL_EMBEDDING_MODEL", "test/chinese"); + vi.stubEnv("LOCAL_EMBEDDING_DIMENSIONS", "3"); + vi.stubEnv("LOCAL_EMBEDDING_POOLING", "cls"); + mock.extractor.mockImplementation(async (texts: string[]) => ({ tolist: () => texts.map(() => [1, 0, 0]) })); + mock.pipeline.mockResolvedValue(mock.extractor); +} +describe("local model configuration", () => { + it("keeps the official model, dimensions and pooling by default", () => { + const p = new LocalEmbeddingProvider(); + expect([p.name, p.model, p.dimensions, p.pooling]).toEqual(["local", "Xenova/all-MiniLM-L6-v2", 384, "mean"]); + }); + it("separates query and document prefixes, including through the guard", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_QUERY_PREFIX", "query: "); vi.stubEnv("LOCAL_EMBEDDING_DOCUMENT_PREFIX", "passage: "); + const p = withDimensionGuard(new LocalEmbeddingProvider()); + await p.embedQuery!("屏幕偏黄"); await p.embedBatch(["色彩配置"]); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8" }); + expect(mock.extractor.mock.calls.map((c) => c[0])).toEqual([["query: 屏幕偏黄"], ["passage: 色彩配置"]]); + expect(p.dimensions).toBe(3); + }); + it("bounds batches and shares initialization across concurrent calls", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_BATCH_SIZE", "2"); + const p = new LocalEmbeddingProvider(); + await Promise.all([p.embedBatch(["a", "b", "c"]), p.embedQuery("q")]); + expect(mock.pipeline).toHaveBeenCalledTimes(1); + expect(mock.extractor.mock.calls.every((c) => c[0].length <= 2)).toBe(true); + }); + it("rejects invalid dimensions and pooling before loading a model", () => { + vi.stubEnv("LOCAL_EMBEDDING_DIMENSIONS", "3oops"); + expect(() => new LocalEmbeddingProvider()).toThrow("positive integer"); + vi.stubEnv("LOCAL_EMBEDDING_DIMENSIONS", "3"); vi.stubEnv("LOCAL_EMBEDDING_POOLING", "none"); + expect(() => new LocalEmbeddingProvider()).toThrow("POOLING"); + }); + it("refuses a wrong-sized or non-finite embedding", async () => { + configure(); mock.extractor.mockResolvedValue({ tolist: () => [[1, 2]] }); + await expect(new LocalEmbeddingProvider().embed("test")).rejects.toThrow("dimensions"); + mock.extractor.mockResolvedValue({ tolist: () => [[1, NaN, 0]] }); + await expect(new LocalEmbeddingProvider().embedQuery("test")).rejects.toThrow("dimensions"); + }); + it("passes the configured cache directory to the pipeline", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "model-cache"); + await new LocalEmbeddingProvider().embed("test"); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8", cache_dir: "model-cache" }); + }); + it("retries loading after a pipeline initialization failure", async () => { + configure(); mock.pipeline.mockRejectedValueOnce(new Error("model unavailable")); + const provider = new LocalEmbeddingProvider(); + await expect(provider.embed("test")).rejects.toThrow("model unavailable"); + expect(await provider.embed("test")).toHaveLength(3); + expect(mock.pipeline).toHaveBeenCalledTimes(2); + }); + it("checks query dimensions through the wrapper", async () => { + const provider = withDimensionGuard({ + name: "test", dimensions: 3, + embed: async () => new Float32Array(3), + embedQuery: async () => new Float32Array(2), + embedBatch: async () => [], + }); + await expect(provider.embedQuery!("test")).rejects.toThrow("dimension mismatch"); + }); + it.each(["0", "-1", "1.5", "9007199254740992"])("rejects invalid batch size %s", (value) => { + vi.stubEnv("LOCAL_EMBEDDING_BATCH_SIZE", value); + expect(() => new LocalEmbeddingProvider()).toThrow("positive integer"); + }); + +}); diff --git a/test/local-embedding-provider.test.ts b/test/local-embedding-provider.test.ts index 69ce049d8b..7a1a4c5d47 100644 --- a/test/local-embedding-provider.test.ts +++ b/test/local-embedding-provider.test.ts @@ -21,7 +21,7 @@ describe("LocalEmbeddingProvider (package unavailable)", () => { describe("LocalEmbeddingProvider (with loaded pipeline)", () => { function mockSuccessModule() { const extractor = vi.fn(async (texts: string[]) => ({ - tolist: () => texts.map(() => [0.1, 0.2, 0.3]), + tolist: () => texts.map(() => Array(384).fill(0.1)), })); const pipeline = vi.fn(() => Promise.resolve(extractor)); vi.doMock("@huggingface/transformers", () => ({ pipeline })); @@ -46,7 +46,7 @@ describe("LocalEmbeddingProvider (with loaded pipeline)", () => { normalize: true, }); expect(vec).toBeInstanceOf(Float32Array); - expect(vec).toEqual(new Float32Array([0.1, 0.2, 0.3])); + expect(vec).toEqual(new Float32Array(Array(384).fill(0.1))); }); it("embedBatch returns one Float32Array per input text", async () => { From 6333cfd3692f597da04b8083f3dfd97d4b6f507c Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Sun, 8 Nov 2026 21:02:05 +0800 Subject: [PATCH 2/9] fix(embedding): preserve deployment and cache configuration Forward local model settings through all supplied Compose deployments, resolve relative cache paths from the CLI directory, and preserve multiline cache values. Capture the cache location in both loading and vector identity. Bound vector cleanup to 32 concurrent deletes and settle each batch before reporting an error or adopting a new identity. Document artifact and cloud model migration boundaries explicitly. Signed-off-by: zhichao-poper --- .env.example | 2 +- README.md | 2 +- deploy/coolify/docker-compose.yml | 8 +++++ deploy/runtime/worker-compose.yaml | 8 +++++ docker-compose.yml | 8 +++++ src/cli/compose-project.ts | 10 +++++++ src/providers/embedding/local.ts | 8 +++-- src/state/embedding-identity.ts | 15 ++++++++-- test/compose-project.test.ts | 21 +++++++++++++- test/deploy-runtime.test.ts | 11 +++++++ test/embedding-identity.test.ts | 45 +++++++++++++++++++++++++++-- test/local-embedding-config.test.ts | 14 +++++++-- 12 files changed, 138 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 7e0831b80c..0d00c88bcb 100644 --- a/.env.example +++ b/.env.example @@ -102,7 +102,7 @@ # LOCAL_EMBEDDING_DOCUMENT_PREFIX= # Exact prefix, including trailing spaces # LOCAL_EMBEDDING_CACHE_DIR= # Optional pipeline cache directory # LOCAL_EMBEDDING_BATCH_SIZE= # Optional positive batch limit -# Model, dimensions, pooling, dtype or document-prefix changes require new vectors. +# Model, dimensions, pooling, dtype, document-prefix or cache-path changes require new vectors. # Unidentified older local indexes require an explicit rebuild too: back up the # store, set AGENTMEMORY_DROP_STALE_INDEX=true for one restart, then unset it. # This removes only derived vectors. Set AGENTMEMORY_VECTOR_BACKFILL=all diff --git a/README.md b/README.md index 8dda5a064e..1804aa5191 100644 --- a/README.md +++ b/README.md @@ -1114,7 +1114,7 @@ Local embeddings ship as an optional dependency but are not enabled by default. Local Transformers.js models can be selected with `LOCAL_EMBEDDING_MODEL`, `LOCAL_EMBEDDING_DIMENSIONS`, `LOCAL_EMBEDDING_POOLING` and `LOCAL_EMBEDDING_DTYPE` (defaults: the model above, `384`, `mean`, `q8`). Optional `LOCAL_EMBEDDING_QUERY_PREFIX` and `LOCAL_EMBEDDING_DOCUMENT_PREFIX` preserve exact prefixes, including trailing spaces. `LOCAL_EMBEDDING_CACHE_DIR` selects the pipeline cache and `LOCAL_EMBEDDING_BATCH_SIZE` bounds extraction batches. Use settings and a Transformers.js export compatible with the selected model. -Startup records the local document-embedding configuration before generating vectors. Changing the model, dimensions, pooling, dtype or document prefix refuses restoration of both checkpoints and pending vectors, even when dimensions match. Older local indexes without this identity also require an explicit rebuild: back up the store, set `AGENTMEMORY_DROP_STALE_INDEX=true` for one restart, then unset it. This removes derived vectors while retaining memories and observations. Set `AGENTMEMORY_VECTOR_BACKFILL=all` to run the full backfill; otherwise it waits for explicit opt-in. Query-prefix, cache-directory and batch-size changes do not require document vectors to be rebuilt. +Startup records the local document-embedding configuration before generating vectors. Changing the model, dimensions, pooling, dtype, document prefix or configured cache directory refuses restoration of both checkpoints and pending vectors, even when dimensions match. Older local indexes without this identity also require an explicit rebuild: back up the store, set `AGENTMEMORY_DROP_STALE_INDEX=true` for one restart, then unset it. This removes derived vectors while retaining memories and observations. Set `AGENTMEMORY_VECTOR_BACKFILL=all` to run the full backfill; otherwise it waits for explicit opt-in. Query-prefix and batch-size changes do not require document vectors to be rebuilt. A cache-directory change conservatively requires rebuilding because it may select different artifacts under the same model ID. This configuration identity does not fingerprint weight files or pin mutable upstream model revisions. For Docker, use a cache path inside the container, such as `/data/models`, and mount or persist that location. ### Cost-aware model selection diff --git a/deploy/coolify/docker-compose.yml b/deploy/coolify/docker-compose.yml index e35233996f..c305ad9ce0 100644 --- a/deploy/coolify/docker-compose.yml +++ b/deploy/coolify/docker-compose.yml @@ -15,6 +15,14 @@ services: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:3113,127.0.0.1:3113,[::1]:3113}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:3113,http://127.0.0.1:3113}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + LOCAL_EMBEDDING_MODEL: "${LOCAL_EMBEDDING_MODEL:-}" + LOCAL_EMBEDDING_DIMENSIONS: "${LOCAL_EMBEDDING_DIMENSIONS:-}" + LOCAL_EMBEDDING_POOLING: "${LOCAL_EMBEDDING_POOLING:-}" + LOCAL_EMBEDDING_DTYPE: "${LOCAL_EMBEDDING_DTYPE:-}" + LOCAL_EMBEDDING_QUERY_PREFIX: "${LOCAL_EMBEDDING_QUERY_PREFIX:-}" + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${LOCAL_EMBEDDING_DOCUMENT_PREFIX:-}" + LOCAL_EMBEDDING_CACHE_DIR: "${LOCAL_EMBEDDING_CACHE_DIR:-}" + LOCAL_EMBEDDING_BATCH_SIZE: "${LOCAL_EMBEDDING_BATCH_SIZE:-}" AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" diff --git a/deploy/runtime/worker-compose.yaml b/deploy/runtime/worker-compose.yaml index d3f8b91308..f7abf6e974 100644 --- a/deploy/runtime/worker-compose.yaml +++ b/deploy/runtime/worker-compose.yaml @@ -74,6 +74,14 @@ containers: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:3113,127.0.0.1:3113,[::1]:3113}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:3113,http://127.0.0.1:3113}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + LOCAL_EMBEDDING_MODEL: "${LOCAL_EMBEDDING_MODEL:-}" + LOCAL_EMBEDDING_DIMENSIONS: "${LOCAL_EMBEDDING_DIMENSIONS:-}" + LOCAL_EMBEDDING_POOLING: "${LOCAL_EMBEDDING_POOLING:-}" + LOCAL_EMBEDDING_DTYPE: "${LOCAL_EMBEDDING_DTYPE:-}" + LOCAL_EMBEDDING_QUERY_PREFIX: "${LOCAL_EMBEDDING_QUERY_PREFIX:-}" + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${LOCAL_EMBEDDING_DOCUMENT_PREFIX:-}" + LOCAL_EMBEDDING_CACHE_DIR: "${LOCAL_EMBEDDING_CACHE_DIR:-}" + LOCAL_EMBEDDING_BATCH_SIZE: "${LOCAL_EMBEDDING_BATCH_SIZE:-}" AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" diff --git a/docker-compose.yml b/docker-compose.yml index 0fd899e20e..67e78b9993 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,14 @@ services: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:${AGENTMEMORY_VIEWER_PORT:-3113},127.0.0.1:${AGENTMEMORY_VIEWER_PORT:-3113},[::1]:${AGENTMEMORY_VIEWER_PORT:-3113}}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:${AGENTMEMORY_VIEWER_PORT:-3113},http://127.0.0.1:${AGENTMEMORY_VIEWER_PORT:-3113}}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + LOCAL_EMBEDDING_MODEL: "${LOCAL_EMBEDDING_MODEL:-}" + LOCAL_EMBEDDING_DIMENSIONS: "${LOCAL_EMBEDDING_DIMENSIONS:-}" + LOCAL_EMBEDDING_POOLING: "${LOCAL_EMBEDDING_POOLING:-}" + LOCAL_EMBEDDING_DTYPE: "${LOCAL_EMBEDDING_DTYPE:-}" + LOCAL_EMBEDDING_QUERY_PREFIX: "${LOCAL_EMBEDDING_QUERY_PREFIX:-}" + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${LOCAL_EMBEDDING_DOCUMENT_PREFIX:-}" + LOCAL_EMBEDDING_CACHE_DIR: "${LOCAL_EMBEDDING_CACHE_DIR:-}" + LOCAL_EMBEDDING_BATCH_SIZE: "${LOCAL_EMBEDDING_BATCH_SIZE:-}" AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 300f8caf7b..9381552a8a 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -170,10 +170,14 @@ function windowsWorkerPath(nodeBin: string, inherited: string | undefined): stri }).join(";"); } +/** Compose reparses YAML; literal C1 characters can be rejected or silently folded. */ function yamlString(value: string): string { if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value)) { throw new Error("A Compose path or configuration value contains an unsupported control character."); } + if (/[\r\n\u0080-\u009f]/.test(value)) { + return JSON.stringify(value).replace(/[\u0080-\u009f]/g, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`); + } return `'${value.replace(/'/g, "''")}'`; } @@ -365,6 +369,12 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { if (filteredEnvFile) lines.push(` env_file: [${yamlString(filteredEnvFile)}]`); const explicitWorkerEnvironment = Object.entries(options.workerEnvironment ?? {}) .filter(([key, value]) => value !== undefined && AGENTMEMORY_WORKER_ENV_ALLOWLIST.has(key) && !COMPOSE_RESERVED_ENV.has(key)) + .map(([key, value]) => { + const path = windows ? win32 : { isAbsolute, resolve }; + return [key, key === "LOCAL_EMBEDDING_CACHE_DIR" && value && !path.isAbsolute(value) + ? path.resolve(options.cwd ?? process.cwd(), value) + : value] as const; + }) .sort(([left], [right]) => left.localeCompare(right)); lines.push( " environment:", diff --git a/src/providers/embedding/local.ts b/src/providers/embedding/local.ts index ba400d43ac..9459de4aaf 100644 --- a/src/providers/embedding/local.ts +++ b/src/providers/embedding/local.ts @@ -1,5 +1,6 @@ import type { EmbeddingProvider } from "../../types.js"; import type { DataType } from "@huggingface/transformers"; +import { resolve } from "node:path"; type FeatureExtractor = ( texts: string[], @@ -22,14 +23,15 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { readonly dimensions = positiveInteger("LOCAL_EMBEDDING_DIMENSIONS", 384); readonly pooling = process.env.LOCAL_EMBEDDING_POOLING?.trim() || "mean"; readonly dtype = (process.env.LOCAL_EMBEDDING_DTYPE?.trim() || "q8") as DataType; + readonly cacheDir = process.env.LOCAL_EMBEDDING_CACHE_DIR ? resolve(process.env.LOCAL_EMBEDDING_CACHE_DIR) : undefined; private readonly batchSize = positiveInteger("LOCAL_EMBEDDING_BATCH_SIZE", Number.MAX_SAFE_INTEGER); private readonly queryPrefix = process.env.LOCAL_EMBEDDING_QUERY_PREFIX || ""; private readonly documentPrefix = process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX || ""; private extractor: Promise | null = null; - /** Query instructions and loading options do not change stored document embeddings. */ + /** A different cache may contain different artifacts under the same model identifier. */ get indexIdentity(): string { - return JSON.stringify(["local-v1", this.model, this.dimensions, this.pooling, this.dtype, this.documentPrefix]); + return JSON.stringify(["local-v2", this.model, this.dimensions, this.pooling, this.dtype, this.documentPrefix, this.cacheDir ?? null]); } constructor() { @@ -96,7 +98,7 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { } return await transformers.pipeline("feature-extraction", this.model, { dtype: this.dtype, - ...(process.env.LOCAL_EMBEDDING_CACHE_DIR ? { cache_dir: process.env.LOCAL_EMBEDDING_CACHE_DIR } : {}), + ...(this.cacheDir ? { cache_dir: this.cacheDir } : {}), }) as FeatureExtractor; } } diff --git a/src/state/embedding-identity.ts b/src/state/embedding-identity.ts index e73a63847f..ae5a81a915 100644 --- a/src/state/embedding-identity.ts +++ b/src/state/embedding-identity.ts @@ -7,6 +7,15 @@ import { safeAudit } from "../functions/audit.js"; const IDENTITY_KEY = "vectors:embedding-identity"; const VECTOR_KEYS = ["vectors", "vectors:manifest", "vectors:meta"]; const VECTOR_SCOPE_PREFIXES = [`${KV.bm25Index}:vec:`, `${KV.bm25Index}:vectors:`]; +const DELETE_CONCURRENCY = 32; + +async function deleteKeys(kv: StateKV, scope: string, keys: string[]): Promise { + for (let offset = 0; offset < keys.length; offset += DELETE_CONCURRENCY) { + const results = await Promise.allSettled(keys.slice(offset, offset + DELETE_CONCURRENCY).map((key) => kv.delete(scope, key))); + const failed = results.find((result) => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; + } +} /** Refuse unverified vectors before startup can migrate checkpoints or replay pending writes. */ export async function ensureEmbeddingIdentity( @@ -37,10 +46,10 @@ export async function ensureEmbeddingIdentity( if (hasVectors) { await kv.set(KV.bm25Index, "vectors:backfill", { v: 2, since: null }); for (const scope of vectorScopes) { - for (const key of await kv.listKeys(scope)) await kv.delete(scope, key); + await deleteKeys(kv, scope, await kv.listKeys(scope)); } - for (const key of pending) await kv.delete(KV.vectorPendingLog, key); - for (const key of VECTOR_KEYS) await kv.delete(KV.bm25Index, key); + await deleteKeys(kv, KV.vectorPendingLog, pending); + await deleteKeys(kv, KV.bm25Index, VECTOR_KEYS); await safeAudit(kv, "index_persist", "mem::embedding-identity", [], { action: "rebuild", vectorScopes: vectorScopes.length, pendingEntries: pending.length, }); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index 7c9d1cda6d..c3ef695a4f 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -28,13 +28,32 @@ describe("Worker Compose project", () => { const settings = { LOCAL_EMBEDDING_MODEL: "test/model", LOCAL_EMBEDDING_DIMENSIONS: "3", LOCAL_EMBEDDING_POOLING: "cls", LOCAL_EMBEDDING_QUERY_PREFIX: "query: ", - LOCAL_EMBEDDING_DOCUMENT_PREFIX: "passage: ", LOCAL_EMBEDDING_CACHE_DIR: "model-cache", + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "passage: ", LOCAL_EMBEDDING_CACHE_DIR: join(root, "model-cache"), LOCAL_EMBEDDING_BATCH_SIZE: "2", LOCAL_EMBEDDING_DTYPE: "fp32", }; const app = parse(renderWorkerCompose(options({ workerEnvironment: settings }))).containers.agentmemory; expect(app.environment).toMatchObject(settings); }); + it("resolves a relative model cache against the CLI directory and escapes line breaks", () => { + const cwd = join(root, "project"); + const value = "model-cache\n雪"; + const text = renderWorkerCompose(options({ cwd, workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: value } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(cwd, value)); + expect(text).toContain("model-cache\\n雪"); + }); + + it("resolves Windows cache paths against the CLI directory", () => { + const text = renderWorkerCompose(options({ windows: true, cwd: "C:\\project", nodeBin: "C:\\node\\node.exe", workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "model-cache" } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe("C:\\project\\model-cache"); + }); + + it("preserves an absolute cache path when forwarding to the worker", () => { + const value = join(root, "cache"); + const text = renderWorkerCompose(options({ cwd: join(root, "other"), workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: value } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(value); + }); + it("pins registry workers and configures the isolated engine, state and HTTP", () => { const text = renderWorkerCompose(options()); expect(text).toContain("namespace: agentmemory-3511"); diff --git a/test/deploy-runtime.test.ts b/test/deploy-runtime.test.ts index d3468a45cf..99f931dcea 100644 --- a/test/deploy-runtime.test.ts +++ b/test/deploy-runtime.test.ts @@ -18,6 +18,17 @@ function fixture() { afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); describe("deployment entrypoint", () => { + it("forwards every local model setting through Docker and runtime Compose", () => { + const keys = ["MODEL", "DIMENSIONS", "POOLING", "DTYPE", "QUERY_PREFIX", "DOCUMENT_PREFIX", "CACHE_DIR", "BATCH_SIZE"]; + for (const file of ["docker-compose.yml", "deploy/coolify/docker-compose.yml", "deploy/runtime/worker-compose.yaml"]) { + const config = parse(readFileSync(file, "utf8")); + const env = file.endsWith("worker-compose.yaml") ? config.containers.agentmemory.environment : config.services.agentmemory.environment; + for (const key of keys) { + const name = `LOCAL_EMBEDDING_${key}`; + expect(env[name]).toBe(`\${${name}:-}`); + } + } + }); it.each(["2000", "500"])("generates file and Redis configurations with save interval %s", (interval) => { const root = fixture(); const program = entrypoint.split("node <<'NODE'\n")[1].split("\nNODE")[0] diff --git a/test/embedding-identity.test.ts b/test/embedding-identity.test.ts index e9be65e11c..7988356c38 100644 --- a/test/embedding-identity.test.ts +++ b/test/embedding-identity.test.ts @@ -52,7 +52,7 @@ describe("persisted embedding identity", () => { expect(restored.has("obs_pending")).toBe(true); }); - it.each([["MODEL", "other/model"], ["POOLING", "cls"], ["DTYPE", "fp32"], ["DOCUMENT_PREFIX", "passage: "]])( + it.each([["MODEL", "other/model"], ["POOLING", "cls"], ["DTYPE", "fp32"], ["DOCUMENT_PREFIX", "passage: "], ["CACHE_DIR", "other-cache"]])( "refuses same-dimension vectors after changing %s without deleting them", async (key, value) => { const kv = mockKV(); await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false); @@ -63,13 +63,12 @@ describe("persisted embedding identity", () => { }, ); - it("does not rebuild document vectors for query-prefix, batch or cache changes", async () => { + it("does not rebuild document vectors for query-prefix or batch changes", async () => { const kv = mockKV(); await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false); await kv.set(KV.bm25Index, "vectors:meta", { v: 3 }); vi.stubEnv("LOCAL_EMBEDDING_QUERY_PREFIX", "query: "); vi.stubEnv("LOCAL_EMBEDDING_BATCH_SIZE", "2"); - vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "cache"); await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).resolves.toBeUndefined(); }); @@ -104,4 +103,44 @@ describe("persisted embedding identity", () => { await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true)).rejects.toThrow("disk unavailable"); await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), false)).rejects.toThrow("unknown"); }); + + it("overlaps independent deletions with bounded concurrency and adopts identity only after they finish", async () => { + const kv = mockKV(); + for (let i = 0; i < 70; i++) await kv.set(KV.vectorPendingLog, `obs_${i}`, {}); + const remove = kv.delete; + let active = 0; + let peak = 0; + kv.delete = async (scope, key) => { + active++; peak = Math.max(peak, active); + expect(await kv.get(KV.bm25Index, "vectors:embedding-identity")).toBeNull(); + await new Promise((resolve) => setTimeout(resolve, 1)); + await remove(scope, key); + active--; + }; + await ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true); + expect(peak).toBeGreaterThan(1); + expect(peak).toBeLessThanOrEqual(32); + expect(active).toBe(0); + expect(await kv.listKeys(KV.vectorPendingLog)).toEqual([]); + expect(await kv.get(KV.bm25Index, "vectors:embedding-identity")).not.toBeNull(); + }); + + it("settles an in-flight deletion batch before surfacing an error", async () => { + const kv = mockKV(); + for (let i = 0; i < 40; i++) await kv.set(KV.vectorPendingLog, `obs_${i}`, {}); + const remove = kv.delete; + let active = 0; + kv.delete = async (scope, key) => { + active++; + try { + if (key === "obs_0") throw new Error("delete failed"); + await new Promise((resolve) => setTimeout(resolve, 1)); + await remove(scope, key); + } finally { active--; } + }; + await expect(ensureEmbeddingIdentity(kv as never, new LocalEmbeddingProvider(), true)).rejects.toThrow("delete failed"); + expect(active).toBe(0); + expect(await kv.get(KV.bm25Index, "vectors:embedding-identity")).toBeNull(); + expect(await kv.get(KV.vectorPendingLog, "obs_39")).not.toBeNull(); + }); }); diff --git a/test/local-embedding-config.test.ts b/test/local-embedding-config.test.ts index 1da0abdeec..3153f64218 100644 --- a/test/local-embedding-config.test.ts +++ b/test/local-embedding-config.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { LocalEmbeddingProvider } from "../src/providers/embedding/local.js"; import { withDimensionGuard } from "../src/providers/embedding/index.js"; +import { resolve } from "node:path"; const mock = vi.hoisted(() => ({ extractor: vi.fn(), pipeline: vi.fn() })); vi.mock("@huggingface/transformers", () => ({ pipeline: mock.pipeline })); beforeEach(() => { @@ -51,7 +52,7 @@ describe("local model configuration", () => { it("passes the configured cache directory to the pipeline", async () => { configure(); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "model-cache"); await new LocalEmbeddingProvider().embed("test"); - expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8", cache_dir: "model-cache" }); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8", cache_dir: resolve("model-cache") }); }); it("retries loading after a pipeline initialization failure", async () => { configure(); mock.pipeline.mockRejectedValueOnce(new Error("model unavailable")); @@ -63,7 +64,16 @@ describe("local model configuration", () => { it("passes the selected dtype with the cache directory", async () => { configure(); vi.stubEnv("LOCAL_EMBEDDING_DTYPE", "fp32"); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "model-cache"); await new LocalEmbeddingProvider().embed("test"); - expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "fp32", cache_dir: "model-cache" }); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "fp32", cache_dir: resolve("model-cache") }); + }); + it("loads from the same cache location used to establish its identity", async () => { + configure(); vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "first-cache"); + const provider = new LocalEmbeddingProvider(); + const identity = provider.indexIdentity; + vi.stubEnv("LOCAL_EMBEDDING_CACHE_DIR", "second-cache"); + expect(provider.indexIdentity).toBe(identity); + await provider.embed("test"); + expect(mock.pipeline).toHaveBeenCalledWith("feature-extraction", "test/chinese", { dtype: "q8", cache_dir: resolve("first-cache") }); }); it("rejects an unsupported dtype before loading", () => { vi.stubEnv("LOCAL_EMBEDDING_DTYPE", "bogus"); From 9796869014b14384eef75b702d6055850c31b700 Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Sun, 8 Nov 2026 21:32:37 +0800 Subject: [PATCH 3/9] fix(cli): normalize exported local model cache settings Read the effective cache value from the selected env file when it is not present in the CLI environment, including dotenv export syntax and quotes. Preserve explicit process-environment precedence and normalize only the cache setting before forwarding it to the worker. Signed-off-by: zhichao-poper --- src/cli/compose-project.ts | 10 ++++++++-- test/compose-project.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 9381552a8a..1c92634db4 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -1,6 +1,7 @@ import { chmodSync, existsSync, readFileSync, mkdirSync, renameSync, writeFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve, win32 } from "node:path"; import { parse, stringify } from "yaml"; +import { parse as parseDotenv } from "dotenv"; import { MAX_STARTUP_TIMEOUT_MS } from "./readiness.js"; import { getStateSaveIntervalMs } from "../config.js"; @@ -422,10 +423,15 @@ export function writeWorkerCompose(options: ComposeProjectOptions): { file: stri chmodSync(join(projectDir, "iii.worker.yaml"), 0o600); } } - const text = renderWorkerCompose(options); + const envFileContents = options.envFile ? readFileSync(options.envFile, "utf8") : ""; + const cacheDir = options.workerEnvironment?.LOCAL_EMBEDDING_CACHE_DIR ?? + (options.envFile ? parseDotenv(envFileContents).LOCAL_EMBEDDING_CACHE_DIR : undefined); + const text = renderWorkerCompose(cacheDir === undefined ? options : { + ...options, workerEnvironment: { ...options.workerEnvironment, LOCAL_EMBEDDING_CACHE_DIR: cacheDir }, + }); if (options.envFile) { const filteredPath = join(paths.directory, ".agentmemory-filtered.env"); - const entries = readFileSync(options.envFile, "utf8").split(/\r?\n/).filter((line) => { + const entries = envFileContents.split(/\r?\n/).filter((line) => { const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/); return !match || !COMPOSE_RESERVED_ENV.has(match[1]!); }); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index c3ef695a4f..54021ab886 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -54,6 +54,29 @@ describe("Worker Compose project", () => { expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(value); }); + it.each(["LOCAL_EMBEDDING_CACHE_DIR=model-cache", "export LOCAL_EMBEDDING_CACHE_DIR='model-cache with spaces'"])( + "resolves an effective cache value from the env file: %s", (line) => { + mkdirSync(root, { recursive: true }); + const envFile = join(root, "model.env"); + writeFileSync(envFile, `${line}\nUNRELATED_SETTING=kept-in-file\n`); + const cwd = join(root, "project"); + const result = writeWorkerCompose(options({ envFile, cwd, workerEnvironment: {} })); + const env = parse(result.text).containers.agentmemory.environment; + const relative = line.startsWith("export ") ? "model-cache with spaces" : "model-cache"; + expect(env.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(cwd, relative)); + expect(env).not.toHaveProperty("UNRELATED_SETTING"); + }, + ); + + it("keeps process-environment cache precedence over the env file", () => { + mkdirSync(root, { recursive: true }); + const envFile = join(root, "model.env"); + writeFileSync(envFile, "export LOCAL_EMBEDDING_CACHE_DIR=file-cache\n"); + const cwd = join(root, "project"); + const result = writeWorkerCompose(options({ envFile, cwd, workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "process-cache" } })); + expect(parse(result.text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(cwd, "process-cache")); + }); + it("pins registry workers and configures the isolated engine, state and HTTP", () => { const text = renderWorkerCompose(options()); expect(text).toContain("namespace: agentmemory-3511"); From e104f4c96fc0f4540ed7301764c41fdb317ebc54 Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Mon, 9 Nov 2026 01:06:44 +0800 Subject: [PATCH 4/9] fix(cli): preserve literal dollars in normalized model cache paths Escape normalized local cache paths for the pinned Compose interpolation pass. Preserve process-environment precedence and interpolation of unrelated settings. Add Windows and literal-dollar unit coverage plus env-file/process real-engine regressions, and include the live suite in the Linux release gate. The two live cases fail against the previous head and pass with this fix under iii 0.25.1. Validation: ordered build, skills generation, docs sync, skills check and full tests passed (4618 passed, 5 skipped), followed by docs and whitespace checks. Signed-off-by: zhichao-poper --- .github/workflows/release-gate.yml | 2 +- src/cli/compose-project.ts | 6 +- test/compose-cache-integration.test.ts | 100 +++++++++++++++++++++++++ test/compose-project.test.ts | 17 +++++ 4 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 test/compose-cache-integration.test.ts diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 7998c1ff56..f0fd683aa5 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -51,7 +51,7 @@ jobs: engine_pin=$(node --import tsx --input-type=module -e 'import { III_PINNED_VERSION } from "./src/version.ts"; console.log(III_PINNED_VERSION)') engine_bin="$HOME/.cache/agentmemory-release-gate/iii-${engine_pin}-linux-x64" test -x "$engine_bin" - AGENTMEMORY_TEST_III="$engine_bin" AGENTMEMORY_REQUIRE_LIVE_TESTS=1 npx vitest run test/compose-supervision-integration.test.ts test/codex-live-daemon.test.ts + AGENTMEMORY_TEST_III="$engine_bin" AGENTMEMORY_REQUIRE_LIVE_TESTS=1 npx vitest run test/compose-supervision-integration.test.ts test/codex-live-daemon.test.ts test/compose-cache-integration.test.ts - run: npm run test:startup:packaged -- --skip-build --out "${{ runner.temp }}/startup-gate" - name: Test packaged vector recovery shell: bash diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 1c92634db4..574b3a7d5e 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -371,10 +371,12 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { const explicitWorkerEnvironment = Object.entries(options.workerEnvironment ?? {}) .filter(([key, value]) => value !== undefined && AGENTMEMORY_WORKER_ENV_ALLOWLIST.has(key) && !COMPOSE_RESERVED_ENV.has(key)) .map(([key, value]) => { + if (key !== "LOCAL_EMBEDDING_CACHE_DIR") return [key, value] as const; const path = windows ? win32 : { isAbsolute, resolve }; - return [key, key === "LOCAL_EMBEDDING_CACHE_DIR" && value && !path.isAbsolute(value) + const cacheDir = value && !path.isAbsolute(value) ? path.resolve(options.cwd ?? process.cwd(), value) - : value] as const; + : value; + return [key, cacheDir?.replaceAll("$", () => "$$")] as const; }) .sort(([left], [right]) => left.localeCompare(right)); lines.push( diff --git a/test/compose-cache-integration.test.ts b/test/compose-cache-integration.test.ts new file mode 100644 index 0000000000..99230b41ff --- /dev/null +++ b/test/compose-cache-integration.test.ts @@ -0,0 +1,100 @@ +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { setTimeout as sleep } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import { composeNamespace, writeWorkerCompose } from "../src/cli/compose-project.js"; +import { III_PINNED_VERSION } from "../src/version.js"; + +const engine = process.env.AGENTMEMORY_TEST_III; +const version = engine ? spawnSync(engine, ["--version"], { encoding: "utf8", timeout: 3000 }) : null; +const available = version?.status === 0 && version.stdout.trim() === III_PINNED_VERSION; +const skipped = !available || process.platform === "win32" || process.env.AGENTMEMORY_SKIP_LIVE_TESTS === "1"; +if (process.env.AGENTMEMORY_REQUIRE_LIVE_TESTS === "1" && skipped) { + throw new Error(`Required Compose cache path test needs iii ${III_PINNED_VERSION} and live tests enabled on macOS or Linux`); +} + +async function freePort(): Promise { + const server = createServer(); + await new Promise((accept, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", accept); + }); + const port = (server.address() as { port: number }).port; + await new Promise((accept) => server.close(() => accept())); + return port; +} + +async function stop(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((accept) => child.once("exit", () => accept())); + child.kill("SIGINT"); + const result = await Promise.race([exited.then(() => true), sleep(5000).then(() => false)]); + if (!result && child.pid) { + try { process.kill(-child.pid, "SIGKILL"); } catch {} + await exited; + } +} + +describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { + it.each(["env-file", "process"])("preserves literal cache paths from %s through Compose interpolation", async (source) => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-compose-cache-")); + let child: ChildProcess | undefined; + try { + const pkg = join(root, "worker"); + const snapshot = join(root, "received.json"); + mkdirSync(join(pkg, "dist"), { recursive: true }); + const value = "model-cache/${HOME}/$MODELS/$$/models"; + const cwd = join(root, "caller$directory"); + const expected = join(cwd, value); + const envFile = join(root, "worker.env"); + writeFileSync(envFile, `export LOCAL_EMBEDDING_CACHE_DIR='${source === "env-file" ? value : "ignored-cache"}'\n`); + const sdk = pathToFileURL(createRequire(import.meta.url).resolve("iii-sdk")).href; + writeFileSync(join(pkg, "dist/index.mjs"), ` + import sdkModule from ${JSON.stringify(sdk)}; + import { renameSync, writeFileSync } from "node:fs"; + const { registerWorker } = sdkModule; + const worker = registerWorker(process.env.III_URL, { workerName: "agentmemory" }); + worker.registerFunction("test::env", async () => process.env.LOCAL_EMBEDDING_CACHE_DIR); + writeFileSync(${JSON.stringify(`${snapshot}.tmp`)}, JSON.stringify(process.env.LOCAL_EMBEDDING_CACHE_DIR)); + renameSync(${JSON.stringify(`${snapshot}.tmp`)}, ${JSON.stringify(snapshot)}); + `); + const ports = { restPort: await freePort(), viewerPort: await freePort(), enginePort: await freePort() }; + const project = writeWorkerCompose({ + ports, projectRoot: pkg, dataDir: join(root, "data"), homeDir: join(root, "home"), + nodeBin: process.execPath, cwd, envFile, + workerEnvironment: source === "process" ? { LOCAL_EMBEDDING_CACHE_DIR: value } : {}, + }); + const header = project.text.slice(0, project.text.indexOf("containers:\n")); + const app = project.text.slice(project.text.indexOf(" agentmemory:\n")) + .replace(" start_after: [state, http]", " start_after: []"); + writeFileSync(project.file, `${header}containers:\n${app}`); + child = spawn(engine!, ["compose", "--namespace", composeNamespace(ports.restPort), "--up", "--file", project.file], { + env: { PATH: process.env.PATH, HOME: join(root, "home"), USERPROFILE: join(root, "home"), + III_COMPOSE_STATE_DIR: project.stateDir, III_TELEMETRY_ENABLED: "false", III_BUNDLE_WORKERS_DISABLED: "1", RUST_LOG: "error" }, + detached: true, stdio: ["ignore", "ignore", "pipe"], + }); + let logs = ""; + child.stderr?.on("data", (chunk) => { logs = `${logs}${chunk}`.slice(-16000); }); + const deadline = Date.now() + 20000; + while (Date.now() < deadline) { + try { + expect(JSON.parse(readFileSync(snapshot, "utf8"))).toBe(expected); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (child.exitCode !== null || child.signalCode !== null) throw new Error(`Compose exited before the worker received its value: ${logs}`); + await sleep(50); + } + throw new Error(`Worker environment snapshot timed out: ${logs}`); + } finally { + if (child) await stop(child); + rmSync(root, { recursive: true, force: true }); + } + }, 30000); +}); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index 54021ab886..da40e3de26 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -43,6 +43,23 @@ describe("Worker Compose project", () => { expect(text).toContain("model-cache\\n雪"); }); + it("escapes dollar signs in normalized cache paths without changing other settings", () => { + const text = renderWorkerCompose(options({ cwd: join(root, "caller$directory"), workerEnvironment: { + LOCAL_EMBEDDING_CACHE_DIR: "${HOME}/$MODELS/$$/cache", + LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${DOCUMENT_PREFIX}", + } })); + const env = parse(text).containers.agentmemory.environment; + expect(env.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(root, "caller$$directory", "$${HOME}/$$MODELS/$$$$/cache")); + expect(env.LOCAL_EMBEDDING_DOCUMENT_PREFIX).toBe("${DOCUMENT_PREFIX}"); + }); + + it("escapes literal dollars in absolute Windows cache paths", () => { + const text = renderWorkerCompose(options({ windows: true, workerEnvironment: { + LOCAL_EMBEDDING_CACHE_DIR: "C:\\$models\\${HOME}\\cache", + } })); + expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe("C:\\$$models\\$${HOME}\\cache"); + }); + it("resolves Windows cache paths against the CLI directory", () => { const text = renderWorkerCompose(options({ windows: true, cwd: "C:\\project", nodeBin: "C:\\node\\node.exe", workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "model-cache" } })); expect(parse(text).containers.agentmemory.environment.LOCAL_EMBEDDING_CACHE_DIR).toBe("C:\\project\\model-cache"); From eb5ff02c1da41bcff7d06629341fe97a66a3016a Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Mon, 9 Nov 2026 01:57:17 +0800 Subject: [PATCH 5/9] fix(cli): preserve embedding prefixes and bound probe lifetimes Forward the newly introduced local embedding settings as literal process values through Compose interpolation, retaining cache path normalization. Preserve HOME references, undefined references, dollars and trailing spaces in both document and query prefixes. Document the native CLI contract. Give test probes a lifetime independent of Compose and its reaper. Always end that lifetime during cleanup, including when Compose has already exited, and verify the owned probe stops before removing its temporary directory. Validation: 31 focused checks passed, including real iii 0.25.1 prefix values and the already-exited-parent cleanup regression. All ordered build, skills, docs and full test gates passed (4619 passed, 5 skipped). Signed-off-by: zhichao-poper --- README.md | 2 +- src/cli/compose-project.ts | 6 +-- test/compose-cache-integration.test.ts | 41 ++++++++++--------- test/compose-probe-cleanup.test.ts | 40 +++++++++++++++++++ test/compose-project.test.ts | 10 ++++- test/helpers/compose-probe-lifetime.ts | 55 ++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 25 deletions(-) create mode 100644 test/compose-probe-cleanup.test.ts create mode 100644 test/helpers/compose-probe-lifetime.ts diff --git a/README.md b/README.md index 1804aa5191..81d41c80b5 100644 --- a/README.md +++ b/README.md @@ -1112,7 +1112,7 @@ Reasoning-class models (`o1`-style with `` blocks) can return empty `cont Local embeddings ship as an optional dependency but are not enabled by default. Set `EMBEDDING_PROVIDER=local` to opt into `Xenova/all-MiniLM-L6-v2` (384-dim). The first embedding request downloads the model; inference is on-device afterward. Without that setting or a remote embedding key, vectors stay disabled, `mem::search` uses BM25, and `smart-search` can still add existing graph matches. -Local Transformers.js models can be selected with `LOCAL_EMBEDDING_MODEL`, `LOCAL_EMBEDDING_DIMENSIONS`, `LOCAL_EMBEDDING_POOLING` and `LOCAL_EMBEDDING_DTYPE` (defaults: the model above, `384`, `mean`, `q8`). Optional `LOCAL_EMBEDDING_QUERY_PREFIX` and `LOCAL_EMBEDDING_DOCUMENT_PREFIX` preserve exact prefixes, including trailing spaces. `LOCAL_EMBEDDING_CACHE_DIR` selects the pipeline cache and `LOCAL_EMBEDDING_BATCH_SIZE` bounds extraction batches. Use settings and a Transformers.js export compatible with the selected model. +Local Transformers.js models can be selected with `LOCAL_EMBEDDING_MODEL`, `LOCAL_EMBEDDING_DIMENSIONS`, `LOCAL_EMBEDDING_POOLING` and `LOCAL_EMBEDDING_DTYPE` (defaults: the model above, `384`, `mean`, `q8`). Optional `LOCAL_EMBEDDING_QUERY_PREFIX` and `LOCAL_EMBEDDING_DOCUMENT_PREFIX` preserve exact prefixes, including trailing spaces. `LOCAL_EMBEDDING_CACHE_DIR` selects the pipeline cache and `LOCAL_EMBEDDING_BATCH_SIZE` bounds extraction batches. Use settings and a Transformers.js export compatible with the selected model. The native CLI preserves literal text in these values, including `$` and `${...}`; relative cache paths resolve from the CLI directory. Startup records the local document-embedding configuration before generating vectors. Changing the model, dimensions, pooling, dtype, document prefix or configured cache directory refuses restoration of both checkpoints and pending vectors, even when dimensions match. Older local indexes without this identity also require an explicit rebuild: back up the store, set `AGENTMEMORY_DROP_STALE_INDEX=true` for one restart, then unset it. This removes derived vectors while retaining memories and observations. Set `AGENTMEMORY_VECTOR_BACKFILL=all` to run the full backfill; otherwise it waits for explicit opt-in. Query-prefix and batch-size changes do not require document vectors to be rebuilt. A cache-directory change conservatively requires rebuilding because it may select different artifacts under the same model ID. This configuration identity does not fingerprint weight files or pin mutable upstream model revisions. For Docker, use a cache path inside the container, such as `/data/models`, and mount or persist that location. diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 574b3a7d5e..cf4f9c12f3 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -371,12 +371,12 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { const explicitWorkerEnvironment = Object.entries(options.workerEnvironment ?? {}) .filter(([key, value]) => value !== undefined && AGENTMEMORY_WORKER_ENV_ALLOWLIST.has(key) && !COMPOSE_RESERVED_ENV.has(key)) .map(([key, value]) => { - if (key !== "LOCAL_EMBEDDING_CACHE_DIR") return [key, value] as const; + if (!key.startsWith("LOCAL_EMBEDDING_")) return [key, value] as const; const path = windows ? win32 : { isAbsolute, resolve }; - const cacheDir = value && !path.isAbsolute(value) + const forwardedValue = key === "LOCAL_EMBEDDING_CACHE_DIR" && value && !path.isAbsolute(value) ? path.resolve(options.cwd ?? process.cwd(), value) : value; - return [key, cacheDir?.replaceAll("$", () => "$$")] as const; + return [key, forwardedValue?.replaceAll("$", () => "$$")] as const; }) .sort(([left], [right]) => left.localeCompare(right)); lines.push( diff --git a/test/compose-cache-integration.test.ts b/test/compose-cache-integration.test.ts index 99230b41ff..e6ab1383a4 100644 --- a/test/compose-cache-integration.test.ts +++ b/test/compose-cache-integration.test.ts @@ -9,6 +9,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { describe, expect, it } from "vitest"; import { composeNamespace, writeWorkerCompose } from "../src/cli/compose-project.js"; import { III_PINNED_VERSION } from "../src/version.js"; +import { prepareProbe, probeLifetime, stopProbe } from "./helpers/compose-probe-lifetime.js"; const engine = process.env.AGENTMEMORY_TEST_III; const version = engine ? spawnSync(engine, ["--version"], { encoding: "utf8", timeout: 3000 }) : null; @@ -29,45 +30,47 @@ async function freePort(): Promise { return port; } -async function stop(child: ChildProcess): Promise { - if (child.exitCode !== null || child.signalCode !== null) return; - const exited = new Promise((accept) => child.once("exit", () => accept())); - child.kill("SIGINT"); - const result = await Promise.race([exited.then(() => true), sleep(5000).then(() => false)]); - if (!result && child.pid) { - try { process.kill(-child.pid, "SIGKILL"); } catch {} - await exited; - } -} describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { it.each(["env-file", "process"])("preserves literal cache paths from %s through Compose interpolation", async (source) => { const root = mkdtempSync(join(tmpdir(), "agentmemory-compose-cache-")); let child: ChildProcess | undefined; try { + prepareProbe(root); const pkg = join(root, "worker"); const snapshot = join(root, "received.json"); mkdirSync(join(pkg, "dist"), { recursive: true }); const value = "model-cache/${HOME}/$MODELS/$$/models"; const cwd = join(root, "caller$directory"); - const expected = join(cwd, value); + const expected = { + cache: join(cwd, value), + document: source === "env-file" ? "passage: ${HOME} $$ " : "passage: ${AGENTMEMORY_TEST_UNDEFINED} $$ ", + query: "query: ${HOME} $$ ", + }; const envFile = join(root, "worker.env"); writeFileSync(envFile, `export LOCAL_EMBEDDING_CACHE_DIR='${source === "env-file" ? value : "ignored-cache"}'\n`); const sdk = pathToFileURL(createRequire(import.meta.url).resolve("iii-sdk")).href; writeFileSync(join(pkg, "dist/index.mjs"), ` - import sdkModule from ${JSON.stringify(sdk)}; - import { renameSync, writeFileSync } from "node:fs"; + import { existsSync, renameSync, writeFileSync } from "node:fs"; + ${probeLifetime(root)} + const { default: sdkModule } = await import(${JSON.stringify(sdk)}); const { registerWorker } = sdkModule; const worker = registerWorker(process.env.III_URL, { workerName: "agentmemory" }); - worker.registerFunction("test::env", async () => process.env.LOCAL_EMBEDDING_CACHE_DIR); - writeFileSync(${JSON.stringify(`${snapshot}.tmp`)}, JSON.stringify(process.env.LOCAL_EMBEDDING_CACHE_DIR)); + const received = { cache: process.env.LOCAL_EMBEDDING_CACHE_DIR, + document: process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX, query: process.env.LOCAL_EMBEDDING_QUERY_PREFIX }; + worker.registerFunction("test::env", async () => received); + writeFileSync(${JSON.stringify(`${snapshot}.tmp`)}, JSON.stringify(received)); renameSync(${JSON.stringify(`${snapshot}.tmp`)}, ${JSON.stringify(snapshot)}); `); const ports = { restPort: await freePort(), viewerPort: await freePort(), enginePort: await freePort() }; const project = writeWorkerCompose({ ports, projectRoot: pkg, dataDir: join(root, "data"), homeDir: join(root, "home"), nodeBin: process.execPath, cwd, envFile, - workerEnvironment: source === "process" ? { LOCAL_EMBEDDING_CACHE_DIR: value } : {}, + workerEnvironment: { + ...(source === "process" ? { LOCAL_EMBEDDING_CACHE_DIR: value } : {}), + LOCAL_EMBEDDING_DOCUMENT_PREFIX: expected.document, + LOCAL_EMBEDDING_QUERY_PREFIX: expected.query, + }, }); const header = project.text.slice(0, project.text.indexOf("containers:\n")); const app = project.text.slice(project.text.indexOf(" agentmemory:\n")) @@ -83,7 +86,7 @@ describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { const deadline = Date.now() + 20000; while (Date.now() < deadline) { try { - expect(JSON.parse(readFileSync(snapshot, "utf8"))).toBe(expected); + expect(JSON.parse(readFileSync(snapshot, "utf8"))).toEqual(expected); return; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; @@ -93,8 +96,8 @@ describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { } throw new Error(`Worker environment snapshot timed out: ${logs}`); } finally { - if (child) await stop(child); - rmSync(root, { recursive: true, force: true }); + try { await stopProbe(child, root); } + finally { rmSync(root, { recursive: true, force: true }); } } }, 30000); }); diff --git a/test/compose-probe-cleanup.test.ts b/test/compose-probe-cleanup.test.ts new file mode 100644 index 0000000000..dfc56aee93 --- /dev/null +++ b/test/compose-probe-cleanup.test.ts @@ -0,0 +1,40 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { expect, it } from "vitest"; +import { prepareProbe, probeLifetime, stopProbe } from "./helpers/compose-probe-lifetime.js"; + +it("stops the owned probe when its Compose parent has already exited", async () => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-probe-cleanup-")); + let parent: ChildProcess | undefined; + let probe: ChildProcess | undefined; + try { + prepareProbe(root); + parent = spawn(process.execPath, ["-e", "process.exit(0)"], { stdio: "ignore" }); + await new Promise((accept) => parent!.once("exit", () => accept())); + probe = spawn(process.execPath, ["--input-type=module", "-e", ` + import { existsSync, writeFileSync } from "node:fs"; + ${probeLifetime(root)} + `], { stdio: "ignore" }); + const deadline = Date.now() + 5000; + while (!existsSync(join(root, "probe-pid"))) { + if (Date.now() >= deadline) throw new Error("Test probe did not start"); + await sleep(20); + } + await stopProbe(parent, root); + expect(parent.exitCode).toBe(0); + expect(probe.exitCode).toBe(0); + } finally { + try { await stopProbe(parent, root); } + finally { + if (probe && probe.exitCode === null && probe.signalCode === null) { + const exited = new Promise((accept) => probe!.once("exit", () => accept())); + probe.kill("SIGKILL"); + await exited; + } + rmSync(root, { recursive: true, force: true }); + } + } +}, 15000); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index da40e3de26..03d0bb5789 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -43,14 +43,20 @@ describe("Worker Compose project", () => { expect(text).toContain("model-cache\\n雪"); }); - it("escapes dollar signs in normalized cache paths without changing other settings", () => { + it("preserves literal local embedding values without changing legacy interpolation", () => { const text = renderWorkerCompose(options({ cwd: join(root, "caller$directory"), workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "${HOME}/$MODELS/$$/cache", LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${DOCUMENT_PREFIX}", + LOCAL_EMBEDDING_QUERY_PREFIX: "query: ${HOME} $$ ", + LOCAL_EMBEDDING_MODEL: "models/${MODEL}", + OPENAI_BASE_URL: "${ENDPOINT}", } })); const env = parse(text).containers.agentmemory.environment; expect(env.LOCAL_EMBEDDING_CACHE_DIR).toBe(join(root, "caller$$directory", "$${HOME}/$$MODELS/$$$$/cache")); - expect(env.LOCAL_EMBEDDING_DOCUMENT_PREFIX).toBe("${DOCUMENT_PREFIX}"); + expect(env.LOCAL_EMBEDDING_DOCUMENT_PREFIX).toBe("$${DOCUMENT_PREFIX}"); + expect(env.LOCAL_EMBEDDING_QUERY_PREFIX).toBe("query: $${HOME} $$$$ "); + expect(env.LOCAL_EMBEDDING_MODEL).toBe("models/$${MODEL}"); + expect(env.OPENAI_BASE_URL).toBe("${ENDPOINT}"); }); it("escapes literal dollars in absolute Windows cache paths", () => { diff --git a/test/helpers/compose-probe-lifetime.ts b/test/helpers/compose-probe-lifetime.ts new file mode 100644 index 0000000000..dcd7e210c2 --- /dev/null +++ b/test/helpers/compose-probe-lifetime.ts @@ -0,0 +1,55 @@ +import type { ChildProcess } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +export function prepareProbe(root: string): void { + writeFileSync(join(root, "probe-active"), "active"); +} + +export function probeLifetime(root: string): string { + // The probe must stop even when Compose exits without a working reaper. + return ` + setInterval(() => { + if (!existsSync(${JSON.stringify(join(root, "probe-active"))})) process.exit(0); + }, 50); + writeFileSync(${JSON.stringify(join(root, "probe-pid"))}, String(process.pid)); + if (!existsSync(${JSON.stringify(join(root, "probe-active"))})) process.exit(0); + `; +} + +function running(pid: number): boolean { + try { process.kill(pid, 0); return true; } + catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } +} + +export async function stopProbe(child: ChildProcess | undefined, root: string): Promise { + rmSync(join(root, "probe-active"), { force: true }); + try { + if (child && child.exitCode === null && child.signalCode === null) { + const exited = new Promise((accept) => child.once("exit", () => accept())); + child.kill("SIGINT"); + const ended = await Promise.race([exited.then(() => true), sleep(5000).then(() => false)]); + if (!ended && child.pid) { + try { process.kill(-child.pid, "SIGKILL"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } + await exited; + } + } + } finally { + let pid: number | undefined; + try { pid = Number(readFileSync(join(root, "probe-pid"), "utf8")); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (pid !== undefined) { + if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid test probe pid"); + const deadline = Date.now() + 5000; + while (running(pid)) { + if (Date.now() >= deadline) throw new Error("Test probe did not exit after its lifetime ended"); + await sleep(50); + } + } + } +} From d5190c21f91cfeb6731f0c197489ff564a678351 Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Mon, 9 Nov 2026 07:50:52 +0800 Subject: [PATCH 6/9] fix(cli): preserve forwarded environment values and cleanup errors Treat allowlisted process environment values as literals through Compose, including non-local settings, while retaining cache path normalization. Document quoted env-file query/passage prefixes so trailing spaces survive. Collect cleanup failures outside finally blocks. Preserve a single original error and report combined termination/timeout failures with AggregateError, while continuing each cleanup stage. Add isolated diagnostics regressions. Validation: previous-head negative cases reproduce non-local interpolation and suppressed termination errors. All 34 focused checks and ordered build, skills, docs and full-test gates passed (4622 passed, 5 skipped). Signed-off-by: zhichao-poper --- .env.example | 5 ++-- README.md | 11 +++++++- src/cli/compose-project.ts | 1 - test/compose-cache-integration.test.ts | 5 +++- test/compose-probe-cleanup.test.ts | 37 ++++++++++++++++++++++++-- test/compose-project.test.ts | 4 +-- test/helpers/compose-probe-lifetime.ts | 17 +++++++----- 7 files changed, 65 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 0d00c88bcb..5fff7da292 100644 --- a/.env.example +++ b/.env.example @@ -98,8 +98,9 @@ # LOCAL_EMBEDDING_DIMENSIONS=384 # Set explicitly for a different model width # LOCAL_EMBEDDING_POOLING=mean # mean | cls | first_token | last_token | eos # LOCAL_EMBEDDING_DTYPE=q8 # Transformers.js dtype, e.g. q8 or fp32 -# LOCAL_EMBEDDING_QUERY_PREFIX= # Exact prefix, including trailing spaces -# LOCAL_EMBEDDING_DOCUMENT_PREFIX= # Exact prefix, including trailing spaces +# Optional exact prefixes. For E5-style inputs, quote env-file values to retain spaces: +# LOCAL_EMBEDDING_QUERY_PREFIX="query: " +# LOCAL_EMBEDDING_DOCUMENT_PREFIX="passage: " # LOCAL_EMBEDDING_CACHE_DIR= # Optional pipeline cache directory # LOCAL_EMBEDDING_BATCH_SIZE= # Optional positive batch limit # Model, dimensions, pooling, dtype, document-prefix or cache-path changes require new vectors. diff --git a/README.md b/README.md index 81d41c80b5..a65e0d1fd9 100644 --- a/README.md +++ b/README.md @@ -1112,7 +1112,16 @@ Reasoning-class models (`o1`-style with `` blocks) can return empty `cont Local embeddings ship as an optional dependency but are not enabled by default. Set `EMBEDDING_PROVIDER=local` to opt into `Xenova/all-MiniLM-L6-v2` (384-dim). The first embedding request downloads the model; inference is on-device afterward. Without that setting or a remote embedding key, vectors stay disabled, `mem::search` uses BM25, and `smart-search` can still add existing graph matches. -Local Transformers.js models can be selected with `LOCAL_EMBEDDING_MODEL`, `LOCAL_EMBEDDING_DIMENSIONS`, `LOCAL_EMBEDDING_POOLING` and `LOCAL_EMBEDDING_DTYPE` (defaults: the model above, `384`, `mean`, `q8`). Optional `LOCAL_EMBEDDING_QUERY_PREFIX` and `LOCAL_EMBEDDING_DOCUMENT_PREFIX` preserve exact prefixes, including trailing spaces. `LOCAL_EMBEDDING_CACHE_DIR` selects the pipeline cache and `LOCAL_EMBEDDING_BATCH_SIZE` bounds extraction batches. Use settings and a Transformers.js export compatible with the selected model. The native CLI preserves literal text in these values, including `$` and `${...}`; relative cache paths resolve from the CLI directory. +Local Transformers.js models can be selected with `LOCAL_EMBEDDING_MODEL`, `LOCAL_EMBEDDING_DIMENSIONS`, `LOCAL_EMBEDDING_POOLING` and `LOCAL_EMBEDDING_DTYPE` (defaults: the model above, `384`, `mean`, `q8`). Optional `LOCAL_EMBEDDING_QUERY_PREFIX` and `LOCAL_EMBEDDING_DOCUMENT_PREFIX` preserve exact prefixes, including trailing spaces. `LOCAL_EMBEDDING_CACHE_DIR` selects the pipeline cache and `LOCAL_EMBEDDING_BATCH_SIZE` bounds extraction batches. Use settings and a Transformers.js export compatible with the selected model. The native CLI forwards process environment values literally, including `$` and `${...}`; relative cache paths resolve from the CLI directory. + +In `~/.agentmemory/.env`, quote prefixes that require trailing spaces. For E5-style inputs, for example: + +```dotenv +LOCAL_EMBEDDING_QUERY_PREFIX="query: " +LOCAL_EMBEDDING_DOCUMENT_PREFIX="passage: " +``` + +Unquoted env-file values are trimmed before the provider reads them. Startup records the local document-embedding configuration before generating vectors. Changing the model, dimensions, pooling, dtype, document prefix or configured cache directory refuses restoration of both checkpoints and pending vectors, even when dimensions match. Older local indexes without this identity also require an explicit rebuild: back up the store, set `AGENTMEMORY_DROP_STALE_INDEX=true` for one restart, then unset it. This removes derived vectors while retaining memories and observations. Set `AGENTMEMORY_VECTOR_BACKFILL=all` to run the full backfill; otherwise it waits for explicit opt-in. Query-prefix and batch-size changes do not require document vectors to be rebuilt. A cache-directory change conservatively requires rebuilding because it may select different artifacts under the same model ID. This configuration identity does not fingerprint weight files or pin mutable upstream model revisions. For Docker, use a cache path inside the container, such as `/data/models`, and mount or persist that location. diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index cf4f9c12f3..8eaa7d94f3 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -371,7 +371,6 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { const explicitWorkerEnvironment = Object.entries(options.workerEnvironment ?? {}) .filter(([key, value]) => value !== undefined && AGENTMEMORY_WORKER_ENV_ALLOWLIST.has(key) && !COMPOSE_RESERVED_ENV.has(key)) .map(([key, value]) => { - if (!key.startsWith("LOCAL_EMBEDDING_")) return [key, value] as const; const path = windows ? win32 : { isAbsolute, resolve }; const forwardedValue = key === "LOCAL_EMBEDDING_CACHE_DIR" && value && !path.isAbsolute(value) ? path.resolve(options.cwd ?? process.cwd(), value) diff --git a/test/compose-cache-integration.test.ts b/test/compose-cache-integration.test.ts index e6ab1383a4..e991c71c18 100644 --- a/test/compose-cache-integration.test.ts +++ b/test/compose-cache-integration.test.ts @@ -46,6 +46,7 @@ describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { cache: join(cwd, value), document: source === "env-file" ? "passage: ${HOME} $$ " : "passage: ${AGENTMEMORY_TEST_UNDEFINED} $$ ", query: "query: ${HOME} $$ ", + endpoint: "https://literal.example/${AGENTMEMORY_TEST_UNDEFINED}/$$", }; const envFile = join(root, "worker.env"); writeFileSync(envFile, `export LOCAL_EMBEDDING_CACHE_DIR='${source === "env-file" ? value : "ignored-cache"}'\n`); @@ -57,7 +58,8 @@ describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { const { registerWorker } = sdkModule; const worker = registerWorker(process.env.III_URL, { workerName: "agentmemory" }); const received = { cache: process.env.LOCAL_EMBEDDING_CACHE_DIR, - document: process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX, query: process.env.LOCAL_EMBEDDING_QUERY_PREFIX }; + document: process.env.LOCAL_EMBEDDING_DOCUMENT_PREFIX, query: process.env.LOCAL_EMBEDDING_QUERY_PREFIX, + endpoint: process.env.OPENAI_BASE_URL }; worker.registerFunction("test::env", async () => received); writeFileSync(${JSON.stringify(`${snapshot}.tmp`)}, JSON.stringify(received)); renameSync(${JSON.stringify(`${snapshot}.tmp`)}, ${JSON.stringify(snapshot)}); @@ -70,6 +72,7 @@ describe.skipIf(skipped)("pinned Compose literal cache path round trip", () => { ...(source === "process" ? { LOCAL_EMBEDDING_CACHE_DIR: value } : {}), LOCAL_EMBEDDING_DOCUMENT_PREFIX: expected.document, LOCAL_EMBEDDING_QUERY_PREFIX: expected.query, + OPENAI_BASE_URL: expected.endpoint, }, }); const header = project.text.slice(0, project.text.indexOf("containers:\n")); diff --git a/test/compose-probe-cleanup.test.ts b/test/compose-probe-cleanup.test.ts index dfc56aee93..4da4ca2882 100644 --- a/test/compose-probe-cleanup.test.ts +++ b/test/compose-probe-cleanup.test.ts @@ -1,9 +1,10 @@ import { spawn, type ChildProcess } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; +import { EventEmitter } from "node:events"; import { prepareProbe, probeLifetime, stopProbe } from "./helpers/compose-probe-lifetime.js"; it("stops the owned probe when its Compose parent has already exited", async () => { @@ -38,3 +39,35 @@ it("stops the owned probe when its Compose parent has already exited", async () } } }, 15000); + + +it.each(["parent", "probe", "both"])("preserves cleanup diagnostics: %s", async (failure) => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-probe-errors-")); + const killFailure = Object.assign(new Error("process-group kill denied"), { code: "EPERM" }); + const child = Object.assign(new EventEmitter(), { + exitCode: null, signalCode: null, pid: 123456, kill: vi.fn(() => true), + }) as unknown as ChildProcess; + const kill = vi.spyOn(process, "kill").mockImplementation((pid) => { + if (pid < 0) throw killFailure; + if (failure === "parent") throw Object.assign(new Error("probe already exited"), { code: "ESRCH" }); + return true; + }); + try { + prepareProbe(root); + writeFileSync(join(root, "probe-pid"), "123456"); + let caught: unknown; + try { await stopProbe(failure === "probe" ? undefined : child, root, 1); } + catch (error) { caught = error; } + if (failure === "parent") expect(caught).toBe(killFailure); + else if (failure === "probe") expect(caught).toHaveProperty("message", "Test probe did not exit after its lifetime ended"); + else { + expect(caught).toBeInstanceOf(AggregateError); + expect((caught as AggregateError).errors).toEqual([ + killFailure, expect.objectContaining({ message: "Test probe did not exit after its lifetime ended" }), + ]); + } + } finally { + kill.mockRestore(); + rmSync(root, { recursive: true, force: true }); + } +}, 15000); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index 03d0bb5789..421bee513a 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -43,7 +43,7 @@ describe("Worker Compose project", () => { expect(text).toContain("model-cache\\n雪"); }); - it("preserves literal local embedding values without changing legacy interpolation", () => { + it("preserves literal process environment values after cache path normalization", () => { const text = renderWorkerCompose(options({ cwd: join(root, "caller$directory"), workerEnvironment: { LOCAL_EMBEDDING_CACHE_DIR: "${HOME}/$MODELS/$$/cache", LOCAL_EMBEDDING_DOCUMENT_PREFIX: "${DOCUMENT_PREFIX}", @@ -56,7 +56,7 @@ describe("Worker Compose project", () => { expect(env.LOCAL_EMBEDDING_DOCUMENT_PREFIX).toBe("$${DOCUMENT_PREFIX}"); expect(env.LOCAL_EMBEDDING_QUERY_PREFIX).toBe("query: $${HOME} $$$$ "); expect(env.LOCAL_EMBEDDING_MODEL).toBe("models/$${MODEL}"); - expect(env.OPENAI_BASE_URL).toBe("${ENDPOINT}"); + expect(env.OPENAI_BASE_URL).toBe("$${ENDPOINT}"); }); it("escapes literal dollars in absolute Windows cache paths", () => { diff --git a/test/helpers/compose-probe-lifetime.ts b/test/helpers/compose-probe-lifetime.ts index dcd7e210c2..5675c6c364 100644 --- a/test/helpers/compose-probe-lifetime.ts +++ b/test/helpers/compose-probe-lifetime.ts @@ -26,30 +26,35 @@ function running(pid: number): boolean { } } -export async function stopProbe(child: ChildProcess | undefined, root: string): Promise { - rmSync(join(root, "probe-active"), { force: true }); +export async function stopProbe(child: ChildProcess | undefined, root: string, timeoutMs = 5000): Promise { + const errors: unknown[] = []; + try { rmSync(join(root, "probe-active"), { force: true }); } + catch (error) { errors.push(error); } try { if (child && child.exitCode === null && child.signalCode === null) { const exited = new Promise((accept) => child.once("exit", () => accept())); child.kill("SIGINT"); - const ended = await Promise.race([exited.then(() => true), sleep(5000).then(() => false)]); + const ended = await Promise.race([exited.then(() => true), sleep(timeoutMs).then(() => false)]); if (!ended && child.pid) { try { process.kill(-child.pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } await exited; } } - } finally { + } catch (error) { errors.push(error); } + try { let pid: number | undefined; try { pid = Number(readFileSync(join(root, "probe-pid"), "utf8")); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } if (pid !== undefined) { if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid test probe pid"); - const deadline = Date.now() + 5000; + const deadline = Date.now() + timeoutMs; while (running(pid)) { if (Date.now() >= deadline) throw new Error("Test probe did not exit after its lifetime ended"); await sleep(50); } } - } + } catch (error) { errors.push(error); } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, "Compose and probe cleanup failed"); } From c71fee0804d434b03bda8a9f60d79a9659f66149 Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Tue, 10 Nov 2026 13:01:16 +0800 Subject: [PATCH 7/9] test: allow engine timeout before fixture entrypoint Signed-off-by: zhichao-poper --- scripts/release-gate/startup.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/release-gate/startup.mjs b/scripts/release-gate/startup.mjs index e9bfe0f0c4..5062235ef6 100644 --- a/scripts/release-gate/startup.mjs +++ b/scripts/release-gate/startup.mjs @@ -414,8 +414,17 @@ async function main() { const binary = join(i.runtime, "bin/iii"); const launchPid = join(i.dir, "launched-compose.pid"); writeFileSync(binary, `#!${process.execPath}\nconst {writeFileSync}=require('node:fs');\nif(process.argv.includes('--version')) { console.log('iii ${installed.dependencies["iii-sdk"]}'); process.exit(0); }\nwriteFileSync(${JSON.stringify(launchPid)},String(process.pid));\nsetInterval(()=>{},1000);\n`); chmodSync(binary, 0o755); Object.assign(d, await expectFailure(i, [], { AGENTMEMORY_ENGINE_READY_TIMEOUT_MS: "300" }, { message: "The engine process started but the REST API never responded." })); - const pid = Number(readFileSync(launchPid, "utf8")); i.ownedPids.add(pid); await poll("owned Compose engine timeout cleanup", () => !alive(pid), 5_000); + const composePid = Number(i.cli.text.match(/Compose daemon started \(pid (\d+)\)/)?.[1]); + assert(Number.isSafeInteger(composePid) && composePid > 0, `No owned Compose daemon was started: ${i.cli.text}`); + i.ownedPids.add(composePid); + // The 300ms readiness budget can expire before the daemon launches iii on + // a busy runner. That is still a valid timeout; always check the daemon, + // and check its engine child too when the fixture reached its entrypoint. + const pid = existsSync(launchPid) ? Number(readFileSync(launchPid, "utf8")) : null; + if (pid !== null) { assert(Number.isSafeInteger(pid) && pid > 0); i.ownedPids.add(pid); } + await poll("owned Compose engine timeout cleanup", () => !alive(composePid) && (pid === null || !alive(pid)), 5_000); assert(!existsSync(join(i.runtime, "engine-state.json"))); + d.engineEntrypointReached = pid !== null; d.transport = "version-compatible iii fixture accepting compose but never opening the engine port; actual installed CLI"; }); await scenario("cancel-owned-startup", async (i, d) => { From 0524a391801a7b2da002fe1ad6deab53aafd6b09 Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Fri, 13 Nov 2026 11:27:01 +0800 Subject: [PATCH 8/9] test: give bulk source import a dedicated timeout Signed-off-by: zhichao-poper --- test/observation-source.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/observation-source.test.ts b/test/observation-source.test.ts index 79d2018de1..68ea34d967 100644 --- a/test/observation-source.test.ts +++ b/test/observation-source.test.ts @@ -277,7 +277,8 @@ describe("retained observation source", () => { expect(await fresh.call("mem::import", { exportData: exported })).toMatchObject({ success: true, observations: 1000, sourceOmitted: 1000 }); expect(await fresh.kv.get(KV.observations(template.sessionId), first[0].id)).toEqual(first[0]); expect(payloadByteLength({ result: await fresh.kv.list(KV.observations(template.sessionId)) })).toBeLessThan(SAFE_PAYLOAD_BYTES); - }); + // Two thousand source-heavy imports can exceed the default budget on busy CI runners. + }, 30_000); it("rejects an oversized replacement before deleting or writing any stored records", async () => { const r = rig(); From 5c6e6cb3281ccc0471179b85a28f1cc61a2e6531 Mon Sep 17 00:00:00 2001 From: zhichao-poper Date: Fri, 13 Nov 2026 11:44:33 +0800 Subject: [PATCH 9/9] test: bound process and bulk fixture setup time Signed-off-by: zhichao-poper --- test/cli-lifecycle-safety.test.ts | 3 ++- test/connect-pi.test.ts | 3 ++- test/graph.test.ts | 3 ++- test/observation-source.test.ts | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/cli-lifecycle-safety.test.ts b/test/cli-lifecycle-safety.test.ts index 3992b2c5f0..28dc4b1f3a 100644 --- a/test/cli-lifecycle-safety.test.ts +++ b/test/cli-lifecycle-safety.test.ts @@ -153,7 +153,8 @@ describe("native removal shutdown", () => { expect(existsSync(result.enginePidfile)).toBe(true); expect(existsSync(result.engineState)).toBe(true); expect(existsSync(result.privateBin)).toBe(true); - }); + // Starting the isolated CLI can exceed the default budget on busy runners. + }, 30_000); it("refuses an instance-scoped global uninstall before deleting shared assets", () => { const { result, privateBin } = runInstanceRemove(); diff --git a/test/connect-pi.test.ts b/test/connect-pi.test.ts index d38ba7109b..3b09e61e39 100644 --- a/test/connect-pi.test.ts +++ b/test/connect-pi.test.ts @@ -135,5 +135,6 @@ describe("Pi package archive", () => { expect(readFileSync(join(root, "LICENSE"), "utf8")).toBe(readFileSync("LICENSE", "utf8")); expect(JSON.parse(readFileSync("package.json", "utf8")).files).toContain("integrations/pi/"); } finally { rmSync(dir, { recursive: true, force: true }); } - }); + // This checks archive contents, not npm/tar wall-clock performance. + }, 30_000); }); diff --git a/test/graph.test.ts b/test/graph.test.ts index 1556a59023..23d1d695da 100644 --- a/test/graph.test.ts +++ b/test/graph.test.ts @@ -256,7 +256,8 @@ describe("Graph Functions", () => { expect(unbounded.limit).toBe(500); expect(unbounded.offset).toBe(0); expect(new Set(unbounded.nodes.map(node => node.id)).size).toBe(500); - }); + // Seeding and snapshotting 1200 nodes is a functional check, not a 5s benchmark. + }, 30_000); it("honors limit and offset for paged graph-query traversal", async () => { for (let i = 0; i < 50; i++) { diff --git a/test/observation-source.test.ts b/test/observation-source.test.ts index 68ea34d967..b23bbc4ba7 100644 --- a/test/observation-source.test.ts +++ b/test/observation-source.test.ts @@ -318,7 +318,8 @@ describe("retained observation source", () => { expect(rows).toHaveLength(1200); expect(rows.reduce((sum, row) => sum + (row.source ? payloadByteLength(row.source) + 10 : 0), 0)).toBeLessThanOrEqual(SESSION_SOURCE_MAX_BYTES); expect(payloadByteLength({ result: rows })).toBeLessThan(SAFE_PAYLOAD_BYTES); - }); + // Both source-heavy batches must complete before checking the shared budget. + }, 30_000); it("refuses a replay result that expands beyond the transport frame", async () => { const r = rig();