diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6c4ef9a..71c96e20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,10 @@ jobs: sdk: runs-on: ubuntu-24.04 services: + qdrant: + image: qdrant/qdrant:v1.20.0 + ports: + - 16333:6333 postgres: image: postgres:17 env: @@ -43,6 +47,7 @@ jobs: --health-timeout 5s --health-retries 10 env: + HY_MEMORY_TEST_QDRANT_URL: http://127.0.0.1:16333 HY_MEMORY_TEST_POSTGRES_DSN: postgresql://hy_memory:hy_memory_test@localhost:5432/hy_memory_test steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -52,7 +57,14 @@ jobs: with: python-version: "3.12" enable-cache: true - - run: uv sync --extra dev --extra faiss --extra qdrant --extra postgres + - run: uv sync --locked --extra dev --extra qdrant-native + - name: Verify native install without FastEmbed + run: uv run --no-sync python -c "import importlib.util; assert importlib.util.find_spec('fastembed') is None" + - name: Native contracts + run: uv run --no-sync pytest tests/native_retrieval -o addopts= -m "not integration" -q -o log_cli=false + - name: Native Qdrant Server integration + run: uv run --no-sync pytest tests/native_retrieval -o addopts= -m integration -q -o log_cli=false + - run: uv sync --locked --extra dev --extra faiss --extra qdrant --extra postgres - name: Test Chroma content updates with an isolated HTTP server run: uv run pytest tests/test_chroma_content_update.py -q -o log_cli=false -m "integration or not integration" - run: uv run pytest tests/ -q -o log_cli=false diff --git a/README.md b/README.md index 0ceb6f5c..d818bd1c 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,8 @@ That version is usually not in the working tree or at HEAD either, so it cannot ### Documentation +- [Qdrant native retrieval](docs/native_retrieval.md) — Server-only text/BM25 and hybrid recall, explicit preparation and validation + - [Usage guide](docs/usage.md) — full configuration reference, API details, deployment modes - [Environment variables](docs/env_reference.md) — settings and defaults - [Search and Reader guide](docs/hy_memory_search_skill.md) — reader selection, schema quotas, rerank, and return channels diff --git a/README.zh.md b/README.zh.md index f92086a4..562016ae 100644 --- a/README.zh.md +++ b/README.zh.md @@ -253,6 +253,8 @@ Coding 与 Working **复用同一套双层 pipeline 和数据 schema**,区别 ### 文档 +- [Qdrant 原生召回](docs/native_retrieval.md) — 仅 Server 的正文/标签 BM25 与混合召回、显式准备及验收 + - [使用指南](docs/usage.md) — 完整配置参考、API 详情、部署模式 - [环境变量参考](docs/env_reference.md) — 配置说明与默认值 - [Search 与 Reader 指南](docs/hy_memory_search_skill.md) — Reader 选择、Schema 配额、Rerank 与返回通道 diff --git a/docs/env_reference.md b/docs/env_reference.md index a10a08b6..20b8fa6d 100644 --- a/docs/env_reference.md +++ b/docs/env_reference.md @@ -154,7 +154,7 @@ Catalog 和 supplement 只由 `user_id` 确定,不作为普通 L6 出现在 Se | 变量 | 默认值 | 说明 | |---|---|---| -| `HY_MEMORY_READER` | `legacy` | `legacy`(向量)/ `hybrid`(向量 + 后端关键词候选) | +| `HY_MEMORY_READER` | `legacy` | `legacy`(向量)/ `hybrid`(向量 + 后端关键词候选)/ `native`(仅 Qdrant Server 1.20.0,显式准备) | | `MEMORY_HYBRID_V2_VDB_WEIGHT_BM25` | `0.35` | Hybrid BM25 加成权重;当前仍保留旧前缀 | | `MEMORY_LOCAL_RERANK_ENABLED` | `false` | 两个 Reader 共用的本地 cross-encoder Rerank | | `MEMORY_LOCAL_RERANK_MODEL` | `nreimers/mmarco-mMiniLMv2-L6-H384-v1` | 小型多语言 Hugging Face 模型名或本地路径 | @@ -244,3 +244,5 @@ client = HyMemoryClient.from_config({ 历史配置不会生效;当前支持项以本页表格和 `.env.example` 为准。Cache backend 当前仅支持 `sqlite` 和 `mysql`。 + +Native reader 的 rerank、schema/intention 扩展与旧评分配置不适用;写侧独立设置 recall.reconcile_retriever。安装、版本、准备、typed threshold 与失败合同见 [Qdrant native retrieval](native_retrieval.md)。 diff --git a/docs/native_retrieval.md b/docs/native_retrieval.md new file mode 100644 index 00000000..f70ac3e7 --- /dev/null +++ b/docs/native_retrieval.md @@ -0,0 +1,155 @@ +# Qdrant Server native retrieval + +Native retrieval lets content and tags recall memories outside the dense +candidate pool, in public search and write-time reconcile. Text scoring and +fusion run entirely in Qdrant; the SDK only validates scope, resolves complete +evolution chains and formats results in backend order. + +Related to [#432](https://github.com/Tencent-Hunyuan/Hy-Memory/pull/432). This +retains its recall goal and replaces the sidecar approach with Qdrant-native +capabilities. The new path supports **Qdrant Server only**. Local Qdrant, +Chroma, FAISS, MemHub integration and the separate Chroma vector-preservation +fix are outside this feature's scope. + +## Install and prepare + +Use Python >=3.10, `qdrant-client==1.20.0` and Qdrant Server `1.20.0`. +Other combinations fail with `reason="unsupported"`. Base SDK Python 3.9 +support and the existing `qdrant`/`all` extras remain available. The unified +lock also upgrades those extras' client to 1.20.0 on Python >=3.10; Python 3.9 +retains 1.16.1. FastEmbed is optional for this new path. + +```sh +pip install 'hy-memory[qdrant-native]' +``` + +Configure the existing Qdrant remote connection and cosine collection: + +```python +from hy_memory import HyMemoryClient, NativeScoreThreshold, NativeRetrievalError + +client = HyMemoryClient.from_config({ + "vector_store": { + "provider": "qdrant", "host": "localhost", "port": 6333, + "collection_name": "memories", "embedding_dims": 1536, + }, + "recall": {"reconcile_retriever": "native"}, # independent write opt-in +}) +prepared = client.prepare_native_retrieval() +# Async: prepared = await client.async_prepare_native_retrieval() +``` + +Preparation builds scored TEXT indexes on `content` and `tags[]` (multilingual +tokenizer, lowercase, default BM25), plus a KEYWORD index on `tags`. It waits +for completed operations and checks schema and a read-only text/RRF probe. +Existing payloads are indexed without being re-upserted. It never drops an +incompatible index: `schema_conflict` identifies the conflicting field. +Interrupted preparation can be retried against the actual schema. Search +never creates indexes; missing indexes produce `not_ready`. A copied or +re-embedded collection needs the same explicit preparation. + +## Search + +```python +result = await client.async_search( + "ECONNRESET src/network/reconnect.py", + user_ids=["alice"], scene="chat", reader="native", + native_mode="hybrid", limit=10, +) +# The synchronous equivalent is client.search(...). +``` + +Modes are `dense`, `keyword` and `hybrid`. Keyword uses the original final +query without embedding. Dense uses the semantic rewrite. Qdrant fuses +content/tags BM25 with native RRF, then fuses the lexical subtree with dense +for hybrid. `queries=[...]` remains context plus a final target, not a batch +search API. `HY_MEMORY_READER=native` selects the same reader by default; +an explicit reader wins without modifying the process environment. + +Native defaults: no legacy `min_score`, `schema_limit=0`, +`intention_limit=0`, `rerank=False`. Explicit float `min_score`, positive +schema/intention limits and rerank are rejected. Omitted legacy arguments +keep their original defaults (0.4, 10, 0, True); explicit zero stays zero. + +Public limit is 0..100; the candidate budget is `max(30, 3*limit)` (max 300). +Blank queries and zero limit skip retrieval after parameter/scope validation. +Native supports pro/ultra chat L2/L3/L4 only. Lite, L1_RAW, `scene="all"`, +working/coding (including automatic routing) fail before entering an old path. + +Each result carries the **unrounded** backend `score`, `score_kind` +(`cosine` for dense, `rrf` otherwise), `mode`, `config_id`, `matched_id` +and representative `memory_id`. Scores are not normalized or reranked. +An optional exclusive raw-score threshold must match all three identifiers: + +```python +threshold = NativeScoreThreshold( + value=0.2, mode="hybrid", score_kind="rrf", + config_id=prepared["config_id"], +) +result = client.search( + "ECONNRESET", user_ids=["alice"], scene="chat", reader="native", + native_mode="hybrid", native_threshold=threshold, +) +``` + +Only scores strictly **greater than** the threshold pass. Equal scores are +excluded. RRF scores are ranking evidence, not cosine or confidence. + +User scope intersects agent/session/isolation conditions before every branch's +top-B. Source exclusion wins, including the existing mixed-source rule. +Evolution may reach a newer head in another agent/session of the same user, +but every node is checked for user/source/layer visibility. Complete components +must have a unique terminal node, no directed cycle, at most 32 hops and 1024 +unique nodes. A live head must be confirmed by `is_latest=True`. A soft-deleted +SHADOW terminal may have `is_latest=False`; its entire validated component is +omitted from search and reconcile, including recalled ancestors. Reciprocal +references and multi-ancestor merges are supported. +Time filtering applies to the final representative. Missing required nodes, +short batches and query failures fail closed; healthy no-hit queries succeed. +Native public searches schedule no dedup, SHADOW or access-count writes. + +## Reconcile and failures + +`recall.reconcile_retriever="native"` is independent of read selection; +the default `"default"` keeps existing retrieval. Unknown values are rejected +both on config loading and before candidate preparation. + +One native nested plan fuses up to 16 new memories and, when the existing +expansion switch is enabled, at most 4 extra queries. Branch/unified budget is +60; at most 20 resolved representatives enter the existing prompt in backend +order. Reconcile retains L2/L4 by default and performs no candidate-phase +storage dedup. Necessary failures return `success=False, ops=[]`; cancellation +propagates. A healthy empty candidate pool still permits the existing ADD path. + +`candidate_scores` contains backend **cosine** evidence restricted to the +resolved candidate IDs. RRF never enters semantic-neighbor scoring. Failure of +only this optional evidence empties that input's semantic neighbors and emits +a diagnostic; normal writes and LLM typed relations remain available. + +`NativeRetrievalError.reason` values: `unsupported`, `invalid_request`, +`not_ready`, `schema_conflict`, `backend`, `integrity`, `scope`, `cycle`, `budget`. +There is no automatic fallback, parallel lexical database or Python-side fusion. +Disabling the read/write opt-ins restores the existing paths; existing +`search_text`/sparse writes continue. Rollback does not drop/rebuild indexes. + +## Validation + +Use a dedicated test Server, never an application database. Fixtures create +unique collections and delete only their own collection. Missing URL, +unreachable Server or wrong version fails selected integration tests. + +```sh +uv sync --locked --python 3.12 --extra dev --extra qdrant-native +docker run --rm -d --name hy-memory-native-test -p 127.0.0.1:16333:6333 qdrant/qdrant:v1.20.0 +uv run --no-sync pytest tests/native_retrieval -o addopts= -m 'not integration' -q +HY_MEMORY_TEST_QDRANT_URL=http://127.0.0.1:16333 uv run --no-sync pytest tests/native_retrieval -o addopts= -m integration -q +``` + +PowerShell: set `$env:HY_MEMORY_TEST_QDRANT_URL='http://127.0.0.1:16333'` +before the integration command. CI separately tests native without FastEmbed, +then restores full extras for the existing SDK suite. + +The new dependency is Apache-2.0 qdrant-client. Fixed capability sources: +[Server 1.20.0](https://github.com/qdrant/qdrant/releases/tag/v1.20.0), +[client 1.20.0](https://github.com/qdrant/qdrant-client/releases/tag/v1.20.0). +Version documentation describes available APIs; it does not certify this SDK. diff --git a/docs/plans/2026-11-12-qdrant-native-execution.md b/docs/plans/2026-11-12-qdrant-native-execution.md new file mode 100644 index 00000000..f1ced281 --- /dev/null +++ b/docs/plans/2026-11-12-qdrant-native-execution.md @@ -0,0 +1,383 @@ +# Qdrant native retrieval — local execution record + +Date: 2026-11-12, Asia/Shanghai. Local implementation and applicable validation +are complete. The complete SDK suite is **not green on this Windows host**; +baseline comparisons and installation limitations are recorded below. No +Issue, PR, draft, remote comment, push, or reopening of #432 was performed. + +## Branch, scope and material preservation + +- Branch: `codex/qdrant-native-retrieval`. +- Fixed upstream base: `01ccf50f72694390e5979065b5396ed4835cccd0` (`origin/main` + fetched at the start). The previous Chroma branch is an ancestor of this + upstream base through merged #438. No private Chroma history is added to the + new branch's diff. +- Implementation commits: `9e91c33`, `fbd616a`; subsequent local validation + commits preserve the resolved dependency versions and add this record. +- The three original acceptance/server/client materials remain untracked and + unchanged. SHA-256 comparisons against backups in the handoff directory's + `preserved-before-implementation/` all returned true. The supplied handoff + directory and ZIP were not changed. +- Only two production modules were added: `data/qdrant_native.py` and + `pipelines/reader_native.py`. No MemHub, Chroma adapter, writer, graph + projector or re-embed implementation was changed. +- One evidence-driven scope addition: `utils/model_call_context.py` receives + only `from __future__ import annotations`. The upstream file evaluates + `str | None` during import and fails on real Python 3.9.12. The declaration + restores import so the legacy route works and native can explicitly reject + Python 3.9. `test_reader_selection.py` also recognizes Python 3.13 class + metadata and registers the new reader. + +## Executed environment + +Windows, PowerShell, `uv 0.11.15`. Successfully tested: + +| Environment | Actual result | +|---|---| +| Python 3.13.13 + client 1.20.0 + Server 1.20.0 | Native contracts and real Server tests pass | +| Same versions, native-only extra | `find_spec('fastembed') is None`; native integration passes | +| Python 3.13.13, dev/faiss/qdrant/postgres extras | 221 focused legacy/graph/reconcile regressions pass; full SDK suite executed | +| Real Python 3.9.12, minimal import/test dependencies | SDK import and explicit native `unsupported`; 7 focused legacy/default/version tests pass | +| Planned full Python 3.12 native install | Not completed: network/DNS/download failures, then disk-full extraction; no Python 3.12 test pass claimed | +| Full Python 3.9 extra installation | Not completed: required wheels absent from offline cache; minimal tests do not certify the complete extra | + +Server image: `qdrant/qdrant:v1.20.0`, digest +`sha256:9bfc89f9914fc14876a88c05270685921a55b8877b254a98bcc1cc2f0a424cd2`. +The actual Server reported commit +`222b2e44ab1c3f17e8246396ead6558e45041df7`. +Only the dedicated `http://127.0.0.1:16333` endpoint was used. Every fixture +owns a random `hy_native_test__2` collection and cleans only that name. +The test Server is disposable; no application database was selected. + +The lock resolves client 1.16.1 for Python 3.9 and 1.20.0 for Python >=3.10, +including the existing `qdrant` and `all` extras. No other dependency version +was upgraded. The project lock metadata also catches up from 2.0.13 to the +checkout's existing 2.0.16. Unrelated lock-format churn from the older local +uv writer was removed while retaining its resolved project/client blocks; +`uv lock --check --offline` and locked native sync both pass. Relative to the +base, the final lock diff is 18 additions and 14 deletions. + +## Commands and results + +The successful native environments were created with: + +```powershell +$env:UV_PROJECT_ENVIRONMENT="$env:TEMP\hy-memory-native-py313-pure" +uv sync --offline --locked --python 3.13 --extra dev --extra qdrant-native +& "$env:UV_PROJECT_ENVIRONMENT\Scripts\python.exe" -c "import importlib.util; assert importlib.util.find_spec('fastembed') is None" +$env:HY_MEMORY_TEST_QDRANT_URL='http://127.0.0.1:16333' +$env:PYTHONUTF8='1' +& "$env:UV_PROJECT_ENVIRONMENT\Scripts\python.exe" -m pytest tests/native_retrieval -o addopts= -q -o log_cli=false --tb=short +``` + +The full-directory run at `fbd616a` plus the version-identical lock cleanup +passed **66 tests**, zero failed/skipped, in 226.44 seconds. The subsequently +added real Server concurrent native/legacy test passed separately, giving +**67 distinct native tests** (41 local contracts, 26 integration tests). +An earlier no-FastEmbed full-directory run passed 62 tests before the final +budget/error/real-chain cases were added. These are separate runs, not a +claim that 67 ran in one invocation. + +Focused regression command (same file set as the acceptance plan): + +```powershell +& "$env:TEMP\hy-memory-native-py313\Scripts\python.exe" -m pytest ` + tests/test_reader_selection.py tests/test_hybrid_min_score_gate.py ` + tests/test_search_channel_bucketing.py tests/test_search_layer_filter.py ` + tests/test_search_rerank_mmr_gates.py tests/test_source_filtering.py ` + tests/test_evolution_chain_shadow_filter.py tests/coding/test_search_scene_routing.py ` + tests/test_reconcile_failure.py tests/test_reconcile_candidate_user_scope.py ` + tests/test_reconcile_chain_fold.py tests/test_reconcile_chain_merge.py ` + tests/test_reconcile_time_order.py tests/test_reconcile_hybrid_v2_align.py ` + tests/test_memory_graph.py tests/test_qdrant_batch_search.py -q -o log_cli=false +``` + +Result: **221 passed, 4 skipped** (unconfigured Neo4j), exit 0, 18.68 seconds. +The earlier run had the Python 3.13 metadata assertion failure; it was fixed +and is not hidden in the final count. + +Real Python 3.9.12 ran `test_reader_selection.py` plus +`test_T18_python39_and_version_rejection` and +`test_T18_legacy_omitted_and_explicit_zero`: **7 passed**, exit 0. This used a +minimal environment installed with `uv pip`, not a successful full SDK-extra +sync. The actual import/native rejection script also passed. + +## Acceptance mapping + +All rows below have executed passing coverage. Coverage can span the native +suite and focused compatibility tests; it is not a claim that every possible +combination of failure modes was exercised against a live Server. + +| ID | Passing evidence | +|---|---| +| T01 | 35 dense distractors; explicit dense top-30 excludes target; public keyword/hybrid include it; actual reconcile prompt includes it | +| T02 | Tags-only target in public results and actual reconcile prompt | +| T03 | Real scored `tags[]` TEXT plus `tags` KEYWORD under strict mode; exact `foo` excludes `foo bar`, text retrieves both | +| T04 | Agentic rewrite removes error code; keyword still uses original text, with an embedding stub that raises if invoked | +| T05 | User union, pre-limit exclusion, mixed-source rule, isolation/user intersection and include/exclude conflict | +| T06 | Historical match resolves a newer same-user head in another agent; distinct matched/display IDs | +| T07 | Real Server missing-head/cycle/multi-ancestor cases; reciprocal links, final-head time boundaries, scope and budget contracts | +| T08 | Raw unrounded score, exclusive equality, mismatched typed threshold rejection; real red/green mutation below | +| T09 | Content/tags update and clear; single and metadata-range deletion | +| T10 | Existing data indexed without upsert; repeated/interrupted prepare; missing schema and preserved incompatible index | +| T11 | Payloads/vectors and task set unchanged; dedup scheduling forbidden; strength writes enabled in config but bypassed | +| T12 | Necessary timeout/auth/error contracts, short embeddings, ID/nonfinite/incomplete response handling and cancellation; reconcile has no operations | +| T13 | Actual prompt follows backend order despite opposite timestamps | +| T14 | Healthy empty Server candidate pool allows existing ADD behavior | +| T15 | Real cosine 0 vs 1 in both ranking directions; writer stores main memory and projects only the true cosine neighbor | +| T16 | Optional cosine failure preserves stored main write and typed `causes` edge; semantic neighbors absent | +| T17 | Concurrent native/legacy actual Server public requests keep mode/user scope; no environment mutation; invalid reconcile configuration rejected | +| T18 | Legacy omitted defaults and explicit zero, actual Python 3.9 import/rejection, old reader execution with upgraded client; full installation gaps above remain open | + +LLM and embeddings are deterministic stubs. Writer graph projection uses a +recording graph sink. Qdrant is real in every integration case; no Local or +mock Qdrant result is counted as Server certification. + +## Failures found and corrected + +- The native text probe showed WORD does not match `烤鸭` inside + `用户喜欢北京烤鸭`; MULTILINGUAL does, for both content and tags. The native + schema/config identity and tests now use MULTILINGUAL. +- Server schema normalizes `scoring=true` into `{type: bm25}`; checks accept + that exact supported shape. A default cosine vector is checked before + preparation/query and optional cosine retrieval. +- Qdrant returns canonical hyphenated UUIDs even for hex input IDs. Integrity + checks now compare UUID values; initial false failures were not accepted. +- Empty/zero requests now skip before scene LLM/reader initialization. Directed + hop budget is checked before fetching another frontier. Pipeline validation + depends only on shared contracts. Standards/spec review findings were fixed; + the spec reviewer found no remaining material gap on re-review. +- A first mutation attempt had an absent test container, so both outcomes were + environment errors and were discarded. The Server was recreated at the same + image digest. Valid mutation: replacing `>` with `>=` made T08 fail its actual + equality assertion (exit 1); restoring `>` passed (exit 0). Source restored. +- A supplementary concurrency test first exposed an incomplete embedding stub; + `embed_with_meta` was added to the fixture. One later run passed the assertion + but failed writing pytest cache because the disk was full. The final repeat + with available disk and cache disabled exited 0. It is the counted result. + +## Full SDK suite: failed, baseline reproduced + +`python -m pytest tests/ -q -o log_cli=false --tb=short` was run with complete +dev/faiss/qdrant/postgres extras on Python 3.13.13: + +- Feature working tree: **1947 passed, 74 failed, 18 errors, 42 skipped, + 69 deselected**; exit 1, 405.44 seconds. +- Isolated archive of upstream `01ccf50`, same interpreter/dependencies: + **1910 passed, 73 failed, 18 errors, 42 skipped, 45 deselected**; exit 1, + 280.46 seconds. Import path was explicitly verified to use the archive. +- 90 failed/error test cases overlap exactly. They include Windows database + file locks, native FAISS file-handle exhaustion, encoding, signals and an + existing LLM pool failure. The upstream-only failure is the Python 3.13 + reader class metadata assertion fixed in this branch. +- The two feature-only full-suite failures are Chroma re-embed HNSW + `Nothing found on disk` cases. Both were subsequently reproduced on the + unchanged archive: 512→512 in a focused three-case run, and 512→1024 on the + third isolated repeat (first two passed). Their shared failure is not + attributed to native retrieval or silently removed from the full result. +- Skips include absent PostgreSQL/Neo4j services and spaCy models, existing + Chroma keyword exclusions, platform symlinks and an optional sparse test. + No native integration test skipped for missing environment/version. + +No unrelated backend/test-lifecycle overhaul is included to make this host's +entire suite green. CI was edited to run native without FastEmbed, explicitly +select real Server integration and then retain the full existing SDK suite. +Remote CI was not triggered or claimed to pass. + +## Evidence, privacy, dependency and rollback + +Raw local evidence is retained under `$env:TEMP`: +`hy-memory-native-final.{log,xml}`, `hy-memory-native-concurrent.xml`, +`hy-memory-native-contracts.{log,xml}`, `hy-memory-native-regression.{log,xml}`, +`hy-memory-native-full-sdk.{log,xml}`, `hy-memory-native-baseline-sdk.{log,xml}` +and `hy-memory-native-mutation.log`. Synthetic inputs only; no production +connection string, customer data or credential is included in this record. +Original material hashes were checked; no telemetry or external application +write was added. Ruff checks on new/affected retrieval paths, compileall and +`git diff --check` passed. Existing unrelated lint findings were not changed. + +The only changed third-party version is Apache-2.0 `qdrant-client` 1.20.0; +its existing transitive dependency versions remain unchanged. Support facts +come from fixed official release/source evidence; execution results above +are separate from those documents. + +Rollback selects `reader="legacy"`/`"hybrid"` and +`recall.reconcile_retriever="default"`. Existing search_text/sparse maintenance +continues. Rollback never automatically drops/rebuilds native indexes. No +rollback on a production deployment was performed. + +Future PR wording remains **Related to #432**, retaining its dense-pool escape +goal through Qdrant-native capabilities with a Server-only boundary. This +record is local evidence, not maintainer approval or a merge/CI certification. + +## PR #447 CI follow-up (2026-11-12) + +After this local record was written, the user authorized creating draft +[PR #447](https://github.com/Tencent-Hunyuan/Hy-Memory/pull/447) and then +parallel investigation/fixes for its failed CI. The draft remains one SDK +feature PR. Three original untracked handoff documents remain unchanged. + +The first PR run at `d19fd63` failed the SDK native-contract step and two +MemHub Core tests. To reproduce the actual merge context, upstream +`a2a1858` was merged as `bf9ad03`; its changes are already on upstream main +and do not add unrelated feature changes to the PR diff. + +Three agents separately investigated the SDK and the two Core failures: + +| Failure | Reproduced cause and minimal repair | Executed validation | +|---|---|---| +| Native backend TimeoutError cause identity | Python 3.12's standard-library Future conversion reconstructs `TimeoutError(*args)`; object identity is not the SDK error contract. `b781edd` preserves `reason=backend`, nonempty cause, exact cause type and args. No production change. | Original assertion: 1 failed/1 passed on real Python 3.12. Revised native error module: 41 passed on Python 3.12 and 41 passed on Python 3.13. | +| Retired read-client lease closure | The old test counted five `sleep(0)` turns while close runs in a worker thread. `1f04c0e` waits on the close callback's completion Event with a five-second bound and retains both exact closure-order assertions. | Real Linux/Python 3.12 isolated wrapper: delayed-close original 1 failed; same delayed seam repaired 1 passed; module 7 passed; controlled-delay repetition 20/20 passed. | +| Permission/live-sync trigger | A watcher can complete the import before an explicit automatic trigger, which then legitimately returns no job. `1f04c0e` checks actual watcher completion, source attribution, evidence count and the subsequent no-op; permission/freeze checks remain and task shutdown is checked. | Watcher-first original 1 failed after evidence count 2 was confirmed; repaired permission module 6 passed and target repeats passed twice. The proposed EOF-baseline explanation was tested and not supported. | + +The two repaired MemHub modules passed together: **13 passed**, exit 0, +13.56 seconds. These are test synchronization repairs only; no MemHub +production implementation, SDK/MemHub integration or new dependency was added. +Both standards and contract reviewers found no actionable issue in the three +test changes (`bf9ad03...1f04c0e`). + +Core local validation used a fresh dedicated Docker runner, Python 3.12.15, +Core 1.0.6 and the unchanged Core lock's SDK 2.0.16. Runtime/dev packages were +installed from a lock export with `--require-hashes`; locked `uv sync` then +completed. Checkout source was mounted read-only. Windows CRLF required a +container-only LF copy of `run-tests-isolated.sh`, with only its ROOT fixed +to `/workspace/memhub`; its real temporary HOME/data/config/ports and all +isolation checks were preserved. Reports use separate `/tmp` directories. +A read-only pytest cache warning in the combined run was not a test failure. + +The SDK job on [CI run 43221674616](https://github.com/Tencent-Hunyuan/Hy-Memory/actions/runs/43221674616/job/131194455491) +at `b781edd` completed successfully on Ubuntu/Python 3.12: + +- Native contracts: **41 passed**, 26 deselected. +- Native Qdrant Server integration, without FastEmbed: **26 passed**, 41 deselected. +- Chroma local/real-HTTP regressions: **53 passed**. +- Complete SDK suite: **2039 passed, 30 skipped, 71 deselected**. + +This provides real Linux/Python 3.12 evidence that was unavailable in the +earlier Windows installation attempts. It does not relabel those historical +Windows failures as passes. MemHub on that same run still failed the original +closure-timing assertion (4798 passed/1 failed), before `1f04c0e` was pushed. +Later current-head CI is tracked on the [PR checks page](https://github.com/Tencent-Hunyuan/Hy-Memory/pull/447/checks); +neither this prior successful SDK run nor local records replace a new head's +actual results. + +Additional local neighbor checks are recorded without hiding their failure: +`test_live_watch.py`, `test_live_sync_job_coalescing.py` and +`test_live_watch_grants.py` produced **28 passed/1 failed**. The failed, +unmodified test was `test_a_new_turn_lands_well_inside_the_ten_second_target` +(append not observed within ten seconds). That invocation did not run either +modified Core module; its specific cause was not established in this record. +No unrelated product change or weakened assertion was used to force it green. + +Diagnostic/reproduction reports remain in the dedicated runner under +`/tmp/hy-memory-pr447-reports/lifecycle-*`, `sync-*` and `combined`. +No raw diagnostic log or private application data was published. + +## PR #447 B1/B2 regressions (2026-11-12) + +Starting head: `6f18457`. Both reported failures were reproduced before the +production fixes, using synthetic users and temporary storage: + +- B1: `ReadRequest("query", "alice")` returned both alice and bob through the + real legacy reader and Chroma. The full historical positional constructor + also failed because native fields shifted its arguments. Native options + now follow every existing field, preserving all twenty legacy positions. +- B2: public native search returned `z-head, a-oldest, b-middle` for a + three-node chain. History now uses the existing `_time_key` descending + ordering (`memory_at`, falling back to `gmt_created`), with the validated + head retained first. Public-path tests cover populated, missing and mixed + `memory_at` values and retain matched ID and raw-score assertions. + +The focused red command was: + +```powershell +$testPython = Join-Path $env:TEMP 'hy-memory-native-py313\Scripts\python.exe' +& $testPython -m pytest tests/test_reader_selection.py tests/native_retrieval/test_errors.py -q -o log_cli=false -k 'positional or orders_history' +``` + +It produced **5 failed** before the fixes, including the exact cross-user +result and incorrect timeline above. After fixing the constructor, the +full-signature test fixture was corrected to include the source filter's +existing normalized `mixed` exclusion; no source-filter behavior changed. + +Final checks on the B1/B2 working tree: + +- Windows/Python 3.13.13: the focused regression command listed earlier in + this record, plus `tests/test_list_evolution_chain.py` and all nonintegration + `tests/native_retrieval` cases: **273 passed, 4 skipped, 26 deselected** in + 15.74 seconds. The four skips are unconfigured Neo4j cases. +- Real dedicated Qdrant Server 1.20.0, client 1.20.0, Python 3.13.13, FastEmbed + absent: `pytest tests/native_retrieval -m integration -q -o log_cli=false`: + **26 passed, 44 deselected**, no skips, in 62.37 seconds. The disposable + Server used only fixture-owned synthetic collections. +- Linux/Python 3.12.15: `pytest tests/test_reader_selection.py + tests/native_retrieval/test_errors.py -q -o log_cli=false + -o cache_dir=/tmp/hy-memory-pr447-b1b2-cache`: **51 passed** in 5.92 seconds, + with checkout source mounted read-only and `PYTHONPATH` selecting it. + The initial Core test environment lacked pytest-asyncio and could not run + async cases (20 passed, 31 failed). Installing pytest-asyncio 1.4.0 into a + separate temporary plugin directory resolved that environment failure; + no project dependency or lock file changed. +- Real Python 3.9.12, existing minimal test environment: full legacy positional + binding plus existing native-version rejection and legacy default/zero + contracts: **3 passed, 48 deselected**. This remains a minimal compatibility + check, not a complete Python 3.9 extras certification. +- Ruff on both changed production modules and both changed test modules, + compileall on both production modules, and `git diff --check`: passed. + +The original red tests provide the targeted regression-injection evidence. +No MemHub production code, dependency, retrieval index or score behavior +changed. Previous CI results remain historical until the new head is checked. + +## 2026-11-15 follow-up: deleted chain heads and RecallConfig positional compatibility + +Based on the exact PR #447 head `b591b40`. The local branch of the same name +also contained an unrelated consolidation commit; the repair branch starts at +the remote PR head and excludes that commit. Existing untracked handoff +materials remain untouched. + +The reported failures were reproduced before changing production code: + +```powershell +$env:HY_MEMORY_TEST_QDRANT_URL='http://127.0.0.1:16333' +.venv\Scripts\python.exe -m pytest tests/native_retrieval/test_errors.py tests/native_retrieval/test_reconcile.py -k 'recall_positional or deleted_native_head or writer_deleted_chain' -o addopts= -q -o log_cli=false +.venv\Scripts\python.exe -m pytest tests/native_retrieval/test_reconcile.py -k writer_deleted_chain -o addopts= -q -o log_cli=false --tb=short +``` + +The positional assertion returned `default_limit_per_layer=5` and +`reconcile_retriever=10`. Deleted-chain resolution raised `integrity` / "no +unique confirmed head". Both real-Server writer-delete scenarios (with and +without a healthy unrelated result) failed at public native search with that +same error. The initial integration attempt used client 1.19.1 and failed +fixture version validation; it was rerun with client 1.20.0 before confirming +the actual deletion failure. + +The resolver now validates the complete graph and unique terminal first. A +SHADOW terminal can have `is_latest=False`, as produced by the writer's +legacy DELETE mapping; the entire component is then omitted, without +returning ancestors. Other unconfirmed heads remain invalid. Missing nodes, +cycles, ambiguous terminals, scope crossings and traversal budgets still fail +closed, including for all-SHADOW components. + +`reconcile_retriever` is appended after all original RecallConfig fields. +Comparison with the base `a2a1858` confirms all **91** old field indexes are +unchanged. The regression also executes the default legacy candidate search +and ADD path using `RecallConfig(10, 100, 7)`. + +Final Windows/Python 3.12.10 checks, Qdrant Server/client 1.20.0: + +- `pytest tests/native_retrieval -o addopts= -q -o log_cli=false --tb=short`: + **85 passed**, no skips, in 187.09 seconds (57 contracts and 28 real-Server + integration tests). The added writer scenario builds two SUPERSEDE versions, + deletes the head, checks that ancestors remain backend candidates, then + verifies public search and a successful subsequent write. Deleted IDs/content + are absent from the subsequent reconcile prompt; healthy candidates survive. +- The focused regression command listed earlier in this record plus + `tests/test_list_evolution_chain.py`: **229 passed, 4 skipped** in 28.88 + seconds. All skips are unconfigured Neo4j cases. +- Ruff on both production and both test modules, compileall on both production + modules and `git diff --check`: passed. + +These are local results. CI for earlier commits does not certify this follow-up. +No MemHub production code, dependency lock, writer behavior or index changes +are included. diff --git a/hy_memory/__init__.py b/hy_memory/__init__.py index e1b2d77b..32163fb6 100644 --- a/hy_memory/__init__.py +++ b/hy_memory/__init__.py @@ -98,7 +98,10 @@ EmbeddingService, EmbeddingServiceProtocol, ) -from .retrieval import AmbiguousMemoryIdError, MemoryKind, MemoryVariant +from .retrieval import ( + AmbiguousMemoryIdError, MemoryKind, MemoryVariant, + NativeRetrievalError, NativeScoreThreshold, +) # ====== Embedding 端点覆盖(per-call,不改 model/dims,只换接入端点)====== from .core.embed_service import embed_endpoint_override @@ -193,6 +196,8 @@ "MemoryKind", "MemoryVariant", "AmbiguousMemoryIdError", + "NativeScoreThreshold", + "NativeRetrievalError", # Embedding 端点覆盖 "embed_endpoint_override", # 高级 API diff --git a/hy_memory/agent/reconciler.py b/hy_memory/agent/reconciler.py index 3795e236..4b9013ef 100644 --- a/hy_memory/agent/reconciler.py +++ b/hy_memory/agent/reconciler.py @@ -394,6 +394,376 @@ async def _generate_search_queries( return [], None + async def _prepare_legacy_candidates( + self, *, new_memories: List[str], search_queries: List[str], + new_memory_embeddings: Optional[Dict[str, List[float]]], + embed_service: Any, vector_store: Any, user_id: str, agent_id: str, + layers: List[Any], source: str, graph_store: Any, cache: Any, + request_id: str, session_id: str, + ) -> Tuple[Dict[str, Any], Dict[str, List[Tuple[str, float]]], Dict[str, List[float]]]: + """Prepare legacy candidates without changing their scoring or dedup lifecycle.""" + from ..models.memory import MemoryLayer, MemoryStatus, MemoryNode + import asyncio + import time as _time + # ── Step 2: 候选搜索,合并候选池 ── + candidate_map: Dict[str, MemoryNode] = {} + candidate_scores: Dict[str, float] = {} + + supplied_embeddings = dict(new_memory_embeddings or {}) + mem_embeddings: List[Optional[List[float]]] = [ + supplied_embeddings.get(text) for text in new_memories + ] + missing_mem_indices = [ + i for i, emb in enumerate(mem_embeddings) if not emb + ] + texts_to_embed = [new_memories[i] for i in missing_mem_indices] + texts_to_embed.extend(search_queries) + computed_embeddings = ( + await embed_service.embed_batch(texts_to_embed) + if texts_to_embed else [] + ) + for i, emb in zip( + missing_mem_indices, + computed_embeddings[: len(missing_mem_indices)], + ): + mem_embeddings[i] = emb + query_embeddings = computed_embeddings[len(missing_mem_indices):] + + def _merge_hits(hits: List[Dict[str, Any]], source: str) -> None: + logger.debug(f"[reconciler] search {source}: {len(hits)} results") + for r in hits: + node = r.get("node") + score = float(r.get("score", 0)) + if not node: + continue + # Canonical states are materialized and versioned by nightly + # evoling using an exact state_key. Letting the free-form S1 + # reconciler mutate them turns a stable state back into an + # event summary and breaks deterministic lifecycle handling. + if is_canonical_state_node(node): + continue + nid = node.node_id + if nid not in candidate_map: + candidate_map[nid] = node + candidate_scores[nid] = score + else: + candidate_scores[nid] = max(candidate_scores[nid], score) + + from ..pipelines._retrieval.reconcile_retrieval import ( + ReconcileHybridRetriever, + ReconcileRetrievalConfig, + ) + + hybrid = ReconcileHybridRetriever( + ReconcileRetrievalConfig.from_env( + limit=self.SEARCH_TOPK, + vec_pool_multiplier=self.VEC_POOL_MULTIPLIER, + min_score=self.HYBRID_MIN_SCORE, + ) + ) + + new_memory_embeddings = { + text: emb + for text, emb in zip(new_memories, mem_embeddings) + if text and emb + } + + hybrid_indices = [ + i for i in range(len(new_memories)) + if i < len(mem_embeddings) and mem_embeddings[i] + ] + + query_indices = [] + if self._enable_search_query and search_queries: + query_indices = [ + i for i in range(len(search_queries)) + if i < len(query_embeddings) + ] + + _t_candidate_search = _time.perf_counter() + hybrid_result = [] + query_result = [] + tasks = [] + if hybrid_indices: + tasks.append(hybrid.search_candidates_batch( + [new_memories[i] for i in hybrid_indices], + [mem_embeddings[i] for i in hybrid_indices], + vector_store=vector_store, + user_id=user_id, + agent_id=agent_id, + layers=layers, + embed_service=embed_service, + graph_store=graph_store, + include_sources=[source], + )) + if query_indices: + tasks.append(vector_store.search_batch([ + { + "query_embedding": query_embeddings[i], + "user_id": user_id, + "agent_ids": None, + "layers": layers, + "limit": self.QUERY_SEARCH_TOPK, + "score_threshold": self.SEARCH_THRESHOLD, + "status_filter": [ + MemoryStatus.ACTIVE, MemoryStatus.SUPERSEDED, + ], + "only_latest": False, + "include_sources": [source], + } + for i in query_indices + ])) + if tasks: + task_results = await asyncio.gather(*tasks, return_exceptions=True) + cursor = 0 + if hybrid_indices: + hybrid_result = task_results[cursor] + cursor += 1 + if query_indices: + query_result = task_results[cursor] + _candidate_search_ms = round( + (_time.perf_counter() - _t_candidate_search) * 1000, 2, + ) + # 关系图用:记下"每条新记忆的 hybrid 候选及其分数"。 + # 这些分数本来算完就丢,写侧拿它建 semantic_neighbor 边是零额外计算的。 + # 用 content 作 key —— 新记忆此刻还没入库,没有 node_id。 + per_memory_candidates: Dict[str, List[Tuple[str, float]]] = {} + + if isinstance(hybrid_result, Exception): + logger.warning("[reconciler] hybrid batch failed: %s", hybrid_result) + hybrid_result = [] + if isinstance(query_result, Exception): + logger.warning("[reconciler] query batch failed: %s", query_result) + query_result = [] + + for source_index, result in zip(hybrid_indices, hybrid_result): + _merge_hits(result, f"memory_hybrid[{source_index}]") + per_memory_candidates[new_memories[source_index]] = [ + (r["node"].node_id, float(r.get("score", 0.0))) + for r in (result or []) + if r.get("node") is not None + and not is_canonical_state_node(r["node"]) + ] + for source_index, result in zip(query_indices, query_result): + _merge_hits( + [ + {"node": r.get("node"), "score": r.get("score", 0)} + for r in result + ], + f"query[{source_index}]", + ) + + logger.debug( + f"[reconciler] search: {len(new_memories)} memories (hybrid) + " + f"{len(search_queries)} queries → {len(candidate_map)} raw candidates" + ) + + # ── Step 2.5: 按演化链合并(复用 reader 侧 _trace_full_chain)── + # 召回池可能含 SUPERSEDED 旧节点(链身)。对每个候选节点双向追溯 + # (supersedes / superseded_by)补全整条链,以链头(is_latest)为代表 + # 去重:同一条链的多个命中(链头+链身)合并为一个候选,链头作代表, + # 链头分数取该链所有命中的最高分。 + # 注:祖先节点仅用于去重/避免重复候选,**不进 prompt**(见 Step 4, + # supersede 时只暴露 head 的 id+content,history 不提供,防止 LLM 误操作)。 + from ..pipelines._retrieval.evolution import trace_full_chains + + chain_head_node: Dict[str, MemoryNode] = {} # head_id → head node + chain_best_score: Dict[str, float] = {} # head_id → 链上最高命中分 + node_to_head: Dict[str, str] = {} # 任一命中 node_id → 其链头 id + chain_members: Dict[str, List[str]] = {} # head_id → 整条链 node_id(head+bodies) + + _t_chain_expand = _time.perf_counter() + try: + traced_chains = await trace_full_chains( + vector_store, list(candidate_map.values()), + ) + except Exception as e: + logger.warning("[reconciler] batch chain trace failed: %s", e) + traced_chains = {} + for nid, node in candidate_map.items(): + if nid in node_to_head: + continue # 已被某条已展开的链覆盖 + chain = traced_chains.get(nid) or [node] + head = chain[0] # _trace_full_chain 保证 [0] 为链头(is_latest 优先) + head_id = head.node_id + chain_head_node.setdefault(head_id, head) + chain_members[head_id] = [cn.node_id for cn in chain] + # 该链上所有“恰好被召回”的节点,分数并入链头候选分 + for cn in chain: + node_to_head[cn.node_id] = head_id + if cn.node_id in candidate_scores: + chain_best_score[head_id] = max( + chain_best_score.get(head_id, 0.0), + candidate_scores[cn.node_id], + ) + + # 按链头候选分排序,取 top(留足余量) + sorted_chains = sorted( + chain_head_node.keys(), + key=lambda hid: chain_best_score.get(hid, 0.0), + reverse=True, + )[: self.FINAL_TOPK * 2] + + # 最终候选:以链头为代表(prompt 只展示 head) + final_candidates: Dict[str, MemoryNode] = {} + for head_id in sorted_chains: + final_candidates[head_id] = chain_head_node[head_id] + candidate_scores[head_id] = chain_best_score.get(head_id, 0.0) + + candidate_map = final_candidates + _chain_expand_ms = round( + (_time.perf_counter() - _t_chain_expand) * 1000, 2, + ) + logger.debug( + f"[reconciler] after chain merge: {len(candidate_map)} chain groups " + f"(traced from {len(node_to_head)} recalled chain nodes)" + ) + + # ── Step 2.7: 候选构成埋点 ── + # 候选检索已放宽到 user 维度(不再按 agent 收窄),候选池里会混入其他 + # agent 的记忆。在 topk 不变的前提下,多 agent 用户的本 agent 候选 + # 可能被挤出去("稀释")。这里把构成落进 pipeline log,上线后可直接 + # 从日志确认稀释是否真的发生,而不必等业务反馈。 + # + # 单 agent 用户 cross_agent_count 恒为 0 —— 放宽前后捞回的是同一批。 + try: + _cand_own_agent = 0 + _cand_cross_agent = 0 + _cand_owner_user = 0 + _cand_owner_agent = 0 + _cross_agent_ids: set = set() + for _n in candidate_map.values(): + _naid = getattr(_n, "agent_id", "") or "" + if _naid and _naid != agent_id: + _cand_cross_agent += 1 + _cross_agent_ids.add(_naid) + else: + _cand_own_agent += 1 + if (getattr(_n, "owner", None) or "user") == "agent": + _cand_owner_agent += 1 + else: + _cand_owner_user += 1 + + _cand_parsed = { + "candidate_count": len(candidate_map), + "own_agent_count": _cand_own_agent, + "cross_agent_count": _cand_cross_agent, + "cross_agent_ids": sorted(_cross_agent_ids)[:20], + "owner_user_count": _cand_owner_user, + "owner_agent_count": _cand_owner_agent, + "recalled_chain_nodes": len(node_to_head), + "final_topk": self.FINAL_TOPK * 2, + # 候选池打满时才可能发生稀释;未打满说明没有竞争 + "topk_saturated": len(candidate_map) >= self.FINAL_TOPK * 2, + "scope": "user", + "new_memory_count": len(new_memories), + "hybrid_query_count": len(hybrid_indices), + "query_query_count": len(query_indices), + "dense_batch_count": int(bool(hybrid_indices)) + + int(bool(query_indices)), + "keyword_batch_count": int(bool(hybrid_indices)), + "candidate_search_ms": _candidate_search_ms, + "chain_expand_ms": _chain_expand_ms, + "vector_store_local": bool( + getattr(vector_store, "_is_local", False) + ), + } + logger.info( + f"[reconciler] candidates={len(candidate_map)} " + f"own_agent={_cand_own_agent} cross_agent={_cand_cross_agent} " + f"owner_user={_cand_owner_user} owner_agent={_cand_owner_agent} " + f"saturated={_cand_parsed['topk_saturated']}" + ) + if cache: + await cache.store_pipeline_log( + request_id=request_id, + user_id=user_id, + agent_id=agent_id, + session_id=session_id, + step="RECONCILE_CANDIDATES", + prompt="", + response="", + parsed=json.dumps(_cand_parsed, ensure_ascii=False), + ) + except Exception as e: + logger.debug(f"[reconciler] store RECONCILE_CANDIDATES log failed: {e}") + + # ── Step 2.6: 候选去重(pre-search)── + # 候选都是链头(is_latest)。只对 L2_FACT + L4_IDENTITY(重复重灾区) + # 判重,其他层不参与(与 search 链路一致)。命中即置 SHADOW(逻辑删除, + # 被判重的链头连带其历史链),并从 candidate_map 移除,避免 LLM 看到 + # 重复候选。 + # + # 分批维度由 owner 决定(group_items_by_owner):owner=user 的记忆属于 + # 用户这一侧,跨 agent 合并判重;owner=agent 的按 agent 隔离。 + try: + import time as _time + from ..pipelines._retrieval.dedup import ( + DedupItem, execute_dedup, group_items_by_owner, + ) + + _DEDUP_LAYERS = {MemoryLayer.L2_FACT, MemoryLayer.L4_IDENTITY} + head_ids = [ + hid for hid, node in candidate_map.items() + if node.layer in _DEDUP_LAYERS + ] + _t_emb = _time.perf_counter() + head_embs = await vector_store.get_embeddings(head_ids) + _get_emb_ms = round((_time.perf_counter() - _t_emb) * 1000, 2) + dedup_items = [] + for hid in head_ids: + node = candidate_map[hid] + emb = head_embs.get(hid) + if not emb: + continue + dedup_items.append(DedupItem( + node_id=hid, + embedding=emb, + content=node.content or "", + is_latest=True, + is_chain_head=True, + gmt_created=(node.gmt_created.timestamp() if node.gmt_created else None), + chain_node_ids=chain_members.get(hid, [hid]), + # 分批维度:owner 决定跨不跨 agent + owner=getattr(node, "owner", None), + user_id=getattr(node, "user_id", "") or user_id, + agent_id=getattr(node, "agent_id", "") or agent_id, + source=getattr(node, "source", "default") or "default", + )) + + if len(dedup_items) >= 2: + # 每批传各自准确的 user_id/agent_id,DEDUP log 归属才正确 + for _batch in group_items_by_owner( + dedup_items, + default_user_id=user_id, + default_agent_id=agent_id, + ): + plan = await execute_dedup( + _batch["items"], + vector_store=vector_store, + cache=cache, + trigger="reconcile", + request_id=request_id, + user_id=_batch["user_id"], + agent_id=_batch["agent_id"], + delete_from_store=True, + extra_parsed={ + "embedding_source": "vdb_get_embeddings", + "get_embeddings_ms": _get_emb_ms, + "embed_count": len(head_ids), + "embed_ok_count": len(dedup_items), + "owner_bucket": _batch["owner"], + "source_bucket": _batch["source"], + "batch_size": len(_batch["items"]), + }, + ) + for did in plan.get("delete_ids", []): + candidate_map.pop(did, None) + except Exception as e: + logger.warning(f"[reconciler] candidate dedup failed: {e}") + + return candidate_map, per_memory_candidates, new_memory_embeddings + async def reconcile( self, new_memories: List[str], @@ -423,14 +793,18 @@ async def reconcile( if not new_memories: return ReconcileResult(ops=[], success=True) - from ..models.memory import MemoryLayer, MemoryStatus, MemoryNode - import asyncio + from ..models.memory import MemoryLayer if layers is None: layers = [MemoryLayer.L2_FACT, MemoryLayer.L4_IDENTITY] try: import time as _time + self.config.recall.validate_reconcile_retriever() + native_reconcile = self.config.recall.reconcile_retriever == "native" + if native_reconcile and len(new_memories) > 16: + from ..retrieval import NativeRetrievalError + raise NativeRetrievalError("budget", "native reconcile supports at most 16 memories") # ── Step 1: Search Query 生成(可消融)── search_queries: List[str] = [] @@ -476,363 +850,22 @@ async def reconcile( except Exception as e: logger.warning(f"[reconciler] store SEARCH_QUERY log failed: {e}") - # ── Step 2: 候选搜索,合并候选池 ── - candidate_map: Dict[str, MemoryNode] = {} - candidate_scores: Dict[str, float] = {} - - supplied_embeddings = dict(new_memory_embeddings or {}) - mem_embeddings: List[Optional[List[float]]] = [ - supplied_embeddings.get(text) for text in new_memories - ] - missing_mem_indices = [ - i for i, emb in enumerate(mem_embeddings) if not emb - ] - texts_to_embed = [new_memories[i] for i in missing_mem_indices] - texts_to_embed.extend(search_queries) - computed_embeddings = ( - await embed_service.embed_batch(texts_to_embed) - if texts_to_embed else [] - ) - for i, emb in zip( - missing_mem_indices, - computed_embeddings[: len(missing_mem_indices)], - ): - mem_embeddings[i] = emb - query_embeddings = computed_embeddings[len(missing_mem_indices):] - - def _merge_hits(hits: List[Dict[str, Any]], source: str) -> None: - logger.debug(f"[reconciler] search {source}: {len(hits)} results") - for r in hits: - node = r.get("node") - score = float(r.get("score", 0)) - if not node: - continue - # Canonical states are materialized and versioned by nightly - # evoling using an exact state_key. Letting the free-form S1 - # reconciler mutate them turns a stable state back into an - # event summary and breaks deterministic lifecycle handling. - if is_canonical_state_node(node): - continue - nid = node.node_id - if nid not in candidate_map: - candidate_map[nid] = node - candidate_scores[nid] = score - else: - candidate_scores[nid] = max(candidate_scores[nid], score) - - from ..pipelines._retrieval.reconcile_retrieval import ( - ReconcileHybridRetriever, - ReconcileRetrievalConfig, - ) - - hybrid = ReconcileHybridRetriever( - ReconcileRetrievalConfig.from_env( - limit=self.SEARCH_TOPK, - vec_pool_multiplier=self.VEC_POOL_MULTIPLIER, - min_score=self.HYBRID_MIN_SCORE, + if native_reconcile: + from ..pipelines._retrieval.reconcile_retrieval import prepare_native_candidates + candidate_map, per_memory_candidates, new_memory_embeddings = await prepare_native_candidates( + new_memories=new_memories, search_queries=search_queries, + supplied_embeddings=new_memory_embeddings, embed_service=embed_service, + vector_store=vector_store, user_id=user_id, layers=layers, source=source, ) - ) - - new_memory_embeddings = { - text: emb - for text, emb in zip(new_memories, mem_embeddings) - if text and emb - } - - hybrid_indices = [ - i for i in range(len(new_memories)) - if i < len(mem_embeddings) and mem_embeddings[i] - ] - - query_indices = [] - if self._enable_search_query and search_queries: - query_indices = [ - i for i in range(len(search_queries)) - if i < len(query_embeddings) - ] - - _t_candidate_search = _time.perf_counter() - hybrid_result = [] - query_result = [] - tasks = [] - if hybrid_indices: - tasks.append(hybrid.search_candidates_batch( - [new_memories[i] for i in hybrid_indices], - [mem_embeddings[i] for i in hybrid_indices], - vector_store=vector_store, - user_id=user_id, - agent_id=agent_id, - layers=layers, - embed_service=embed_service, - graph_store=graph_store, - include_sources=[source], - )) - if query_indices: - tasks.append(vector_store.search_batch([ - { - "query_embedding": query_embeddings[i], - "user_id": user_id, - "agent_ids": None, - "layers": layers, - "limit": self.QUERY_SEARCH_TOPK, - "score_threshold": self.SEARCH_THRESHOLD, - "status_filter": [ - MemoryStatus.ACTIVE, MemoryStatus.SUPERSEDED, - ], - "only_latest": False, - "include_sources": [source], - } - for i in query_indices - ])) - if tasks: - task_results = await asyncio.gather(*tasks, return_exceptions=True) - cursor = 0 - if hybrid_indices: - hybrid_result = task_results[cursor] - cursor += 1 - if query_indices: - query_result = task_results[cursor] - _candidate_search_ms = round( - (_time.perf_counter() - _t_candidate_search) * 1000, 2, - ) - # 关系图用:记下"每条新记忆的 hybrid 候选及其分数"。 - # 这些分数本来算完就丢,写侧拿它建 semantic_neighbor 边是零额外计算的。 - # 用 content 作 key —— 新记忆此刻还没入库,没有 node_id。 - per_memory_candidates: Dict[str, List[Tuple[str, float]]] = {} - - if isinstance(hybrid_result, Exception): - logger.warning("[reconciler] hybrid batch failed: %s", hybrid_result) - hybrid_result = [] - if isinstance(query_result, Exception): - logger.warning("[reconciler] query batch failed: %s", query_result) - query_result = [] - - for source_index, result in zip(hybrid_indices, hybrid_result): - _merge_hits(result, f"memory_hybrid[{source_index}]") - per_memory_candidates[new_memories[source_index]] = [ - (r["node"].node_id, float(r.get("score", 0.0))) - for r in (result or []) - if r.get("node") is not None - and not is_canonical_state_node(r["node"]) - ] - for source_index, result in zip(query_indices, query_result): - _merge_hits( - [ - {"node": r.get("node"), "score": r.get("score", 0)} - for r in result - ], - f"query[{source_index}]", - ) - - logger.debug( - f"[reconciler] search: {len(new_memories)} memories (hybrid) + " - f"{len(search_queries)} queries → {len(candidate_map)} raw candidates" - ) - - # ── Step 2.5: 按演化链合并(复用 reader 侧 _trace_full_chain)── - # 召回池可能含 SUPERSEDED 旧节点(链身)。对每个候选节点双向追溯 - # (supersedes / superseded_by)补全整条链,以链头(is_latest)为代表 - # 去重:同一条链的多个命中(链头+链身)合并为一个候选,链头作代表, - # 链头分数取该链所有命中的最高分。 - # 注:祖先节点仅用于去重/避免重复候选,**不进 prompt**(见 Step 4, - # supersede 时只暴露 head 的 id+content,history 不提供,防止 LLM 误操作)。 - from ..pipelines._retrieval.evolution import trace_full_chains - - chain_head_node: Dict[str, MemoryNode] = {} # head_id → head node - chain_best_score: Dict[str, float] = {} # head_id → 链上最高命中分 - node_to_head: Dict[str, str] = {} # 任一命中 node_id → 其链头 id - chain_members: Dict[str, List[str]] = {} # head_id → 整条链 node_id(head+bodies) - - _t_chain_expand = _time.perf_counter() - try: - traced_chains = await trace_full_chains( - vector_store, list(candidate_map.values()), - ) - except Exception as e: - logger.warning("[reconciler] batch chain trace failed: %s", e) - traced_chains = {} - for nid, node in candidate_map.items(): - if nid in node_to_head: - continue # 已被某条已展开的链覆盖 - chain = traced_chains.get(nid) or [node] - head = chain[0] # _trace_full_chain 保证 [0] 为链头(is_latest 优先) - head_id = head.node_id - chain_head_node.setdefault(head_id, head) - chain_members[head_id] = [cn.node_id for cn in chain] - # 该链上所有“恰好被召回”的节点,分数并入链头候选分 - for cn in chain: - node_to_head[cn.node_id] = head_id - if cn.node_id in candidate_scores: - chain_best_score[head_id] = max( - chain_best_score.get(head_id, 0.0), - candidate_scores[cn.node_id], - ) - - # 按链头候选分排序,取 top(留足余量) - sorted_chains = sorted( - chain_head_node.keys(), - key=lambda hid: chain_best_score.get(hid, 0.0), - reverse=True, - )[: self.FINAL_TOPK * 2] - - # 最终候选:以链头为代表(prompt 只展示 head) - final_candidates: Dict[str, MemoryNode] = {} - for head_id in sorted_chains: - final_candidates[head_id] = chain_head_node[head_id] - candidate_scores[head_id] = chain_best_score.get(head_id, 0.0) - - candidate_map = final_candidates - _chain_expand_ms = round( - (_time.perf_counter() - _t_chain_expand) * 1000, 2, - ) - logger.debug( - f"[reconciler] after chain merge: {len(candidate_map)} chain groups " - f"(traced from {len(node_to_head)} recalled chain nodes)" - ) - - # ── Step 2.7: 候选构成埋点 ── - # 候选检索已放宽到 user 维度(不再按 agent 收窄),候选池里会混入其他 - # agent 的记忆。在 topk 不变的前提下,多 agent 用户的本 agent 候选 - # 可能被挤出去("稀释")。这里把构成落进 pipeline log,上线后可直接 - # 从日志确认稀释是否真的发生,而不必等业务反馈。 - # - # 单 agent 用户 cross_agent_count 恒为 0 —— 放宽前后捞回的是同一批。 - try: - _cand_own_agent = 0 - _cand_cross_agent = 0 - _cand_owner_user = 0 - _cand_owner_agent = 0 - _cross_agent_ids: set = set() - for _n in candidate_map.values(): - _naid = getattr(_n, "agent_id", "") or "" - if _naid and _naid != agent_id: - _cand_cross_agent += 1 - _cross_agent_ids.add(_naid) - else: - _cand_own_agent += 1 - if (getattr(_n, "owner", None) or "user") == "agent": - _cand_owner_agent += 1 - else: - _cand_owner_user += 1 - - _cand_parsed = { - "candidate_count": len(candidate_map), - "own_agent_count": _cand_own_agent, - "cross_agent_count": _cand_cross_agent, - "cross_agent_ids": sorted(_cross_agent_ids)[:20], - "owner_user_count": _cand_owner_user, - "owner_agent_count": _cand_owner_agent, - "recalled_chain_nodes": len(node_to_head), - "final_topk": self.FINAL_TOPK * 2, - # 候选池打满时才可能发生稀释;未打满说明没有竞争 - "topk_saturated": len(candidate_map) >= self.FINAL_TOPK * 2, - "scope": "user", - "new_memory_count": len(new_memories), - "hybrid_query_count": len(hybrid_indices), - "query_query_count": len(query_indices), - "dense_batch_count": int(bool(hybrid_indices)) - + int(bool(query_indices)), - "keyword_batch_count": int(bool(hybrid_indices)), - "candidate_search_ms": _candidate_search_ms, - "chain_expand_ms": _chain_expand_ms, - "vector_store_local": bool( - getattr(vector_store, "_is_local", False) - ), - } - logger.info( - f"[reconciler] candidates={len(candidate_map)} " - f"own_agent={_cand_own_agent} cross_agent={_cand_cross_agent} " - f"owner_user={_cand_owner_user} owner_agent={_cand_owner_agent} " - f"saturated={_cand_parsed['topk_saturated']}" - ) - if cache: - await cache.store_pipeline_log( - request_id=request_id, - user_id=user_id, - agent_id=agent_id, - session_id=session_id, - step="RECONCILE_CANDIDATES", - prompt="", - response="", - parsed=json.dumps(_cand_parsed, ensure_ascii=False), - ) - except Exception as e: - logger.debug(f"[reconciler] store RECONCILE_CANDIDATES log failed: {e}") - - # ── Step 2.6: 候选去重(pre-search)── - # 候选都是链头(is_latest)。只对 L2_FACT + L4_IDENTITY(重复重灾区) - # 判重,其他层不参与(与 search 链路一致)。命中即置 SHADOW(逻辑删除, - # 被判重的链头连带其历史链),并从 candidate_map 移除,避免 LLM 看到 - # 重复候选。 - # - # 分批维度由 owner 决定(group_items_by_owner):owner=user 的记忆属于 - # 用户这一侧,跨 agent 合并判重;owner=agent 的按 agent 隔离。 - try: - import time as _time - from ..pipelines._retrieval.dedup import ( - DedupItem, execute_dedup, group_items_by_owner, + else: + candidate_map, per_memory_candidates, new_memory_embeddings = await self._prepare_legacy_candidates( + new_memories=new_memories, search_queries=search_queries, + new_memory_embeddings=new_memory_embeddings, embed_service=embed_service, + vector_store=vector_store, user_id=user_id, agent_id=agent_id, + layers=layers, source=source, graph_store=graph_store, cache=cache, + request_id=request_id, session_id=session_id, ) - _DEDUP_LAYERS = {MemoryLayer.L2_FACT, MemoryLayer.L4_IDENTITY} - head_ids = [ - hid for hid, node in candidate_map.items() - if node.layer in _DEDUP_LAYERS - ] - _t_emb = _time.perf_counter() - head_embs = await vector_store.get_embeddings(head_ids) - _get_emb_ms = round((_time.perf_counter() - _t_emb) * 1000, 2) - dedup_items = [] - for hid in head_ids: - node = candidate_map[hid] - emb = head_embs.get(hid) - if not emb: - continue - dedup_items.append(DedupItem( - node_id=hid, - embedding=emb, - content=node.content or "", - is_latest=True, - is_chain_head=True, - gmt_created=(node.gmt_created.timestamp() if node.gmt_created else None), - chain_node_ids=chain_members.get(hid, [hid]), - # 分批维度:owner 决定跨不跨 agent - owner=getattr(node, "owner", None), - user_id=getattr(node, "user_id", "") or user_id, - agent_id=getattr(node, "agent_id", "") or agent_id, - source=getattr(node, "source", "default") or "default", - )) - - if len(dedup_items) >= 2: - # 每批传各自准确的 user_id/agent_id,DEDUP log 归属才正确 - for _batch in group_items_by_owner( - dedup_items, - default_user_id=user_id, - default_agent_id=agent_id, - ): - plan = await execute_dedup( - _batch["items"], - vector_store=vector_store, - cache=cache, - trigger="reconcile", - request_id=request_id, - user_id=_batch["user_id"], - agent_id=_batch["agent_id"], - delete_from_store=True, - extra_parsed={ - "embedding_source": "vdb_get_embeddings", - "get_embeddings_ms": _get_emb_ms, - "embed_count": len(head_ids), - "embed_ok_count": len(dedup_items), - "owner_bucket": _batch["owner"], - "source_bucket": _batch["source"], - "batch_size": len(_batch["items"]), - }, - ) - for did in plan.get("delete_ids", []): - candidate_map.pop(did, None) - except Exception as e: - logger.warning(f"[reconciler] candidate dedup failed: {e}") - # ── Step 3: 如果没有任何候选,全部 ADD(supersedes=[])── if not candidate_map: logger.debug( @@ -963,7 +996,8 @@ def _sort_key(n): t = n.memory_at or n.gmt_created return t.isoformat() if t else "" - sorted_candidates = sorted(candidate_map.values(), key=_sort_key) + sorted_candidates = (list(candidate_map.values()) if native_reconcile + else sorted(candidate_map.values(), key=_sort_key)) existing_mem_list = [] for node in sorted_candidates: diff --git a/hy_memory/client.py b/hy_memory/client.py index 3600ebb5..d43eedfd 100644 --- a/hy_memory/client.py +++ b/hy_memory/client.py @@ -70,6 +70,8 @@ MessageList = List[Dict[str, str]] AddInput = Union[str, MessageList] +_SEARCH_UNSET = object() + # working(coding)路两层的默认阈值。刻意不复用 chat 路的 min_score: # chat 匹配的是完整语义单元,0.4 是长期在用的经验值;working 事实层匹配的是一句话 # 结论,同样语义相关度下分数系统性偏低,0.4 @@ -1844,6 +1846,15 @@ def finalize_session( include_runtime_diag=True, ) + def prepare_native_retrieval(self) -> Dict[str, Any]: + """Explicitly prepare Qdrant Server text indexes; never rewrites memories.""" + return self._search_loop_thread.run(self.async_prepare_native_retrieval()) + + @_ensure_search_loop + async def async_prepare_native_retrieval(self) -> Dict[str, Any]: + """Async form of :meth:`prepare_native_retrieval`.""" + return await self._vector_store.prepare_native_retrieval() + def search( self, query: Optional[str] = None, @@ -1857,14 +1868,16 @@ def search( include_sources: Optional[List[str]] = None, exclude_sources: Optional[List[str]] = None, limit: int = 10, - min_score: float = 0.4, + min_score: Any = _SEARCH_UNSET, working_fact_min_score: Optional[float] = None, working_experience_min_score: Optional[float] = None, schema_min_score: float = 0.4, - schema_limit: int = 10, + schema_limit: Any = _SEARCH_UNSET, intention_limit: int = 0, created_after: Optional[float] = None, reader: str = "", + native_mode: str = "hybrid", + native_threshold: Any = None, workspace_id: Optional[str] = None, branch: Optional[str] = None, request_id: Optional[str] = None, @@ -1872,7 +1885,7 @@ def search( statuses: Optional[List[Any]] = None, embed_override: Optional[Dict[str, Any]] = None, query_embedding: Optional[List[float]] = None, - rerank: bool = True, + rerank: Any = _SEARCH_UNSET, ) -> Dict[str, Any]: """ 搜索记忆。 @@ -2005,7 +2018,7 @@ def search( working_experience_min_score=working_experience_min_score, schema_min_score=schema_min_score, schema_limit=schema_limit, intention_limit=intention_limit, created_after=created_after, - reader=reader, + reader=reader, native_mode=native_mode, native_threshold=native_threshold, workspace_id=workspace_id, branch=branch, request_id=request_id, layers=layers, statuses=statuses, embed_override=embed_override, @@ -3933,14 +3946,16 @@ async def async_search( include_sources: Optional[List[str]] = None, exclude_sources: Optional[List[str]] = None, limit: int = 10, - min_score: float = 0.4, + min_score: Any = _SEARCH_UNSET, working_fact_min_score: Optional[float] = None, working_experience_min_score: Optional[float] = None, schema_min_score: float = 0.4, - schema_limit: int = 10, + schema_limit: Any = _SEARCH_UNSET, intention_limit: int = 0, created_after: Optional[float] = None, reader: str = "", + native_mode: str = "hybrid", + native_threshold: Any = None, workspace_id: Optional[str] = None, branch: Optional[str] = None, request_id: Optional[str] = None, @@ -3948,7 +3963,7 @@ async def async_search( statuses: Optional[List[Any]] = None, embed_override: Optional[Dict[str, Any]] = None, query_embedding: Optional[List[float]] = None, - rerank: bool = True, + rerank: Any = _SEARCH_UNSET, ) -> Dict[str, Any]: """搜索记忆(异步)。详见 search() docstring。 @@ -3974,7 +3989,7 @@ async def async_search( working_experience_min_score=working_experience_min_score, schema_min_score=schema_min_score, schema_limit=schema_limit, intention_limit=intention_limit, created_after=created_after, - reader=reader, workspace_id=workspace_id, branch=branch, + reader=reader, native_mode=native_mode, native_threshold=native_threshold, workspace_id=workspace_id, branch=branch, request_id=request_id, layers=layers, statuses=statuses, query_embedding=query_embedding, rerank=rerank, @@ -3989,7 +4004,7 @@ async def async_search( working_experience_min_score=working_experience_min_score, schema_min_score=schema_min_score, schema_limit=schema_limit, intention_limit=intention_limit, created_after=created_after, - reader=reader, workspace_id=workspace_id, branch=branch, + reader=reader, native_mode=native_mode, native_threshold=native_threshold, workspace_id=workspace_id, branch=branch, request_id=request_id, layers=layers, statuses=statuses, query_embedding=query_embedding, rerank=rerank, @@ -4008,27 +4023,51 @@ async def _async_search_impl( include_sources: Optional[List[str]] = None, exclude_sources: Optional[List[str]] = None, limit: int = 10, - min_score: float = 0.4, + min_score: Any = _SEARCH_UNSET, working_fact_min_score: Optional[float] = None, working_experience_min_score: Optional[float] = None, schema_min_score: float = 0.4, - schema_limit: int = 10, + schema_limit: Any = _SEARCH_UNSET, intention_limit: int = 0, created_after: Optional[float] = None, reader: str = "", + native_mode: str = "hybrid", + native_threshold: Any = None, workspace_id: Optional[str] = None, branch: Optional[str] = None, request_id: Optional[str] = None, layers: Optional[List[Any]] = None, statuses: Optional[List[Any]] = None, query_embedding: Optional[List[float]] = None, - rerank: bool = True, + rerank: Any = _SEARCH_UNSET, ) -> Dict[str, Any]: """搜索记忆(异步)。详见 search() docstring。 layers(默认空):只召回指定层级,接受字符串或 MemoryLayer。 过滤在召回阶段生效(而非召回后再筛),避免无关层级挤占 limit 名额。 """ + from .pipelines._retrieval.config import resolve_reader_name + from .retrieval import NativeRetrievalError + reader = resolve_reader_name(reader) + is_native = reader == "native" + if is_native: + from .retrieval import validate_native_threshold + validate_native_threshold(native_threshold, native_mode) + if native_mode not in ("dense", "keyword", "hybrid"): + raise NativeRetrievalError("invalid_request", "unknown native mode") + if min_score is not _SEARCH_UNSET and min_score is not None: + raise NativeRetrievalError("invalid_request", "use native_threshold instead of min_score") + if schema_limit is not _SEARCH_UNSET and (type(schema_limit) is not int or schema_limit != 0): + raise NativeRetrievalError("invalid_request", "native schema_limit must be zero") + if intention_limit != 0 or (rerank is not _SEARCH_UNSET and rerank is not False): + raise NativeRetrievalError("invalid_request", "native intention/rerank is unsupported") + if type(limit) is not int or not 0 <= limit <= 100: + raise NativeRetrievalError("invalid_request", "native limit must be 0..100") + min_score, schema_limit, rerank = None, 0, False + else: + min_score = 0.4 if min_score is _SEARCH_UNSET else min_score + schema_limit = 10 if schema_limit is _SEARCH_UNSET else schema_limit + rerank = True if rerank is _SEARCH_UNSET else rerank # ── 入参归一化 ── if queries is not None and query is not None: raise ValueError("query and queries are mutually exclusive; choose one") @@ -4106,6 +4145,33 @@ async def _async_search_impl( "Use single agent_id with multiple session_ids, or multiple agent_ids without session_ids." ) + if is_native: + from .models.memory import MemoryLayer + from .pipelines._retrieval.config import VDB_RECALL_LAYERS + normalized_native_layers = self._normalize_layers(layers) + if (lite_mode or scene_all or scene_norm == "productivity" or statuses is not None + or (normalized_native_layers and any(layer not in VDB_RECALL_LAYERS for layer in normalized_native_layers))): + raise NativeRetrievalError("unsupported", "native reader supports pro/ultra chat L2/L3/L4 only") + if any(not isinstance(u, str) or not u.strip() for u in effective_user_ids): + raise NativeRetrievalError("invalid_request", "native user IDs must be nonempty strings") + from .data.vector_store_base import NativeQuery, validate_native_scope + validate_native_scope(NativeQuery( + texts=[target_query], embeddings=[], user_ids=effective_user_ids, + layers=normalized_native_layers or list(VDB_RECALL_LAYERS), + mode=native_mode, limit=max(30, 3 * limit), + )) + if not isinstance(target_query, str): + raise NativeRetrievalError("invalid_request", "native query must be a string") + if not target_query.strip() or limit == 0: + return { + "request_id": request_id, + "memories": {"schema": [], "proactive": [], "normal": []}, + "native": {"skipped": True, "mode": native_mode}, + "rewrite": {"used": False, "original_query": target_query, + "effective_query": target_query, "created_after": None, + "scene": "normal"}, + "elapsed_ms": round((time.perf_counter() - t0) * 1000, 2), + } if statuses is not None: from .models.memory import MemoryLayer @@ -4337,6 +4403,9 @@ def _ensure_llm(): f"target({len(target_query or '')} chars)" ) + if is_native and decided_scene == "productivity": + raise NativeRetrievalError("unsupported", "native reader cannot route to working/coding") + # ──────────────────────────────────────────────────────────── # 场景判定落 pipeline log(search 的第一步,在 reader 之前) # ──────────────────────────────────────────────────────────── @@ -4457,6 +4526,7 @@ def _ensure_llm(): read_req = ReadRequest( query=effective_target_query, + raw_query=target_query, native_mode=native_mode, native_threshold=native_threshold, user_ids=effective_user_ids, agent_ids=effective_agent_ids, session_ids=effective_session_ids, @@ -4486,6 +4556,19 @@ def _ensure_llm(): logger.debug(f"[search] using reader: {reader_impl.VERSION}") result: ReadResponse = await reader_impl.read(read_req) + if is_native: + if not result.success: + raise NativeRetrievalError("backend", result.error_message or "native search failed") + return { + "request_id": request_id, + "memories": {"schema": [], "proactive": [], "normal": result.memories}, + "native": result.extra.get("native", {}), + "rewrite": {"used": rewrite_used, "original_query": target_query, + "effective_query": effective_target_query, + "created_after": rewrite_created_after, "scene": decided_scene}, + "elapsed_ms": round((time.perf_counter() - t0) * 1000, 2), + } + if not result.success: logger.warning(f"[search] failed: {result.error_message}") return { diff --git a/hy_memory/config.py b/hy_memory/config.py index 707846ac..31bc30f3 100644 --- a/hy_memory/config.py +++ b/hy_memory/config.py @@ -451,6 +451,14 @@ class RecallConfig: # 最小分数阈值 min_score_threshold: float = 0.0 + # Append new options to preserve the legacy positional constructor. + reconcile_retriever: str = "default" + + def validate_reconcile_retriever(self) -> None: + """Reject unknown retrieval strategies, including mutated configurations.""" + if self.reconcile_retriever not in ("default", "native"): + raise ValueError("recall.reconcile_retriever must be 'default' or 'native'") + def __post_init__(self): self.default_limit = _get_env_int("MEMORY_RECALL_DEFAULT_LIMIT", 10) self.max_limit = _get_env_int("MEMORY_RECALL_MAX_LIMIT", 100) @@ -1386,6 +1394,7 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "MemoryConfig": setattr(sub, key, value) config.business_store.validate() + config.recall.validate_reconcile_retriever() # basic_profile: fields 是 Dict[str, str],整体替换(非空时) if "basic_profile" in config_dict: @@ -1449,6 +1458,7 @@ def to_dict(self) -> Dict[str, Any]: "embedding_dims": self.embedder.embedding_dims, }, "recall": { + "reconcile_retriever": self.recall.reconcile_retriever, "default_limit": self.recall.default_limit, "semantic_weight": self.recall.semantic_weight, "recency_weight": self.recall.recency_weight, diff --git a/hy_memory/data/qdrant_native.py b/hy_memory/data/qdrant_native.py new file mode 100644 index 00000000..617ebe51 --- /dev/null +++ b/hy_memory/data/qdrant_native.py @@ -0,0 +1,222 @@ +"""Qdrant Server native text scoring and query trees (no local retrieval state).""" + +import sys +from importlib.metadata import version +from typing import Any, Dict + +from ..retrieval import NativeRetrievalError +from ..utils.source_filter import normalize_source_filters +from .vector_store_base import NativeQuery + +VERSION = "1.20.0" + + +def check_version(client: Any, is_local: bool) -> None: + if sys.version_info < (3, 10) or is_local or version("qdrant-client") != VERSION: + raise NativeRetrievalError( + "unsupported", + "native retrieval requires Python >=3.10 and qdrant-client 1.20.0 with Server", + ) + if client.info().version != VERSION: + raise NativeRetrievalError( + "unsupported", "native retrieval requires Qdrant Server 1.20.0" + ) + + +def schemas() -> Dict[str, Any]: + from qdrant_client import models as m + + return { + "content": m.TextIndexParams( + type="text", + tokenizer=m.TokenizerType.MULTILINGUAL, + lowercase=True, + scoring=True, + ), + "tags[]": m.TextIndexParams( + type="text", + tokenizer=m.TokenizerType.MULTILINGUAL, + lowercase=True, + scoring=True, + ), + "tags": m.PayloadSchemaType.KEYWORD, + } + + +def check_schema(client: Any, collection: str, prepare: bool = False) -> None: + expected = schemas() + info = client.get_collection(collection) + check_vectors(info) + actual = info.payload_schema or {} + missing = [] + for field, wanted in expected.items(): + entry = actual.get(field) + if entry is None: + missing.append(field) + continue + valid = entry.data_type == ("keyword" if field == "tags" else "text") + if field != "tags": + params = entry.params + scoring = getattr(params, "scoring", None) + bm25 = scoring is True or ( + getattr(scoring, "type", None) == "bm25" + and all( + value is None + for key, value in scoring.model_dump().items() + if key != "type" + ) + ) + valid = ( + valid + and params is not None + and params.tokenizer == wanted.tokenizer + and params.lowercase is not False + and bm25 + ) + valid = ( + valid and params.min_token_len is None and params.max_token_len is None + ) + # Non-default text analysis changes the score/query contract. + if params is not None: + for key in ( + "stopwords", + "stemmer", + "phrase_matching", + "ascii_folding", + "nested", + ): + valid = valid and not getattr(params, key, None) + if not valid: + raise NativeRetrievalError( + "schema_conflict", f"incompatible native index: {field}" + ) + if missing and not prepare: + raise NativeRetrievalError( + "not_ready", + f"missing native indexes: {', '.join(missing)}; call prepare_native_retrieval()", + ) + for field in missing: + result = client.create_payload_index( + collection, field, field_schema=expected[field], wait=True + ) + if str(result.status) != "completed": + raise NativeRetrievalError( + "not_ready", f"native index operation incomplete: {field}" + ) + if missing: + check_schema(client, collection) + + +def check_vectors(info: Any) -> None: + """The adapter's dense query uses the default (possibly named-empty) vector.""" + from qdrant_client import models as m + + vectors = info.config.params.vectors + params = vectors.get("") if isinstance(vectors, dict) else vectors + if params is None or params.distance != m.Distance.COSINE: + raise NativeRetrievalError( + "unsupported", "native retrieval requires a default cosine vector" + ) + + +def scope_filter(request: NativeQuery, seed: bool = True) -> Any: + from qdrant_client import models as m + + include, exclude = normalize_source_filters( + request.include_sources, request.exclude_sources + ) + must = [ + m.FieldCondition(key="user_id", match=m.MatchAny(any=request.user_ids)), + m.FieldCondition( + key="layer", match=m.MatchAny(any=[layer.value for layer in request.layers]) + ), + m.FieldCondition(key="status", match=m.MatchAny(any=["active", "superseded"])), + ] + for key, values in ( + ("agent_id", request.agent_ids), + ("session_id", request.session_ids), + ("isolation_key", request.isolation_keys), + ): + if seed and values: + must.append(m.FieldCondition(key=key, match=m.MatchAny(any=values))) + if include: + must.append(m.FieldCondition(key="source", match=m.MatchAny(any=include))) + return m.Filter( + must=must, + must_not=[m.FieldCondition(key="source", match=m.MatchAny(any=exclude))] + if exclude + else None, + ) + + +def lexical_tree(text: str, query_filter: Any, limit: int) -> Any: + from qdrant_client import models as m + + return m.Prefetch( + prefetch=[ + m.Prefetch( + query=m.TextQuery(text=text), + using=field, + filter=query_filter, + limit=limit, + ) + for field in ("content", "tags[]") + ], + query=m.FusionQuery(fusion=m.Fusion.RRF), + filter=query_filter, + limit=limit, + ) + + +def query_tree(request: NativeQuery) -> Dict[str, Any]: + from qdrant_client import models as m + + filt = scope_filter(request) + branches = [] + for i, text in enumerate(request.texts): + lex = ( + lexical_tree(text, filt, request.limit) if request.mode != "dense" else None + ) + dense = ( + m.Prefetch(query=request.embeddings[i], filter=filt, limit=request.limit) + if request.mode != "keyword" + else None + ) + if request.mode == "hybrid": + branch = m.Prefetch( + prefetch=[dense, lex], + query=m.FusionQuery(fusion=m.Fusion.RRF), + filter=filt, + limit=request.limit, + ) + else: + branch = lex or dense + branches.append(branch) + if len(branches) == 1: + branch = branches[0] + return dict( + query=branch.query, + using=branch.using, + prefetch=branch.prefetch, + query_filter=filt, + limit=request.limit, + ) + return dict( + prefetch=branches, + query=m.FusionQuery(fusion=m.Fusion.RRF), + query_filter=filt, + limit=request.limit, + ) + + +def probe(client: Any, collection: str) -> None: + + # Always exercise text scoring and fusion, including on an empty collection. + tree = lexical_tree("hy_memory_native_capability_probe", None, 1) + client.query_points( + collection, + query=tree.query, + prefetch=tree.prefetch, + limit=1, + with_payload=False, + ) diff --git a/hy_memory/data/vector_store_base.py b/hy_memory/data/vector_store_base.py index c0ef5d09..3db7d829 100644 --- a/hy_memory/data/vector_store_base.py +++ b/hy_memory/data/vector_store_base.py @@ -33,12 +33,88 @@ from typing import Optional, List, Dict, Any import logging import uuid +import math +import sys +from dataclasses import dataclass, field from ..models.memory import ( MemoryNode, MemoryLayer, MemoryStatus, MetaCognitionTag, ) from ..config import MemoryConfig +from ..retrieval import NativeRetrievalError + + +@dataclass +class NativeQuery: + """Internal bounded native query; every branch uses the same seed scope.""" + + texts: List[str] + embeddings: List[List[float]] + user_ids: List[str] + layers: List[MemoryLayer] + mode: str = "hybrid" + limit: int = 30 + agent_ids: List[str] = field(default_factory=list) + session_ids: List[str] = field(default_factory=list) + isolation_keys: List[str] = field(default_factory=list) + include_sources: List[str] = field(default_factory=list) + exclude_sources: List[str] = field(default_factory=list) + + +def validate_native_scope(request: NativeQuery) -> None: + """Validate real scope/options even when a query is skipped.""" + if sys.version_info < (3, 10): + raise NativeRetrievalError( + "unsupported", "native retrieval requires Python >=3.10" + ) + if request.mode not in ("dense", "keyword", "hybrid"): + raise NativeRetrievalError("invalid_request", "unknown native mode") + if not request.user_ids or any( + not isinstance(u, str) or not u.strip() for u in request.user_ids + ): + raise NativeRetrievalError( + "invalid_request", "native retrieval requires nonempty user IDs" + ) + if not request.layers or any( + layer.value not in ("l2_fact", "l3_summary", "l4_identity") + for layer in request.layers + ): + raise NativeRetrievalError( + "invalid_request", "native retrieval supports L2/L3/L4 only" + ) + if type(request.limit) is not int or not 1 <= request.limit <= 300: + raise NativeRetrievalError( + "invalid_request", "native pool limit must be 1..300" + ) + + +def validate_native_query(request: NativeQuery) -> None: + """Reject incomplete plans before issuing any RPC.""" + validate_native_scope(request) + if ( + not request.texts + or len(request.texts) > 20 + or any(not isinstance(t, str) or not t.strip() for t in request.texts) + ): + raise NativeRetrievalError( + "invalid_request", "native queries must contain 1..20 nonblank texts" + ) + if request.mode != "keyword": + if len(request.embeddings) != len(request.texts): + raise NativeRetrievalError( + "integrity", "embedding count does not match query count" + ) + for emb in request.embeddings: + if not emb or any( + isinstance(v, bool) + or not isinstance(v, (int, float)) + or not math.isfinite(v) + for v in emb + ): + raise NativeRetrievalError( + "invalid_request", "embeddings must be finite nonempty vectors" + ) # ================================================================ @@ -56,6 +132,7 @@ # 以及 owner 字段引入前的历史数据可能为空;回退保证这些节点的行为与改动前一致, # 不会因为换了字段而被静默过滤掉。 + def node_effective_time(payload: Dict[str, Any]) -> Optional[float]: """取 payload 的有效时间戳:memory_at 优先,回退 gmt_created;都没有返回 None。 @@ -117,6 +194,22 @@ def __init__(self, config: MemoryConfig): _supports_retrieval_clauses: bool = False _supports_narrow_episode_queries: bool = False + async def prepare_native_retrieval(self) -> Dict[str, Any]: + """Explicitly prepare the backend's native indexes; never called by reads.""" + raise NativeRetrievalError("unsupported", "native retrieval requires Qdrant Server") + + async def native_query(self, request: NativeQuery) -> List[Dict[str, Any]]: + """Execute one ordered backend query plan, without application-side fusion.""" + raise NativeRetrievalError("unsupported", "native retrieval requires Qdrant Server") + + async def native_get_by_ids(self, node_ids: List[str]) -> List[MemoryNode]: + """Fetch exactly the requested IDs or raise an integrity error.""" + raise NativeRetrievalError("unsupported", "strict native ID retrieval is unsupported") + + async def native_cosine(self, embedding: List[float], node_ids: List[str], request: NativeQuery) -> List[Dict[str, Any]]: + """Obtain backend cosine evidence restricted to resolved candidate IDs.""" + raise NativeRetrievalError("unsupported", "native cosine retrieval is unsupported") + # ================================================================ # 生命周期 # ================================================================ diff --git a/hy_memory/data/vector_store_qdrant.py b/hy_memory/data/vector_store_qdrant.py index d62d02e6..6369549c 100644 --- a/hy_memory/data/vector_store_qdrant.py +++ b/hy_memory/data/vector_store_qdrant.py @@ -30,7 +30,8 @@ ) from ..config import MemoryConfig from ..utils.async_executor import run_in_executor_guarded -from .vector_store_base import VectorStoreBase +from .vector_store_base import NativeQuery, VectorStoreBase, validate_native_query +from ..retrieval import NATIVE_CONFIG_ID, native_score_kind from .vector_store_chroma import _vdb_executor logger = logging.getLogger(__name__) @@ -146,6 +147,165 @@ class QdrantVectorStore(VectorStoreBase): _supports_retrieval_clauses = True _supports_narrow_episode_queries = True + async def _native_call(self, callback: Callable[[], Any]) -> Any: + from ..retrieval import NativeRetrievalError + + try: + return await _run_in_vdb_pool(callback) + except NativeRetrievalError: + raise + except Exception as exc: + raise NativeRetrievalError( + "backend", "Qdrant native operation failed" + ) from exc + + async def prepare_native_retrieval(self) -> Dict[str, Any]: + """Prepare native indexes on existing payloads, without re-upserting data.""" + from . import qdrant_native as native + + def prepare(): + native.check_version(self._client, self._is_local) + native.check_schema(self._client, self._collection_name, prepare=True) + native.probe(self._client, self._collection_name) + return {"config_id": NATIVE_CONFIG_ID, "server_version": native.VERSION} + + return await self._native_call(prepare) + + async def native_query(self, request: NativeQuery) -> List[Dict[str, Any]]: + """Execute one native query plan and return backend-ranked raw scores.""" + from . import qdrant_native as native + + validate_native_query(request) + + def query(): + native.check_version(self._client, self._is_local) + native.check_schema(self._client, self._collection_name) + response = self._client.query_points( + self._collection_name, + **native.query_tree(request), + with_payload=True, + with_vectors=False, + ) + return self._native_hits(response.points, native_score_kind(request.mode)) + + return await self._native_call(query) + + def _native_hits(self, points: List[Any], kind: str) -> List[Dict[str, Any]]: + import math + from ..retrieval import NativeRetrievalError + + hits = [] + seen = set() + for point in points: + node = self._payload_to_node(point.payload) + if ( + not node.node_id + or not self._native_id_matches(point.id, node.node_id) + or node.node_id in seen + ): + raise NativeRetrievalError( + "integrity", "native response IDs do not match payloads" + ) + if not math.isfinite(point.score): + raise NativeRetrievalError("integrity", "nonfinite native score") + seen.add(node.node_id) + hits.append( + dict( + node=node, + node_id=node.node_id, + matched_id=node.node_id, + score=float(point.score), + score_kind=kind, + config_id=NATIVE_CONFIG_ID, + ) + ) + return hits + + def _native_id_matches(self, point_id: Any, node_id: str) -> bool: + """Qdrant canonicalizes valid UUID strings, including IDs stored as hex.""" + import uuid + + try: + return uuid.UUID(str(point_id)) == uuid.UUID( + str(self._node_id_to_point_id(node_id)) + ) + except (ValueError, TypeError, AttributeError): + return False + + async def native_get_by_ids(self, node_ids: List[str]) -> List[MemoryNode]: + """Fetch all required IDs in request order, failing on missing/mismatched IDs.""" + from ..retrieval import NativeRetrievalError + from . import qdrant_native as native + + ids = list(dict.fromkeys(node_ids)) + if not ids: + return [] + + def fetch(): + native.check_version(self._client, self._is_local) + points = self._client.retrieve( + self._collection_name, + ids=[self._node_id_to_point_id(nid) for nid in ids], + with_payload=True, + with_vectors=False, + ) + nodes = {} + for point in points: + node = self._payload_to_node(point.payload) + if ( + node.node_id not in ids + or node.node_id in nodes + or not self._native_id_matches(point.id, node.node_id) + ): + raise NativeRetrievalError( + "integrity", "strict ID response mismatch" + ) + nodes[node.node_id] = node + if set(nodes) != set(ids): + raise NativeRetrievalError( + "integrity", "required evolution node is missing" + ) + return [nodes[nid] for nid in ids] + + return await self._native_call(fetch) + + async def native_cosine( + self, embedding: List[float], node_ids: List[str], request: NativeQuery + ) -> List[Dict[str, Any]]: + """Get complete cosine evidence restricted to resolved candidate IDs.""" + from . import qdrant_native as native + from qdrant_client import models as m + from ..retrieval import NativeRetrievalError + + ids = list(dict.fromkeys(node_ids)) + if not ids: + return [] + + def query(): + native.check_version(self._client, self._is_local) + info = self._client.get_collection(self._collection_name) + native.check_vectors(info) + filt = native.scope_filter(request, seed=False) + filt.must.append( + m.HasIdCondition(has_id=[self._node_id_to_point_id(nid) for nid in ids]) + ) + response = self._client.query_points( + self._collection_name, + query=embedding, + query_filter=filt, + limit=len(ids), + with_payload=True, + with_vectors=False, + ) + hits = self._native_hits(response.points, "cosine") + if {hit["node_id"] for hit in hits} != set(ids): + raise NativeRetrievalError( + "integrity", "cosine evidence response is incomplete" + ) + return hits + + return await self._native_call(query) + def __init__(self, config: MemoryConfig): super().__init__(config) self._client = None # 同步 QdrantClient diff --git a/hy_memory/pipelines/_retrieval/config.py b/hy_memory/pipelines/_retrieval/config.py index a5a5a0c9..a17d1275 100644 --- a/hy_memory/pipelines/_retrieval/config.py +++ b/hy_memory/pipelines/_retrieval/config.py @@ -9,6 +9,8 @@ import os +from ...models.memory import MemoryLayer + # ======================================================================== # Reader 分发 @@ -16,8 +18,12 @@ READER_LEGACY = "legacy" READER_HYBRID = "hybrid" +READER_NATIVE = "native" + +ALL_READERS = (READER_LEGACY, READER_HYBRID, READER_NATIVE) + -ALL_READERS = (READER_LEGACY, READER_HYBRID) +VDB_RECALL_LAYERS = [MemoryLayer.L2_FACT, MemoryLayer.L3_SUMMARY, MemoryLayer.L4_IDENTITY] def resolve_reader_name(override: str = "") -> str: diff --git a/hy_memory/pipelines/_retrieval/evolution.py b/hy_memory/pipelines/_retrieval/evolution.py index f6cb4ade..f5fb0409 100644 --- a/hy_memory/pipelines/_retrieval/evolution.py +++ b/hy_memory/pipelines/_retrieval/evolution.py @@ -14,7 +14,10 @@ """ from datetime import datetime -from typing import Any, Dict, List, Optional, Sequence, Set +from typing import Any, Dict, List, Optional, Sequence, Set, TYPE_CHECKING + +if TYPE_CHECKING: + from ...data.vector_store_base import NativeQuery, VectorStoreBase import asyncio import logging @@ -23,6 +26,170 @@ logger = logging.getLogger(__name__) +def _native_evolution_edges(known: Dict[str, MemoryNode]) -> Dict[str, Set[str]]: + """Validate partial declarations before requesting another evolution frontier.""" + from ...retrieval import NativeRetrievalError + + outgoing = {nid: set() for nid in known} + for nid, node in known.items(): + for newer in node.superseded_by or []: + outgoing[nid].add(newer) + outgoing.setdefault(newer, set()) + for older in node.supersedes or []: + outgoing.setdefault(older, set()).add(nid) + + visiting, depths = set(), {} + + def depth(nid: str) -> int: + if nid in visiting: + raise NativeRetrievalError("cycle", "directed evolution cycle") + if nid in depths: + return depths[nid] + visiting.add(nid) + if len(visiting) > 33: + raise NativeRetrievalError("budget", "native evolution exceeds 32 hops") + value = max((1 + depth(newer) for newer in outgoing[nid]), default=0) + visiting.remove(nid) + if value > 32: + raise NativeRetrievalError("budget", "native evolution exceeds 32 hops") + depths[nid] = value + return value + + for nid in outgoing: + depth(nid) + + return outgoing + + +async def resolve_native_hits( + vector_store: "VectorStoreBase", + hits: List[Dict[str, Any]], + request: "NativeQuery", + *, + created_after: Optional[float] = None, +) -> List[Dict[str, Any]]: + """Resolve complete directed evolution components, preserving backend order. + + Reciprocal supersedes/superseded_by declarations describe the same edge. + User/source/layer visibility is checked on every fetched node; agent and + session scope apply only to the original retrieval seeds. + """ + from ...retrieval import NativeRetrievalError + from ...models.memory import MemoryStatus + from ...utils.source_filter import source_allowed + + def check_scope(node: MemoryNode) -> None: + if ( + node.user_id not in request.user_ids + or node.layer not in request.layers + or not source_allowed( + node.source, request.include_sources, request.exclude_sources + ) + ): + raise NativeRetrievalError( + "scope", "evolution component crosses the visible scope" + ) + + known = {} + for hit in hits: + node = hit.get("node") + if node is None or hit.get("node_id") != node.node_id: + raise NativeRetrievalError("integrity", "missing or mismatched native node") + check_scope(node) + known[node.node_id] = node + frontier = list(known.values()) + while frontier: + _native_evolution_edges(known) + missing = list( + dict.fromkeys( + nid for n in frontier for nid in _linked_ids(n) if nid not in known + ) + ) + if len(known) + len(missing) > 1024: + raise NativeRetrievalError("budget", "native evolution exceeds 1024 nodes") + if not missing: + break + fetched = await vector_store.native_get_by_ids(missing) + if len(fetched) != len(missing) or {n.node_id for n in fetched} != set(missing): + raise NativeRetrievalError("integrity", "incomplete evolution batch") + for node in fetched: + check_scope(node) + known[node.node_id] = node + frontier = fetched + + outgoing = _native_evolution_edges(known) + adjacency = {nid: set() for nid in known} + for older, newer_ids in outgoing.items(): + for newer in newer_ids: + adjacency[older].add(newer) + adjacency[newer].add(older) + + components = {} + for nid in known: + if nid in components: + continue + stack, members = [nid], set() + while stack: + current = stack.pop() + if current in members: + continue + members.add(current) + stack.extend(adjacency[current] - members) + sinks = [known[mid] for mid in members if not outgoing[mid]] + if len(sinks) != 1: + raise NativeRetrievalError( + "integrity", "evolution has no unique confirmed head" + ) + head = sinks[0] + # A writer shadow-only UPDATE (legacy DELETE) clears is_latest while + # preserving the links. Its unique sink tombstones the whole component; + # the output filter below must not fall back to a recalled ancestor. + # Validate the complete graph first, even when its sink is deleted. + if not head.is_latest and head.status != MemoryStatus.SHADOW: + raise NativeRetrievalError( + "integrity", "evolution has no unique confirmed head" + ) + for mid in members: + components[mid] = (head, members) + + output, seen = [], set() + for hit in hits: + head, members = components[hit["node_id"]] + if head.node_id in seen: + continue + seen.add(head.node_id) + if head.status != MemoryStatus.ACTIVE: + continue + when = head.memory_at or head.gmt_created + if ( + created_after is not None + and when is not None + and when.timestamp() < created_after + ): + continue + visible = [ + known[mid] + for mid in members + if mid != head.node_id + and known[mid].status in (MemoryStatus.ACTIVE, MemoryStatus.SUPERSEDED) + ] + # Keep the validated head first; order history like the existing readers. + visible.sort(key=_time_key, reverse=True) + output.append( + { + **hit, + "node": head, + "node_id": head.node_id, + "matched_id": hit.get("matched_id", hit["node_id"]), + "evolution_chain": [_node_to_chain_item(n) for n in [head] + visible] + if len(members) > 1 + else [], + "is_evolved": len(members) > 1, + } + ) + return output + + def _fmt_time(node: MemoryNode) -> Optional[float]: """返回 node.memory_at 的 Unix timestamp;缺失返回 None。""" t = node.memory_at diff --git a/hy_memory/pipelines/_retrieval/reconcile_retrieval.py b/hy_memory/pipelines/_retrieval/reconcile_retrieval.py index 526aa0d5..8454eaf5 100644 --- a/hy_memory/pipelines/_retrieval/reconcile_retrieval.py +++ b/hy_memory/pipelines/_retrieval/reconcile_retrieval.py @@ -37,6 +37,77 @@ logger = logging.getLogger(__name__) +async def prepare_native_candidates( + *, + new_memories: List[str], + search_queries: List[str], + vector_store: "VectorStoreBase", + embed_service: Any, + user_id: str, + layers: List[MemoryLayer], + source: str, + supplied_embeddings: Optional[Dict[str, List[float]]] = None, +) -> Tuple[ + Dict[str, MemoryNode], Dict[str, List[Tuple[str, float]]], Dict[str, List[float]] +]: + """One backend-ranked plan, strict representatives, optional cosine evidence.""" + from ...data.vector_store_base import NativeQuery + from ...retrieval import NativeRetrievalError + from .evolution import resolve_native_hits + + if not 1 <= len(new_memories) <= 16: + raise NativeRetrievalError( + "budget", "native reconcile supports at most 16 memories" + ) + texts = list(new_memories) + list(search_queries[:4]) + supplied = supplied_embeddings or {} + missing = [i for i, text in enumerate(texts) if not supplied.get(text)] + computed = ( + await embed_service.embed_batch([texts[i] for i in missing]) if missing else [] + ) + if len(computed) != len(missing): + raise NativeRetrievalError("integrity", "reconcile embedding count mismatch") + embeddings = [supplied.get(text) for text in texts] + for i, embedding in zip(missing, computed): + embeddings[i] = embedding + plan = NativeQuery( + texts=texts, + embeddings=embeddings, + user_ids=[user_id], + layers=layers, + limit=60, + include_sources=[source], + ) + hits = await vector_store.native_query(plan) + resolved = await resolve_native_hits(vector_store, hits, plan) + # Canonical state projections retain their exact-key owner. + resolved = [ + hit + for hit in resolved + if (hit["node"].custom or {}).get("memory_kind") != "canonical_state" + ][:20] + candidates = {hit["node_id"]: hit["node"] for hit in resolved} + per_memory = {} + for text, embedding in zip(new_memories, embeddings[: len(new_memories)]): + try: + evidence = await vector_store.native_cosine( + embedding, list(candidates), plan + ) + per_memory[text] = [(hit["node_id"], hit["score"]) for hit in evidence] + except Exception as exc: + # Only this optional graph evidence is allowed to fail open. + logger.warning( + "[reconciler] native cosine evidence unavailable: %s", + type(exc).__name__, + ) + per_memory[text] = [] + return ( + candidates, + per_memory, + dict(zip(new_memories, embeddings[: len(new_memories)])), + ) + + def _float_env(name: str, default: float) -> float: raw = os.getenv(name) if raw is None or not str(raw).strip(): diff --git a/hy_memory/pipelines/base.py b/hy_memory/pipelines/base.py index f6ae3c80..5aa7459b 100644 --- a/hy_memory/pipelines/base.py +++ b/hy_memory/pipelines/base.py @@ -462,6 +462,11 @@ class ReadRequest: # 版本特有参数 (V2: token_budget, use_llm_understanding 等) extra: Dict[str, Any] = field(default_factory=dict) + # Append native options to preserve the public positional argument order. + raw_query: Optional[str] = None + native_mode: str = "hybrid" + native_threshold: Any = None + def __post_init__(self) -> None: from ..utils.source_filter import normalize_source_filters self.include_sources, self.exclude_sources = normalize_source_filters( diff --git a/hy_memory/pipelines/reader.py b/hy_memory/pipelines/reader.py index 7143a1a4..d5821133 100644 --- a/hy_memory/pipelines/reader.py +++ b/hy_memory/pipelines/reader.py @@ -34,9 +34,13 @@ def _build_impl( vector_store: Optional[VectorStoreBase], graph_store: Optional[GraphStoreBase] = None, cache: Any = None, + reader_name: str = "", ) -> ReadPipeline: """按环境变量选择内部实现类并构造实例。""" - name = _retrieval_config.resolve_reader_name() + name = _retrieval_config.resolve_reader_name(reader_name) + if name == _retrieval_config.READER_NATIVE: + from .reader_native import NativeReadPipeline + return NativeReadPipeline(config, embed_service, vector_store) if name == _retrieval_config.READER_HYBRID: try: from .reader_hybrid import HybridReadPipeline @@ -73,10 +77,12 @@ def __init__( vector_store: Optional[VectorStoreBase] = None, graph_store: Optional[GraphStoreBase] = None, cache: Any = None, + reader_name: str = "", ): self.config = config self._impl: ReadPipeline = _build_impl( config, embed_service, vector_store, graph_store=graph_store, cache=cache, + reader_name=reader_name, ) logger.info( f"[reader-dispatch] active impl = {self._impl.__class__.__name__} " diff --git a/hy_memory/pipelines/reader_base.py b/hy_memory/pipelines/reader_base.py index 16bf1300..b726b40c 100644 --- a/hy_memory/pipelines/reader_base.py +++ b/hy_memory/pipelines/reader_base.py @@ -28,6 +28,7 @@ import asyncio import json import os +from ._retrieval.config import VDB_RECALL_LAYERS import logging from .base import ReadPipeline, ReadRequest, ReadResponse, PipelineContext @@ -143,11 +144,7 @@ # 参与语义+BM25 融合排序,最终归入 normal 通道(不再算 profile)。 # 与隐藏 L0 / Graph L6 通道互斥,杜绝双重召回。 # 不含 L1_RAW(原始对话,append-only 不召回)、L5_KNOWLEDGE、L7(graph 专属通道)。 -_VDB_RECALL_LAYERS = [ - MemoryLayer.L2_FACT, - MemoryLayer.L3_SUMMARY, - MemoryLayer.L4_IDENTITY, -] +_VDB_RECALL_LAYERS = VDB_RECALL_LAYERS def _graph_custom(hit: Dict[str, Any]) -> Dict[str, Any]: diff --git a/hy_memory/pipelines/reader_native.py b/hy_memory/pipelines/reader_native.py new file mode 100644 index 00000000..5689ca72 --- /dev/null +++ b/hy_memory/pipelines/reader_native.py @@ -0,0 +1,135 @@ +"""Read-only native retrieval using the factory's shared data and embed services.""" + +import time +from typing import Any, Optional + +from .base import PipelineContext, ReadPipeline, ReadRequest, ReadResponse +from ._retrieval.config import VDB_RECALL_LAYERS +from ._retrieval.evolution import resolve_native_hits +from ..data.vector_store_base import NativeQuery, validate_native_scope +from ..data.vector_store_base import VectorStoreBase +from ..config import MemoryConfig +from ..retrieval import NativeRetrievalError, validate_native_threshold +from ..utils.tracer import PipelineTracer + + +class NativeReadPipeline(ReadPipeline): + """Qdrant Server native ranking; no legacy postprocessing or resource ownership.""" + + VERSION = "native" + + def __init__( + self, + config: MemoryConfig, + embed_service: Any, + vector_store: Optional[VectorStoreBase], + ) -> None: + self.config = config + self.embed_service = embed_service + self.vector_store = vector_store + + async def initialize(self) -> None: + # Registry owns initialization of the shared store. Preparation is explicit. + if self.vector_store is None: + raise NativeRetrievalError( + "unsupported", "native reader requires a vector store" + ) + if not getattr(self.vector_store, "_client", None): + await self.vector_store.initialize() + + async def close(self) -> None: + """Shared resources are closed by their owner.""" + + async def read( + self, + request: ReadRequest, + ctx: Optional[PipelineContext] = None, + tracer: Optional[PipelineTracer] = None, + ) -> ReadResponse: + from ..models.memory import MemoryLayer + + started = time.perf_counter() + users = request.user_ids or ([request.user_id] if request.user_id else []) + if type(request.limit) is not int or not 0 <= request.limit <= 100: + raise NativeRetrievalError( + "invalid_request", "native public limit must be 0..100" + ) + validate_native_threshold(request.native_threshold, request.native_mode) + raw = request.raw_query if request.raw_query is not None else request.query + if not isinstance(raw, str): + raise NativeRetrievalError( + "invalid_request", "native query must be a string" + ) + layers = ( + [MemoryLayer(layer) for layer in request.layers] + if request.layers + else list(VDB_RECALL_LAYERS) + ) + plan = NativeQuery( + texts=[raw], + embeddings=[], + user_ids=users, + layers=layers, + mode=request.native_mode, + limit=max(30, 3 * request.limit), + agent_ids=request.agent_ids + or ([request.agent_id] if request.agent_id else []), + session_ids=request.session_ids + or ([request.session_id] if request.session_id else []), + include_sources=request.include_sources, + exclude_sources=request.exclude_sources, + ) + # Validate scope and mode even for skipped requests, before embedding. + validate_native_scope(plan) + if not raw.strip() or request.limit == 0: + return ReadResponse(success=True, extra={"native": {"skipped": True}}) + if request.native_mode != "keyword": + embedding = request.query_embedding + if embedding is None: + embedding = await self.embed_service.embed(request.query) + plan.embeddings = [embedding] + hits = await self.vector_store.native_query(plan) + resolved = await resolve_native_hits( + self.vector_store, hits, plan, created_after=request.created_after + ) + threshold = request.native_threshold + memories = [] + for hit in resolved: + if threshold is not None and not hit["score"] > threshold.value: + continue + node = hit["node"] + memories.append( + { + "memory_id": node.node_id, + "content": node.content, + "layer": node.layer.value, + "score": hit["score"], + "score_kind": hit["score_kind"], + "mode": request.native_mode, + "config_id": hit["config_id"], + "matched_id": hit["matched_id"], + "user_id": node.user_id, + "agent_id": node.agent_id, + "session_id": node.session_id, + "source": node.source, + "tags": list(node.tags or []), + "owner": node.owner, + "speculate": node.speculate, + "gmt_created": node.gmt_created.timestamp() + if node.gmt_created + else None, + "meta": (node.custom or {}).get("meta") or [], + "memory_at": node.memory_at.timestamp() if node.memory_at else None, + "evolution_chain": hit["evolution_chain"], + "is_evolved": hit["is_evolved"], + } + ) + if len(memories) >= request.limit: + break + return ReadResponse( + success=True, + memories=memories, + total_found=len(memories), + elapsed_ms=(time.perf_counter() - started) * 1000, + extra={"native": {"skipped": False, "mode": request.native_mode}}, + ) diff --git a/hy_memory/pipelines/registry.py b/hy_memory/pipelines/registry.py index 221d399d..db77fafe 100644 --- a/hy_memory/pipelines/registry.py +++ b/hy_memory/pipelines/registry.py @@ -136,10 +136,6 @@ async def get_reader(self, reader_name: str = "", **kwargs) -> ReadPipeline: if effective_name in self._readers: return self._readers[effective_name] - # 向后兼容:无指定时走旧的 _reader 单例 - if not reader_name and self._reader is not None: - return self._reader - reader_kwargs = { "config": self.config, "embed_service": self._get_shared_embed_service(), @@ -149,19 +145,9 @@ async def get_reader(self, reader_name: str = "", **kwargs) -> ReadPipeline: if self._shared_graph_store is not None: reader_kwargs["graph_store"] = self._shared_graph_store - # 用 _build_impl 直接构建指定 reader(绕过 MemoryReader dispatcher) - import os - old_env = os.environ.get("HY_MEMORY_READER", "") - os.environ["HY_MEMORY_READER"] = effective_name - try: - from .reader import MemoryReader - reader = MemoryReader(**reader_kwargs) - await reader.initialize() - finally: - if old_env: - os.environ["HY_MEMORY_READER"] = old_env - else: - os.environ.pop("HY_MEMORY_READER", None) + from .reader import MemoryReader + reader = MemoryReader(**reader_kwargs, reader_name=effective_name) + await reader.initialize() # 缓存 self._readers[effective_name] = reader diff --git a/hy_memory/retrieval.py b/hy_memory/retrieval.py index b6321325..139e1581 100644 --- a/hy_memory/retrieval.py +++ b/hy_memory/retrieval.py @@ -1,6 +1,8 @@ """Public types and errors for exact memory-ID lookup.""" -from typing import Literal, Sequence +from dataclasses import dataclass +import math +from typing import Literal, Optional, Sequence MemoryKind = Literal["chat", "experience", "fact"] @@ -20,4 +22,58 @@ def __init__(self, memory_id: str, namespaces: Sequence[str]): ) -__all__ = ["MemoryKind", "MemoryVariant", "AmbiguousMemoryIdError"] +class NativeRetrievalError(RuntimeError): + """Native retrieval failed closed; ``reason`` is a stable machine-readable code.""" + + def __init__(self, reason: str, message: str) -> None: + self.reason = reason + super().__init__(message) + + +@dataclass(frozen=True) +class NativeScoreThreshold: + """Exclusive raw-score threshold bound to a mode and query configuration.""" + + value: float + mode: str + score_kind: str + config_id: str + + def __post_init__(self) -> None: + if ( + isinstance(self.value, bool) + or not isinstance(self.value, (int, float)) + or not math.isfinite(self.value) + ): + raise NativeRetrievalError("invalid_request", "threshold must be finite") + + +NATIVE_CONFIG_ID = "qdrant-native-v1-multilingual-lowercase-rrf2" + + +def native_score_kind(mode: str) -> str: + return "cosine" if mode == "dense" else "rrf" + + +def validate_native_threshold( + threshold: Optional[NativeScoreThreshold], mode: str +) -> None: + if threshold is None: + return + if not isinstance(threshold, NativeScoreThreshold) or ( + threshold.mode, + threshold.score_kind, + threshold.config_id, + ) != (mode, native_score_kind(mode), NATIVE_CONFIG_ID): + raise NativeRetrievalError( + "invalid_request", "native threshold mode, score kind or config mismatch" + ) + + +__all__ = [ + "MemoryKind", + "MemoryVariant", + "AmbiguousMemoryIdError", + "NativeRetrievalError", + "NativeScoreThreshold", +] diff --git a/hy_memory/utils/model_call_context.py b/hy_memory/utils/model_call_context.py index 3821d985..e840967e 100644 --- a/hy_memory/utils/model_call_context.py +++ b/hy_memory/utils/model_call_context.py @@ -1,5 +1,7 @@ """Context propagated to host-provided HTTP transports for cost accounting.""" +from __future__ import annotations + from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar, Token diff --git a/memhub/apps/core/tests/test_search_without_profile.py b/memhub/apps/core/tests/test_search_without_profile.py index f0749bda..1d6542f6 100644 --- a/memhub/apps/core/tests/test_search_without_profile.py +++ b/memhub/apps/core/tests/test_search_without_profile.py @@ -2,6 +2,7 @@ import asyncio from pathlib import Path +from threading import Event from types import SimpleNamespace import pytest @@ -139,6 +140,7 @@ def __init__(self, data_dir: Path) -> None: self.data_dir = data_dir self.opened: list[tuple[str, int | None, str | None]] = [] self.closed: list[str] = [] + self.closed_event = Event() def __call__( self, @@ -156,9 +158,14 @@ def __call__( config = build_sdk_config(profile, space, mode, api_key=api_key, data_dir=data_dir) name = config.vector_store.collection_name self.opened.append((name, config.vector_store.embedding_dims, api_key)) + + def close() -> None: + self.closed.append(name) + self.closed_event.set() + return SimpleNamespace( client=SimpleNamespace(collection=name), - close=lambda: self.closed.append(name), + close=close, ) @@ -277,8 +284,8 @@ async def read() -> str: finish.set() assert await in_flight == old - for _ in range(5): - await asyncio.sleep(0) + # Closing runs in a worker thread; loop turns do not establish completion. + assert await asyncio.to_thread(recorder.closed_event.wait, 5.0) assert recorder.closed == [old] await handle.close() assert recorder.closed == [old, new] diff --git a/memhub/apps/core/tests/test_source_sync_permissions.py b/memhub/apps/core/tests/test_source_sync_permissions.py index 743dea4e..1b4fe26f 100644 --- a/memhub/apps/core/tests/test_source_sync_permissions.py +++ b/memhub/apps/core/tests/test_source_sync_permissions.py @@ -197,11 +197,17 @@ async def scenario(): for job in connected.json()["jobs"]: await app.state.import_supervisor.wait(job["job_id"]) assert app.state.evidence_store.count() == 1 - assert app.state.live_sync_watch is not None + watcher = app.state.live_sync_watch + assert watcher is not None write_turn(root, "after-thaw") - synced = await app.state.live_sync_supervisor.trigger("claude-code") - assert synced is not None + # The OS watcher can finish before an explicit automatic trigger; + # a settled source correctly returns no job on that second call. + changed_sources = await asyncio.wait_for( + watcher.handle_changes({root / "after-thaw.jsonl"}), timeout=10 + ) + assert changed_sources == ("claude-code",) assert app.state.evidence_store.count() == 2 + assert await app.state.live_sync_supervisor.trigger("claude-code") is None grant = app.state.source_grant_store.require_receipt( user_id="local", source_id="claude-code" ) @@ -211,6 +217,7 @@ async def scenario(): ) assert stopped.status_code == 200 and not stopped.json()["sync_enabled"] assert app.state.live_sync_watch is None + assert watcher._task is None write_turn(root, "after-sync-disabled") assert await app.state.live_sync_supervisor.trigger("claude-code") is None assert app.state.evidence_store.count() == 2 diff --git a/pyproject.toml b/pyproject.toml index 210ac70a..b1e1ba84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ [project.optional-dependencies] qdrant = ["qdrant-client", "fastembed"] +qdrant-native = ["qdrant-client==1.20.0; python_version >= '3.10'"] faiss = ["faiss-cpu"] graph = ["neo4j"] mysql = ["aiomysql"] diff --git a/tests/native_retrieval/conftest.py b/tests/native_retrieval/conftest.py new file mode 100644 index 00000000..1fdd05c7 --- /dev/null +++ b/tests/native_retrieval/conftest.py @@ -0,0 +1,142 @@ +"""Local native fixtures. Integration uses only an explicit dedicated Server URL.""" + +import asyncio +import json +import os +import time +import uuid +from types import SimpleNamespace +from urllib.parse import urlparse + +import pytest + +from hy_memory.agent.llm_provider import LLMResponse +from hy_memory.client import HyMemoryClient +from hy_memory.config import MemoryConfig +from hy_memory.data.vector_store_qdrant import QdrantVectorStore +from hy_memory.models.memory import MemoryLayer, MemoryNode +from hy_memory.pipelines.registry import ComponentFactory + + +class FixedEmbed: + def __init__(self): + self.calls = [] + + async def embed(self, text, **kwargs): + self.calls.append(text) + return [1.0, 0.0] + + async def embed_batch(self, texts, **kwargs): + return [await self.embed(text) for text in texts] + + async def embed_with_meta(self, text, **kwargs): + return await self.embed(text), {"mock": True} + + +class CaptureLLM: + def __init__(self, ops=None): + self.prompts = [] + self.ops = ops or [] + + async def complete(self, prompt, **kwargs): + self.prompts.append(prompt) + return LLMResponse(content=json.dumps(self.ops), tokens_used=1) + + +def node(content="ordinary", *, node_id=None, **kwargs): + return MemoryNode( + node_id=node_id or uuid.uuid4().hex, + content=content, + user_id=kwargs.pop("user_id", "alice"), + agent_id=kwargs.pop("agent_id", "a"), + session_id=kwargs.pop("session_id", "s"), + layer=kwargs.pop("layer", MemoryLayer.L2_FACT), + embedding=kwargs.pop("embedding", [1.0, 0.0]), + **kwargs, + ) + + +def public_client(store, embed=None): + client = HyMemoryClient.__new__(HyMemoryClient) + client._config = store.config + client._mode = "pro" + client._registry = ComponentFactory( + store.config, embedding_service=embed or FixedEmbed() + ) + client._registry._shared_vector_store = store + client._registry._shared_vector_store_initialized = True + client._vector_store = store + client._cache = None + client._history_store = None + client._coding_llm_provider = CaptureLLM() + client._search_loop_thread = SimpleNamespace(_loop=asyncio.get_running_loop()) + return client + + +@pytest.fixture +async def server_store(): + from qdrant_client import QdrantClient + + url = os.environ.get("HY_MEMORY_TEST_QDRANT_URL") + if not url: + pytest.fail("integration requires dedicated HY_MEMORY_TEST_QDRANT_URL") + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + pytest.fail("HY_MEMORY_TEST_QDRANT_URL must be a Server URL") + client = QdrantClient(url=url, timeout=30) + deadline = time.monotonic() + 20 + while True: + try: + info = client.info() + break + except Exception: + if time.monotonic() >= deadline: + client.close() + pytest.fail("dedicated Qdrant Server did not become ready within 20s") + await asyncio.sleep(0.1) + assert info.version == "1.20.0", f"uncertified test Server {info.version}" + config = MemoryConfig() + config.coding.enable = False + config.llm.enable_query_rewrite = False + config.vector_store.embedding_dims = 2 + config.vector_store.collection_name = "hy_native_test_" + uuid.uuid4().hex + store = QdrantVectorStore(config) + # The only connection is the explicit test URL, never config/from_env. + store._client = client + from qdrant_client import models as m + + try: + client.create_collection( + store._collection_name, + vectors_config=m.VectorParams(size=2, distance=m.Distance.COSINE), + ) + for field in ( + "user_id", + "agent_id", + "session_id", + "isolation_key", + "source", + "layer", + "status", + ): + client.create_payload_index( + store._collection_name, field, field_schema="keyword", wait=True + ) + await store.prepare_native_retrieval() + yield store + finally: + if client.collection_exists(store._collection_name): + client.delete_collection(store._collection_name) + client.close() + + +@pytest.fixture +async def populated(server_store): + store = server_store + target = node( + "ECONNRESET reconnectmarker src/network/reconnect.py 中文关键词", + embedding=[0.0, 1.0], + ) + distractors = [node(f"unrelated distractor {i}") for i in range(35)] + await store.upsert_batch(distractors + [target]) + return store, target diff --git a/tests/native_retrieval/test_errors.py b/tests/native_retrieval/test_errors.py new file mode 100644 index 00000000..139002c5 --- /dev/null +++ b/tests/native_retrieval/test_errors.py @@ -0,0 +1,395 @@ +import asyncio +import os +from dataclasses import fields +from datetime import datetime +from types import SimpleNamespace + +import pytest + +from hy_memory.agent.reconciler import MemoryReconciler +from hy_memory.config import MemoryConfig, RecallConfig +from hy_memory.data.qdrant_native import check_version +from hy_memory.retrieval import NATIVE_CONFIG_ID as CONFIG_ID +from hy_memory.retrieval import validate_native_threshold as validate_threshold +from hy_memory.data.vector_store_base import validate_native_query as validate_query +from hy_memory.data.vector_store_base import NativeQuery, VectorStoreBase +from hy_memory.data.vector_store_qdrant import QdrantVectorStore +from hy_memory.models.memory import MemoryLayer, MemoryStatus +from hy_memory.pipelines._retrieval.evolution import resolve_native_hits +from hy_memory.pipelines.base import ReadResponse +from hy_memory.pipelines.registry import ComponentFactory +from hy_memory.retrieval import NativeRetrievalError, NativeScoreThreshold +from conftest import CaptureLLM, FixedEmbed, node, public_client + + +def plan(**kwargs): + return NativeQuery( + texts=["query"], + embeddings=[[1.0, 0.0]], + user_ids=["alice"], + layers=[MemoryLayer.L2_FACT], + **kwargs, + ) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), True, "0.5"]) +def test_T08_nonfinite_and_untyped_threshold(value): + with pytest.raises(NativeRetrievalError): + NativeScoreThreshold(value, "hybrid", "rrf", CONFIG_ID) + + +@pytest.mark.parametrize( + "threshold", + [ + 0.5, + NativeScoreThreshold(0, "dense", "cosine", CONFIG_ID), + NativeScoreThreshold(0, "hybrid", "rrf", "wrong"), + ], +) +def test_T08_mismatched_threshold(threshold): + with pytest.raises(NativeRetrievalError): + validate_threshold(threshold, "hybrid") + + +@pytest.mark.parametrize( + "field,value", + [ + ("user_ids", []), + ("mode", "invalid"), + ("embeddings", []), + ("embeddings", [[float("nan")]]), + ("limit", 301), + ], +) +def test_invalid_plan(field, value): + query = plan() + setattr(query, field, value) + with pytest.raises(NativeRetrievalError): + validate_query(query) + + +async def test_unsupported_base_and_local(): + with pytest.raises(NativeRetrievalError, match="Qdrant Server"): + await VectorStoreBase.native_query(object(), plan()) + with pytest.raises(NativeRetrievalError) as exc: + check_version(None, True) + assert exc.value.reason == "unsupported" + + +def test_T18_python39_and_version_rejection(monkeypatch): + monkeypatch.setattr("hy_memory.data.vector_store_base.sys.version_info", (3, 9)) + with pytest.raises(NativeRetrievalError, match="Python"): + validate_query(plan()) + monkeypatch.setattr("hy_memory.data.vector_store_base.sys.version_info", (3, 12)) + monkeypatch.setattr("hy_memory.data.qdrant_native.version", lambda _: "1.19.1") + with pytest.raises(NativeRetrievalError): + check_version(None, False) + monkeypatch.setattr("hy_memory.data.qdrant_native.version", lambda _: "1.20.0") + with pytest.raises(NativeRetrievalError): + check_version( + SimpleNamespace(info=lambda: SimpleNamespace(version="1.19.1")), False + ) + + +@pytest.mark.parametrize( + "option,value", + [ + ("min_score", 0), + ("schema_limit", 1), + ("intention_limit", 1), + ("rerank", True), + ("native_mode", "bad"), + ("limit", -1), + ("limit", 101), + ("scene", "all"), + ("scene", "working"), + ("layers", ["l1_raw"]), + ], +) +async def test_public_rejects_before_backend(option, value): + store = SimpleNamespace(config=MemoryConfig()) + client = public_client(store) + kwargs = dict(reader="native", user_ids=["alice"]) + kwargs[option] = value + with pytest.raises((NativeRetrievalError, ValueError)): + await client.async_search("query", **kwargs) + + +async def test_T18_legacy_omitted_and_explicit_zero(monkeypatch): + store = SimpleNamespace(config=MemoryConfig()) + client = public_client(store) + requests = [] + + class Reader: + VERSION = "legacy" + + async def read(self, request): + requests.append(request) + return ReadResponse(success=True) + + async def reader(**kwargs): + return Reader() + + monkeypatch.setattr(client._registry, "get_reader", reader) + monkeypatch.setattr(client, "_spawn_search_dedup", lambda *args, **kwargs: None) + await client.async_search("query", user_ids=["alice"]) + assert (requests[-1].min_score, requests[-1].schema_limit, requests[-1].rerank) == ( + 0.4, + 10, + True, + ) + await client.async_search( + "query", user_ids=["alice"], min_score=0, schema_limit=0, rerank=False + ) + assert (requests[-1].min_score, requests[-1].schema_limit, requests[-1].rerank) == ( + 0, + 0, + False, + ) + + +async def test_T17_factory_concurrent_names_without_environment_mutation(monkeypatch): + config = MemoryConfig() + factory = ComponentFactory(config, embedding_service=FixedEmbed()) + factory._shared_vector_store = SimpleNamespace(_client=True) + monkeypatch.setenv("HY_MEMORY_READER", "legacy") + before = dict(os.environ) + native, legacy = await asyncio.gather( + factory.get_reader("native"), factory.get_reader("legacy") + ) + assert native.impl.VERSION == "native" and legacy.impl.VERSION == "legacy" + assert os.environ == before + + +async def test_T17_illegal_reconcile_configuration(): + with pytest.raises(ValueError): + MemoryConfig.from_dict({"recall": {"reconcile_retriever": "typo"}}) + config = MemoryConfig() + config.recall.reconcile_retriever = "typo" + llm = CaptureLLM() + result = await MemoryReconciler(config, llm).reconcile( + ["new"], "alice", "a", object(), FixedEmbed() + ) + assert not result.success and result.ops == [] and not llm.prompts + + +async def test_recall_positional_arguments_preserve_legacy_reconcile(): + config = MemoryConfig() + config.recall = RecallConfig(10, 100, 7) + assert config.recall.default_limit_per_layer == 7 + assert config.recall.reconcile_retriever == "default" + assert [field.name for field in fields(RecallConfig)][-2:] == [ + "min_score_threshold", "reconcile_retriever", + ] + config.recall.validate_reconcile_retriever() + + class LegacyStore: + calls = 0 + + async def search_batch(self, queries): + self.calls += 1 + return [[] for _ in queries] + + store, llm = LegacyStore(), CaptureLLM() + rec = MemoryReconciler(config, llm) + rec._enable_search_query = False + result = await rec.reconcile(["new fact"], "alice", "a", store, FixedEmbed()) + assert result.success and store.calls == 1 + assert [op.op for op in result.ops] == ["ADD"] + assert not llm.prompts + + +@pytest.mark.parametrize("query,limit", [(" ", 10), ("query", 0), ("", 10)]) +async def test_skipped_before_reader_embedding_or_scene_llm(query, limit, monkeypatch): + client = public_client(SimpleNamespace(config=MemoryConfig())) + + async def forbidden(*args, **kwargs): + raise AssertionError("skipped request must perform no work") + + monkeypatch.setattr(client._registry, "get_reader", forbidden) + monkeypatch.setattr("hy_memory.coding.judge.analyze_queries", forbidden) + result = await client.async_search( + query, reader="native", user_ids=["alice"], search_mode="agentic", limit=limit + ) + assert result["native"]["skipped"] + assert not result["memories"]["normal"] + + +class StrictStore: + def __init__(self, nodes): + self.nodes = {n.node_id: n for n in nodes} + + async def native_get_by_ids(self, ids): + return [self.nodes[nid] for nid in ids if nid in self.nodes] + + +def hit(n): + return dict(node=n, node_id=n.node_id, score=0.123456789, matched_id=n.node_id) + + +async def test_T07_reciprocal_and_multi_ancestor_are_not_cycles(): + a, b, head = node(), node(), node() + for old in (a, b): + old.superseded_by, old.is_latest = [head.node_id], False + old.status = MemoryStatus.SUPERSEDED + head.supersedes = [a.node_id, b.node_id] + result = await resolve_native_hits( + StrictStore([a, b, head]), [hit(a), hit(b)], plan() + ) + assert len(result) == 1 and result[0]["node_id"] == head.node_id + assert result[0]["score"] == 0.123456789 and result[0]["matched_id"] == a.node_id + + +async def test_deleted_native_head_drops_whole_component_keeps_healthy_hit(): + head = node(status=MemoryStatus.SHADOW, is_latest=False) + old = node( + status=MemoryStatus.SUPERSEDED, + is_latest=False, + superseded_by=[head.node_id], + ) + head.supersedes = [old.node_id] + healthy = node() + result = await resolve_native_hits( + StrictStore([old, head, healthy]), + [hit(old), hit(healthy), hit(head)], + plan(), + ) + assert [item["node_id"] for item in result] == [healthy.node_id] + assert result[0]["score"] == hit(healthy)["score"] + + +@pytest.mark.parametrize("status", [s for s in MemoryStatus if s != MemoryStatus.SHADOW]) +async def test_non_deleted_native_head_still_requires_latest(status): + head = node(status=status, is_latest=False) + with pytest.raises(NativeRetrievalError) as failure: + await resolve_native_hits(StrictStore([head]), [hit(head)], plan()) + assert failure.value.reason == "integrity" + + +@pytest.mark.parametrize("time_field", ["memory_at", "gmt_created", "mixed"]) +async def test_public_native_evolution_chain_orders_history_by_time(time_field): + oldest, middle, head = [ + node(node_id=nid, **{ + "gmt_created": datetime(2026, 11, day), + "memory_at": datetime(2026, 11, day) + if time_field == "memory_at" or (time_field == "mixed" and day == 2) + else None, + }) + for nid, day in [("a-oldest", 1), ("b-middle", 2), ("z-head", 3)] + ] + for old, newer in [(oldest, middle), (middle, head)]: + old.superseded_by = [newer.node_id] + old.is_latest = False + old.status = MemoryStatus.SUPERSEDED + newer.supersedes = [old.node_id] + + class Store(StrictStore): + config = MemoryConfig() + _client = True + + async def native_query(self, request): + return [{**hit(oldest), "score_kind": "rrf", "config_id": CONFIG_ID}] + + result = await public_client(Store([oldest, middle, head])).async_search( + "query", reader="native", native_mode="keyword", user_ids=["alice"] + ) + memories = result["memories"]["normal"] + assert [m["memory_id"] for m in memories] == [head.node_id] + assert [n["node_id"] for n in memories[0]["evolution_chain"]] == [ + head.node_id, middle.node_id, oldest.node_id, + ] + assert memories[0]["matched_id"] == oldest.node_id + assert memories[0]["score"] == 0.123456789 + + +@pytest.mark.parametrize( + "case", ["missing", "cycle", "scope", "ambiguous", "hops", "nodes"] +) +@pytest.mark.parametrize("shadow", [False, True]) +async def test_T07_integrity_and_budgets(case, shadow): + seed = node(status=MemoryStatus.SHADOW if shadow else MemoryStatus.ACTIVE) + nodes = [seed] + if case == "missing": + seed.superseded_by = ["missing"] + elif case == "cycle": + b = node(superseded_by=[seed.node_id]) + seed.superseded_by = [b.node_id] + nodes.append(b) + elif case == "scope": + head = node(user_id="other") + seed.superseded_by = [head.node_id] + nodes.append(head) + elif case == "ambiguous": + nodes += [node(), node()] + seed.superseded_by = [n.node_id for n in nodes[1:]] + elif case == "hops": + for _ in range(33): + head = node() + nodes[-1].superseded_by = [head.node_id] + nodes[-1].is_latest = False + nodes.append(head) + else: + nodes += [node() for _ in range(1024)] + seed.superseded_by = [n.node_id for n in nodes[1:]] + if shadow: + for n in nodes: + n.status = MemoryStatus.SHADOW + n.is_latest = False + with pytest.raises(NativeRetrievalError): + await resolve_native_hits(StrictStore(nodes), [hit(seed)], plan()) + + +def test_T12_native_response_id_mismatch_and_nonfinite(): + store = QdrantVectorStore(MemoryConfig()) + n = node() + payload = store._node_to_payload(n) + for point in [ + SimpleNamespace(payload=payload, id="wrong", score=0.5), + SimpleNamespace( + payload=payload, + id=store._node_id_to_point_id(n.node_id), + score=float("nan"), + ), + ]: + with pytest.raises(NativeRetrievalError): + store._native_hits([point], "rrf") + + +@pytest.mark.parametrize( + "error", [PermissionError("401"), TimeoutError("query timed out")] +) +async def test_T12_backend_failure_is_never_empty_success(error): + store = QdrantVectorStore(MemoryConfig()) + + def fail(): + raise error + + with pytest.raises(NativeRetrievalError) as failure: + await store._native_call(fail) + assert failure.value.reason == "backend" + # Python 3.12's asyncio Future conversion reconstructs TimeoutError when + # transferring it from the executor; its identity is not an SDK contract. + # Preserve the diagnostic exception type and arguments on every runtime. + cause = failure.value.__cause__ + assert cause is not None + assert type(cause) is type(error) + assert cause.args == error.args + + +async def test_hop_budget_stops_reads_early(): + nodes = [node() for _ in range(100)] + for old, newer in zip(nodes, nodes[1:]): + old.superseded_by = [newer.node_id] + old.is_latest = False + store = StrictStore(nodes) + calls = [] + original = store.native_get_by_ids + + async def record(ids): + calls.append(ids) + return await original(ids) + + store.native_get_by_ids = record + with pytest.raises(NativeRetrievalError) as failure: + await resolve_native_hits(store, [hit(nodes[0])], plan()) + assert failure.value.reason == "budget" and len(calls) <= 32 diff --git a/tests/native_retrieval/test_public_search.py b/tests/native_retrieval/test_public_search.py new file mode 100644 index 00000000..8f66cc99 --- /dev/null +++ b/tests/native_retrieval/test_public_search.py @@ -0,0 +1,271 @@ +import asyncio +from datetime import datetime + +import pytest + +from hy_memory.retrieval import NATIVE_CONFIG_ID as CONFIG_ID +from hy_memory.models.memory import MemoryStatus +from hy_memory.retrieval import NativeRetrievalError, NativeScoreThreshold +from conftest import node, public_client + +pytestmark = pytest.mark.integration + + +async def test_T01_dense_pool_escape_public(populated): + store, target = populated + client = public_client(store) + dense = await client.async_search( + "reconnectmarker", + reader="native", + native_mode="dense", + user_ids=["alice"], + limit=10, + ) + assert target.node_id not in {m["memory_id"] for m in dense["memories"]["normal"]} + # Prove the full dense top-30 excludes the target, independently of public limit. + from hy_memory.data.vector_store_base import NativeQuery + from hy_memory.models.memory import MemoryLayer + + raw_dense = await store.native_query( + NativeQuery( + texts=["reconnectmarker"], + embeddings=[[1.0, 0.0]], + user_ids=["alice"], + layers=[MemoryLayer.L2_FACT], + mode="dense", + limit=30, + ) + ) + assert len(raw_dense) == 30 + assert target.node_id not in {h["node_id"] for h in raw_dense} + for mode in ("keyword", "hybrid"): + result = await client.async_search( + "reconnectmarker", reader="native", native_mode=mode, user_ids=["alice"] + ) + assert target.node_id in {m["memory_id"] for m in result["memories"]["normal"]} + + +async def test_T02_tags_only_public(server_store): + target = node(tags=["tagonlymarker"], embedding=[0.0, 1.0]) + await server_store.upsert_batch( + [node(f"distractor {i}") for i in range(35)] + [target] + ) + result = await public_client(server_store).async_search( + "tagonlymarker", reader="native", user_ids=["alice"] + ) + assert target.node_id in {m["memory_id"] for m in result["memories"]["normal"]} + + +async def test_T04_raw_lexical_query_survives_rewrite_no_keyword_embedding( + populated, monkeypatch +): + store, target = populated + + class NoEmbed: + async def embed(self, *args, **kwargs): + raise AssertionError("keyword must not embed") + + client = public_client(store, NoEmbed()) + + async def rewrite(*args, **kwargs): + return dict( + ok=True, scene="chat", rewrite_query="network problem", created_after=None + ) + + monkeypatch.setattr("hy_memory.coding.judge.analyze_queries", rewrite) + result = await client.async_search( + queries=["context", "ECONNRESET"], + search_mode="agentic", + reader="native", + native_mode="keyword", + user_ids=["alice"], + ) + assert result["rewrite"]["effective_query"] == "network problem" + assert result["memories"]["normal"][0]["memory_id"] == target.node_id + + +async def test_T06_cross_agent_head_and_T07_final_time(server_store): + store = server_store + head = node("current state", agent_id="b", memory_at=datetime(2026, 11, 10)) + old = node( + "historicalmarker", + memory_at=datetime(2026, 11, 1), + status=MemoryStatus.SUPERSEDED, + is_latest=False, + superseded_by=[head.node_id], + ) + head.supersedes = [old.node_id] + await store.upsert_batch([head, old]) + client = public_client(store) + opts = dict( + reader="native", native_mode="keyword", user_ids=["alice"], agent_ids=["a"] + ) + result = await client.async_search( + "historicalmarker", created_after=datetime(2026, 11, 5).timestamp(), **opts + ) + hit = result["memories"]["normal"][0] + assert hit["memory_id"] == head.node_id and hit["matched_id"] == old.node_id + assert hit["agent_id"] == "b" + assert len(hit["evolution_chain"]) == 2 + assert not ( + await client.async_search( + "historicalmarker", created_after=datetime(2026, 11, 11).timestamp(), **opts + ) + )["memories"]["normal"] + # Equality is included for time, unlike the score threshold contract. + assert ( + await client.async_search( + "historicalmarker", created_after=head.memory_at.timestamp(), **opts + ) + )["memories"]["normal"] + + +async def test_T08_raw_score_exclusive_threshold(server_store): + target = node("weakmarker") + await server_store.upsert(target) + client = public_client(server_store) + opts = dict(reader="native", native_mode="keyword", user_ids=["alice"]) + hit = (await client.async_search("weakmarker", **opts))["memories"]["normal"][0] + assert hit["score"] != 1.0 + assert hit["config_id"] == CONFIG_ID and hit["score_kind"] == "rrf" + threshold = NativeScoreThreshold(hit["score"], "keyword", "rrf", CONFIG_ID) + assert not ( + await client.async_search("weakmarker", native_threshold=threshold, **opts) + )["memories"]["normal"] + below = NativeScoreThreshold(hit["score"] - 1e-10, "keyword", "rrf", CONFIG_ID) + same = (await client.async_search("weakmarker", native_threshold=below, **opts))[ + "memories" + ]["normal"][0] + assert same["score"] == hit["score"] + with pytest.raises(NativeRetrievalError): + await client.async_search( + "weakmarker", + native_threshold=NativeScoreThreshold(0, "hybrid", "rrf", CONFIG_ID), + **opts, + ) + + +async def test_T11_public_search_is_read_only(populated, monkeypatch): + store, _ = populated + client = public_client(store) + client._config.recall.strength_enabled = True + + def forbidden(*args, **kwargs): + raise AssertionError("native read must not schedule storage writes") + + monkeypatch.setattr(client, "_spawn_search_dedup", forbidden) + before, _ = store._client.scroll( + store._collection_name, limit=100, with_vectors=True + ) + tasks = set(asyncio.all_tasks()) + await client.async_search("reconnectmarker", reader="native", user_ids=["alice"]) + after, _ = store._client.scroll( + store._collection_name, limit=100, with_vectors=True + ) + assert before == after + assert set(asyncio.all_tasks()) == tasks + + +async def test_T17_automatic_working_route_rejected(server_store, monkeypatch): + client = public_client(server_store) + + async def working(*args, **kwargs): + return dict(ok=True, scene="working", rewrite_query="work", created_after=None) + + monkeypatch.setattr("hy_memory.coding.judge.analyze_queries", working) + with pytest.raises(NativeRetrievalError, match="cannot route"): + await client.async_search( + "work", search_mode="agentic", reader="native", user_ids=["alice"] + ) + + +async def test_sync_public_search_and_prepare(populated): + from hy_memory.client import _LoopThread + + store, target = populated + client = public_client(store) + loop = _LoopThread() + client._search_loop_thread = loop + try: + prepared = await asyncio.to_thread(client.prepare_native_retrieval) + assert prepared["config_id"] == CONFIG_ID + result = await asyncio.to_thread( + client.search, + "reconnectmarker", + reader="native", + native_mode="keyword", + user_ids=["alice"], + ) + assert result["memories"]["normal"][0]["memory_id"] == target.node_id + finally: + loop.stop() + + +async def test_T07_real_server_incomplete_cycle_and_multi_ancestor(server_store): + store = server_store + client = public_client(store) + opts = dict(reader="native", native_mode="keyword", user_ids=["alice"]) + missing = node( + "missingheadmarker", + superseded_by=["missing-id"], + is_latest=False, + status=MemoryStatus.SUPERSEDED, + ) + await store.upsert(missing) + with pytest.raises(NativeRetrievalError) as failure: + await client.async_search("missingheadmarker", **opts) + assert failure.value.reason == "integrity" + first, second = node("cyclemarker"), node("cyclemarker") + first.superseded_by, second.superseded_by = [second.node_id], [first.node_id] + await store.upsert_batch([first, second]) + with pytest.raises(NativeRetrievalError) as failure: + await client.async_search("cyclemarker", **opts) + assert failure.value.reason == "cycle" + head = node("merged state") + ancestors = [ + node( + "multiancestormarker", + superseded_by=[head.node_id], + is_latest=False, + status=MemoryStatus.SUPERSEDED, + ) + for _ in range(2) + ] + head.supersedes = [n.node_id for n in ancestors] + await store.upsert_batch([head] + ancestors) + result = await client.async_search("multiancestormarker", **opts) + assert [m["memory_id"] for m in result["memories"]["normal"]] == [head.node_id] + + +async def test_T17_T18_concurrent_native_and_legacy_real_server( + server_store, monkeypatch +): + import os + + store = server_store + alice = node("alicemarker", embedding=[0.0, 1.0]) + bob = node("bobmarker", user_id="bob") + await store.upsert_batch([alice, bob]) + client = public_client(store) + monkeypatch.setattr(client, "_spawn_search_dedup", lambda *a, **k: None) + before = dict(os.environ) + native, legacy = await asyncio.gather( + client.async_search( + "alicemarker", + reader="native", + native_mode="keyword", + user_ids=["alice"], + limit=1, + ), + client.async_search( + "bobmarker", + reader="legacy", + user_ids=["bob"], + limit=1, + schema_limit=0, + rerank=False, + ), + ) + assert [m["memory_id"] for m in native["memories"]["normal"]] == [alice.node_id] + assert [m["memory_id"] for m in legacy["memories"]["normal"]] == [bob.node_id] + assert dict(os.environ) == before diff --git a/tests/native_retrieval/test_qdrant_contract.py b/tests/native_retrieval/test_qdrant_contract.py new file mode 100644 index 00000000..76fa35d3 --- /dev/null +++ b/tests/native_retrieval/test_qdrant_contract.py @@ -0,0 +1,152 @@ +import pytest + +from hy_memory.data.vector_store_base import NativeQuery +from hy_memory.models.memory import MemoryLayer, MemoryNode +from hy_memory.retrieval import NativeRetrievalError +from conftest import node + +pytestmark = pytest.mark.integration + + +def plan(text, **kwargs): + return NativeQuery( + texts=[text], + embeddings=kwargs.pop("embeddings", []), + user_ids=kwargs.pop("user_ids", ["alice"]), + layers=[MemoryLayer.L2_FACT], + mode=kwargs.pop("mode", "keyword"), + **kwargs, + ) + + +async def test_T03_dual_tags_strict(server_store): + from qdrant_client import models as m + + store = server_store + compound, exact = node(tags=["foo bar"]), node(tags=["foo"]) + await store.upsert_batch([compound, exact]) + store._client.update_collection( + store._collection_name, + strict_mode_config=m.StrictModeConfig( + enabled=True, unindexed_filtering_retrieve=False + ), + ) + records, _ = store._client.scroll( + store._collection_name, + scroll_filter=m.Filter( + must=[m.FieldCondition(key="tags", match=m.MatchValue(value="foo"))] + ), + ) + assert [r.payload["node_id"] for r in records] == [exact.node_id] + assert {hit["node_id"] for hit in await store.native_query(plan("foo"))} == { + exact.node_id, + compound.node_id, + } + + +async def test_chinese_content_and_tags_native_tokenization(server_store): + content = node("用户喜欢北京烤鸭") + tagged = node("ordinary", tags=["用户喜欢北京烤鸭"]) + await server_store.upsert_batch([content, tagged]) + assert {h["node_id"] for h in await server_store.native_query(plan("烤鸭"))} == { + content.node_id, + tagged.node_id, + } + + +async def test_T05_scope_intersection_and_sources(server_store): + store = server_store + alice, bob = node("scopeword"), node("scopeword", user_id="bob") + outside = [node("scopeword", user_id="outside") for _ in range(40)] + mixed = node("scopeword", source="mixed") + cursor = node("scopeword", source="cursor") + await store.upsert_batch([alice, bob, mixed, cursor] + outside) + result = await store.native_query( + plan("scopeword", user_ids=["alice", "bob"], exclude_sources=["cursor"]) + ) + assert {h["node_id"] for h in result} == {alice.node_id, bob.node_id} + isolation = MemoryNode.build_isolation_key("bob", "a", "s") + assert await store.native_query(plan("scopeword", isolation_keys=[isolation])) == [] + assert ( + await store.native_query( + plan("scopeword", include_sources=["cursor"], exclude_sources=["cursor"]) + ) + == [] + ) + + +async def test_T09_update_clear_and_delete(server_store): + store = server_store + target = node("oldcontentmarker", tags=["oldtagmarker"]) + await store.upsert(target) + assert await store.native_query(plan("oldcontentmarker")) + assert await store.update_payload( + target.node_id, {"content": "newcontentmarker", "tags": ["newtagmarker"]} + ) + assert not await store.native_query(plan("oldcontentmarker")) + assert not await store.native_query(plan("oldtagmarker")) + assert await store.native_query(plan("newcontentmarker")) + assert await store.native_query(plan("newtagmarker")) + assert await store.update_payload(target.node_id, {"content": "", "tags": []}) + assert not await store.native_query(plan("newtagmarker")) + assert not await store.native_query(plan("newcontentmarker")) + await store.update_payload(target.node_id, {"content": "deleteword"}) + assert await store.delete(target.node_id) + assert not await store.native_query(plan("deleteword")) + await store.upsert(node("rangedeleteword")) + await store.delete_by_metadata("alice", agent_id="a", session_id="s") + assert not await store.native_query(plan("rangedeleteword")) + + +async def test_T10_prepare_existing_idempotent_and_conflict(server_store, monkeypatch): + store = server_store + store._client.delete_payload_index(store._collection_name, "content", wait=True) + target = node("beforeprepareword") + await store.upsert(target) + with pytest.raises(NativeRetrievalError, match="missing native") as failure: + await store.native_query(plan("beforeprepareword")) + assert failure.value.reason == "not_ready" + + def no_upsert(*args, **kwargs): + raise AssertionError("prepare must not re-upsert") + + monkeypatch.setattr(store._client, "upsert", no_upsert) + await store.prepare_native_retrieval() + await store.prepare_native_retrieval() + assert (await store.native_query(plan("beforeprepareword")))[0][ + "node_id" + ] == target.node_id + store._client.delete_payload_index(store._collection_name, "content", wait=True) + store._client.create_payload_index( + store._collection_name, "content", field_schema="keyword", wait=True + ) + with pytest.raises(NativeRetrievalError, match="content") as failure: + await store.prepare_native_retrieval() + assert failure.value.reason == "schema_conflict" + assert ( + store._client.get_collection(store._collection_name) + .payload_schema["content"] + .data_type + == "keyword" + ) + + +async def test_T10_interrupted_prepare_retries_actual_schema(server_store, monkeypatch): + store = server_store + for field in ("content", "tags[]", "tags"): + store._client.delete_payload_index(store._collection_name, field, wait=True) + original = store._client.create_payload_index + count = 0 + + def interrupt(*args, **kwargs): + nonlocal count + count += 1 + if count == 2: + raise TimeoutError("interrupted") + return original(*args, **kwargs) + + monkeypatch.setattr(store._client, "create_payload_index", interrupt) + with pytest.raises(NativeRetrievalError): + await store.prepare_native_retrieval() + monkeypatch.setattr(store._client, "create_payload_index", original) + await store.prepare_native_retrieval() diff --git a/tests/native_retrieval/test_reconcile.py b/tests/native_retrieval/test_reconcile.py new file mode 100644 index 00000000..eadd4f38 --- /dev/null +++ b/tests/native_retrieval/test_reconcile.py @@ -0,0 +1,307 @@ +import asyncio +from datetime import datetime + +import pytest + +from hy_memory.agent.reconciler import MemoryReconciler +from conftest import CaptureLLM, FixedEmbed, node, public_client + +pytestmark = pytest.mark.integration + + +def reconciler(store, llm=None): + store.config.recall.reconcile_retriever = "native" + return MemoryReconciler(store.config, llm or CaptureLLM()) + + +@pytest.mark.parametrize("with_healthy", [False, True]) +async def test_writer_deleted_chain_allows_native_search_and_reconcile( + server_store, with_healthy +): + from hy_memory.data.vector_store_base import NativeQuery + from hy_memory.models.memory import MemoryLayer, MemoryStatus + from hy_memory.pipelines.base import WriteRequest + from hy_memory.pipelines.writer import MemoryWriter + + store = server_store + oldest = node("deletedchainmarker version 0") + healthy = node("healthychainmarker") + await store.upsert_batch([oldest] + ([healthy] if with_healthy else [])) + llm = CaptureLLM() + writer = MemoryWriter(store.config, embed_service=FixedEmbed(), vector_store=store) + writer._vector_store, writer._vector_store_initialized = store, True + writer._reconciler = reconciler(store, llm) + writer._reconciler._enable_search_query = False + + async def write(text, request_id): + request = WriteRequest( + user_id="alice", agent_id="a", session_id="s", content=text, + request_id=request_id, + ) + return await writer._reconcile_and_store( + [text], [{"content": text, "layer": "l2_fact"}], request, store, request_id + ) + + # Build the evolution chain through the real writer before deleting its head. + ancestors, head = [oldest], oldest + for version in (1, 2): + text = f"deletedchainmarker version {version}" + llm.ops = [{ + "op": "SUPERSEDE", "memory_id": head.node_id, + "content": text, "layer": "L2_FACT", + }] + stored, error, _, _ = await write(text, f"supersede-{version}") + assert error is None and len(stored) == 1 + head = (await store.native_get_by_ids(stored))[0] + assert head.supersedes == [ancestors[-1].node_id] + assert head.status == MemoryStatus.ACTIVE and head.is_latest + if version == 1: + ancestors.append(head) + persisted = await store.native_get_by_ids([n.node_id for n in ancestors]) + assert all(n.status == MemoryStatus.SUPERSEDED and not n.is_latest for n in persisted) + + llm.prompts.clear() + llm.ops = [{"op": "DELETE", "memory_id": head.node_id}] + stored, error, _, _ = await write("deletedchainmarker", "delete-chain-head") + assert error is None and stored == [] + assert len(llm.prompts) == 1 and head.node_id in llm.prompts[0] + deleted_head = (await store.native_get_by_ids([head.node_id]))[0] + assert deleted_head.status == MemoryStatus.SHADOW and not deleted_head.is_latest + # The backend still recalls ancestors; resolution must suppress the whole chain. + raw_hits = await store.native_query(NativeQuery( + texts=["deletedchainmarker"], embeddings=[[1.0, 0.0]], + user_ids=["alice"], layers=[MemoryLayer.L2_FACT], mode="dense", + )) + assert {n.node_id for n in ancestors} <= {h["node_id"] for h in raw_hits} + result = await public_client(store).async_search( + "deletedchainmarker", reader="native", native_mode="dense", user_ids=["alice"] + ) + assert {m["memory_id"] for m in result["memories"]["normal"]} == ( + {healthy.node_id} if with_healthy else set() + ) + + llm.prompts.clear() + llm.ops = [{"op": "ADD", "content": "fresh fact", "layer": "L2_FACT"}] + stored, error, _, _ = await write("fresh fact", "write-after-delete") + assert error is None and len(stored) == 1 + assert (await store.native_get_by_ids(stored))[0].content == "fresh fact" + assert len(llm.prompts) == int(with_healthy) + if with_healthy: + assert healthy.node_id in llm.prompts[0] + assert "deletedchainmarker" not in llm.prompts[0] + assert all(n.node_id not in llm.prompts[0] for n in ancestors + [head]) + + +async def test_T01_T02_actual_reconcile_prompt_escapes_dense_pool(server_store): + store = server_store + content_target = node("contentonlymarker", embedding=[0.0, 1.0]) + tag_target = node("tagged fact", tags=["tagonlymarker"], embedding=[0.0, 1.0]) + await store.upsert_batch( + [node(f"noise {i}") for i in range(35)] + [content_target, tag_target] + ) + llm = CaptureLLM() + result = await reconciler(store, llm).reconcile( + ["contentonlymarker", "tagonlymarker"], "alice", "a", store, FixedEmbed() + ) + assert result.success + assert len(llm.prompts) == 1 + assert ( + content_target.node_id in llm.prompts[0] + and tag_target.node_id in llm.prompts[0] + ) + + +async def test_T12_necessary_failure_no_operations_or_mutations(populated, monkeypatch): + store, _ = populated + before, _ = store._client.scroll(store._collection_name, limit=100) + llm = CaptureLLM() + + async def fail(*args, **kwargs): + raise TimeoutError("required query timed out") + + monkeypatch.setattr(store, "native_query", fail) + result = await reconciler(store, llm).reconcile( + ["reconnectmarker"], "alice", "a", store, FixedEmbed() + ) + assert not result.success and result.ops == [] and not llm.prompts + after, _ = store._client.scroll(store._collection_name, limit=100) + assert before == after + + +async def test_T12_short_embedding_is_failure(server_store): + class ShortEmbed: + async def embed_batch(self, texts): + return [[1.0, 0.0]] + + result = await reconciler(server_store).reconcile( + ["first", "second"], "alice", "a", server_store, ShortEmbed() + ) + assert not result.success and result.ops == [] + + +async def test_T13_backend_order_is_actual_prompt_order(server_store): + store = server_store + newest = node("orderword", memory_at=datetime(2026, 11, 10)) + oldest = node("unrelated", memory_at=datetime(2026, 11, 1)) + await store.upsert_batch([newest, oldest]) + llm = CaptureLLM() + # Inspect the exact production backend plan and require a contrary time order. + from hy_memory.data.vector_store_base import NativeQuery + from hy_memory.models.memory import MemoryLayer + + plan = NativeQuery( + texts=["orderword"], + embeddings=[[1.0, 0.0]], + user_ids=["alice"], + layers=[MemoryLayer.L2_FACT, MemoryLayer.L4_IDENTITY], + limit=60, + include_sources=["default"], + ) + hits = await store.native_query(plan) + assert [h["node_id"] for h in hits] == [newest.node_id, oldest.node_id] + result = await reconciler(store, llm).reconcile( + ["orderword"], "alice", "a", store, FixedEmbed() + ) + assert result.success + assert llm.prompts[0].index(newest.node_id) < llm.prompts[0].index(oldest.node_id) + + +async def test_T14_healthy_empty_allows_add(server_store): + llm = CaptureLLM() + result = await reconciler(server_store, llm).reconcile( + ["new fact"], "alice", "a", server_store, FixedEmbed() + ) + assert result.success and len(result.ops) == 1 and result.ops[0].op == "ADD" + assert not llm.prompts + + +async def test_T15_cosine_evidence_both_directions(server_store): + store = server_store + lexical = node("semanticword", tags=["semanticword"], embedding=[0.0, 1.0]) + dense = node("ordinary", embedding=[1.0, 0.0]) + await store.upsert_batch([lexical, dense]) + result = await reconciler(store).reconcile( + ["semanticword"], "alice", "a", store, FixedEmbed() + ) + assert result.success + scores = dict(result.candidate_scores["semanticword"]) + assert scores[lexical.node_id] == pytest.approx(0.0) + assert scores[dense.node_id] == pytest.approx(1.0) + + +async def test_T16_optional_cosine_failure_keeps_typed_relation( + server_store, monkeypatch +): + store = server_store + target = node("relationword") + await store.upsert(target) + llm = CaptureLLM( + [ + { + "op": "ADD", + "content": "relationword", + "layer": "L2_FACT", + "related": [{"target_id": target.node_id, "relation_type": "causes"}], + } + ] + ) + + async def fail(*args, **kwargs): + raise TimeoutError("optional cosine evidence") + + monkeypatch.setattr(store, "native_cosine", fail) + result = await reconciler(store, llm).reconcile( + ["relationword"], "alice", "a", store, FixedEmbed() + ) + assert result.success and result.ops[0].op == "ADD" + assert result.ops[0].related == [ + {"target_id": target.node_id, "relation_type": "causes"} + ] + assert result.candidate_scores["relationword"] == [] + + +async def test_T12_cancellation_propagates_before_llm(server_store, monkeypatch): + llm = CaptureLLM() + + async def cancel(*args, **kwargs): + raise asyncio.CancelledError() + + monkeypatch.setattr(server_store, "native_query", cancel) + with pytest.raises(asyncio.CancelledError): + await reconciler(server_store, llm).reconcile( + ["query"], "alice", "a", server_store, FixedEmbed() + ) + assert not llm.prompts + + +@pytest.mark.parametrize("cosine_fails", [False, True]) +async def test_T15_T16_writer_stores_and_projects_real_cosine( + server_store, monkeypatch, cosine_fails +): + from hy_memory.pipelines.writer import MemoryWriter + from hy_memory.pipelines.base import WriteRequest + + class Graph: + def __init__(self): + self.nodes, self.edges = [], [] + + async def upsert_memory_node(self, n): + self.nodes.append(n) + return n.node_id + + async def add_edge(self, src, dst, edge_type, properties=None): + self.edges.append( + (src, dst, properties["relation_type"], properties["weight"]) + ) + return True + + store = server_store + lexical = node("semanticword", tags=["semanticword"], embedding=[0.0, 1.0]) + dense = node("ordinary", embedding=[1.0, 0.0]) + await store.upsert_batch([lexical, dense]) + llm = CaptureLLM( + [ + { + "op": "ADD", + "content": "semanticword", + "layer": "L2_FACT", + "related": [{"target_id": lexical.node_id, "relation_type": "causes"}], + } + ] + ) + graph = Graph() + writer = MemoryWriter( + store.config, embed_service=FixedEmbed(), vector_store=store, graph_store=graph + ) + writer._vector_store, writer._vector_store_initialized = store, True + writer._reconciler = reconciler(store, llm) + if cosine_fails: + + async def fail(*args, **kwargs): + raise TimeoutError("optional cosine") + + monkeypatch.setattr(store, "native_cosine", fail) + request = WriteRequest( + user_id="alice", + agent_id="a", + session_id="s", + content="semanticword", + request_id="native-graph-test", + ) + stored, error, _, _ = await writer._reconcile_and_store( + ["semanticword"], + [{"content": "semanticword", "layer": "l2_fact"}], + request, + store, + request.request_id, + ) + assert error is None and len(stored) == 1 + assert (await store.native_get_by_ids(stored))[0].content == "semanticword" + task = getattr(request, "_graph_projection_task", None) + if task is not None: + await task + assert graph.nodes and (stored[0], lexical.node_id, "causes", 1.0) in graph.edges + semantic_targets = { + dst for _, dst, kind, _ in graph.edges if kind == "semantic_neighbor" + } + assert semantic_targets == (set() if cosine_fails else {dense.node_id}) diff --git a/tests/test_reader_selection.py b/tests/test_reader_selection.py index 59cb5ad4..0e537beb 100644 --- a/tests/test_reader_selection.py +++ b/tests/test_reader_selection.py @@ -1,15 +1,18 @@ import asyncio from hy_memory.config import MemoryConfig +from hy_memory.models.memory import MemoryLayer, MemoryNode +from hy_memory.pipelines.base import ReadRequest from hy_memory.pipelines.reader_base import BaseReadPipeline from hy_memory.pipelines.reader_hybrid import HybridReadPipeline from hy_memory.pipelines.reader_legacy import LegacyReadPipeline -def test_only_legacy_and_hybrid_are_registered(monkeypatch): +def test_legacy_hybrid_and_native_are_registered(monkeypatch): from hy_memory.pipelines._retrieval import config - assert config.ALL_READERS == ("legacy", "hybrid") + assert config.ALL_READERS == ("legacy", "hybrid", "native") + assert config.resolve_reader_name("native") == "native" assert config.resolve_reader_name("legacy") == "legacy" assert config.resolve_reader_name("hybrid") == "hybrid" assert config.resolve_reader_name("hybrid_v2") == "legacy" @@ -34,6 +37,7 @@ def test_reader_difference_is_keyword_recall_only(): assert HybridReadPipeline.ENABLE_KEYWORD_RECALL is True runtime_metadata = { "__module__", "__doc__", "__abstractmethods__", "_abc_impl", + "__firstlineno__", "__static_attributes__", # Python 3.13 class metadata } assert set(LegacyReadPipeline.__dict__) - runtime_metadata == { "VERSION", "ENABLE_KEYWORD_RECALL", @@ -69,3 +73,51 @@ def test_local_rerank_defaults_off_and_is_shared(): assert config.recall.local_rerank_model == "nreimers/mmarco-mMiniLMv2-L6-H384-v1" assert LegacyReadPipeline._read_once is BaseReadPipeline._read_once assert HybridReadPipeline._read_once is BaseReadPipeline._read_once + + +def test_read_request_preserves_all_legacy_positional_arguments(): + values = dict( + query="query", user_id="alice", user_ids=["alice"], agent_id="agent", + agent_ids=["agent"], session_ids=["session"], + include_sources=["codex"], exclude_sources=["cursor", "mixed"], + limit=3, layers=["l2_fact"], min_score=0.2, + schema_min_score=0.3, schema_limit=2, intention_limit=1, + rerank=False, created_after=123.0, query_embedding=[1.0, 0.0], + session_id="session", request_id="request", extra={"key": "value"}, + ) + request = ReadRequest(*values.values()) + assert {name: getattr(request, name) for name in values} == values + assert request.raw_query is None + assert request.native_mode == "hybrid" + assert request.native_threshold is None + + +async def test_legacy_positional_request_keeps_user_isolation(tmp_path): + from hy_memory.data.vector_store_chroma import ChromaVectorStore + + config = MemoryConfig.from_dict({"vector_store": { + "provider": "chroma", "collection_name": "positional_scope", + "persist_directory": str(tmp_path / "chroma"), "embedding_dims": 2, + }}) + store = ChromaVectorStore(config) + await store.initialize() + try: + await store.upsert_batch([ + MemoryNode( + node_id=user, user_id=user, content=f"{user} likes hiking", + layer=MemoryLayer.L2_FACT, embedding=[1.0, 0.0], + ) + for user in ("alice", "bob") + ]) + reader = LegacyReadPipeline(config, vector_store=store) + # Keep the historical two-positional-argument call at the actual reader seam. + request = ReadRequest("query", "alice") + request.query_embedding = [1.0, 0.0] + request.rerank = False + request.schema_limit = 0 + response = await reader.read(request) + assert response.success, response.error_message + assert [m["memory_id"] for m in response.memories] == ["alice"] + assert {m["user_id"] for m in response.memories} == {"alice"} + finally: + await store.close() diff --git a/uv.lock b/uv.lock index 4956a4e9..6041eb12 100644 --- a/uv.lock +++ b/uv.lock @@ -1715,7 +1715,7 @@ wheels = [ [[package]] name = "hy-memory" -version = "2.0.13" +version = "2.0.16" source = { editable = "." } dependencies = [ { name = "chromadb" }, @@ -1756,7 +1756,7 @@ all = [ { name = "psycopg", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version < '3.10'" }, { name = "psycopg", version = "3.3.6", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version >= '3.10'" }, { name = "qdrant-client", version = "1.16.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "qdrant-client", version = "1.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "qdrant-client", version = "1.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "torch", version = "2.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "torch", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "transformers", version = "4.57.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, @@ -1789,7 +1789,10 @@ qdrant = [ { name = "fastembed", version = "0.7.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "fastembed", version = "0.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "qdrant-client", version = "1.16.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "qdrant-client", version = "1.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "qdrant-client", version = "1.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +qdrant-native = [ + { name = "qdrant-client", version = "1.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] rerank = [ { name = "torch", version = "2.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, @@ -1825,6 +1828,7 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'" }, { name = "python-dotenv" }, { name = "pyyaml" }, + { name = "qdrant-client", marker = "python_full_version >= '3.10' and extra == 'qdrant-native'", specifier = "==1.20.0" }, { name = "qdrant-client", marker = "extra == 'all'" }, { name = "qdrant-client", marker = "extra == 'qdrant'" }, { name = "rank-bm25" }, @@ -1836,7 +1840,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'all'" }, { name = "transformers", marker = "extra == 'rerank'" }, ] -provides-extras = ["qdrant", "faiss", "graph", "mysql", "postgres", "rerank", "all", "dev"] +provides-extras = ["qdrant", "qdrant-native", "faiss", "graph", "mysql", "postgres", "rerank", "all", "dev"] [[package]] name = "hyperframe" @@ -6027,7 +6031,7 @@ wheels = [ [[package]] name = "qdrant-client" -version = "1.19.1" +version = "1.20.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", @@ -6037,19 +6041,19 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "grpcio", version = "1.84.0", source = { registry = "https://pypi.org/simple" } }, - { name = "httpx", extra = ["http2"] }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "grpcio", version = "1.84.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "httpx", extra = ["http2"], marker = "python_full_version >= '3.10'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "portalocker" }, - { name = "protobuf", version = "7.36.2", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" } }, - { name = "urllib3", version = "2.8.0", source = { registry = "https://pypi.org/simple" } }, + { name = "portalocker", marker = "python_full_version >= '3.10'" }, + { name = "protobuf", version = "7.36.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "urllib3", version = "2.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/20/c8fcd645d3f595b086fa11a085980e9f641fd56fc6221fb325d634b8c4fa/qdrant_client-1.19.1.tar.gz", hash = "sha256:8f1d851a8463ce8cc11cf39ed8a9c9fb4b5f9de60e9a096ff56da42d1f074907", size = 360625, upload-time = "2026-09-16T06:43:13.818Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/59/212766131e3f032baa54cfed47230e209538d686229c0d6389ad339f3cd7/qdrant_client-1.20.0.tar.gz", hash = "sha256:58ff8aec0c56d2dd4d9f0cae1940c280b364bfde18715d4ccc10c3e118bcd053", size = 417239, upload-time = "2026-10-22T10:26:16.275Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/9f/becebdda02beddd422587eba0d7dfac5b1f1e0aa1ada5bcf9b9e6f1c3717/qdrant_client-1.19.1-py3-none-any.whl", hash = "sha256:fca1a96c3f90f5fff853f6ee6877838a5768a04c963df9891a655a63313af8a0", size = 406533, upload-time = "2026-09-16T06:43:12.428Z" }, + { url = "https://files.pythonhosted.org/packages/63/f0/f37da771d09d779313ad470945600cb5d3e1a26864126def802a3254b536/qdrant_client-1.20.0-py3-none-any.whl", hash = "sha256:0ab261197a497d16f3bf8a663c22e994f485c9fd7212d3c8b07c1a9de32fc152", size = 471157, upload-time = "2026-10-22T10:26:14.838Z" }, ] [[package]]