From 25089b5beac9af09dd40277400bc4b3d584a6b42 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Mon, 24 Aug 2026 01:42:55 +0800 Subject: [PATCH 1/7] feat: add links_to directive for ambiguous relations Signed-off-by: Mike Mikemikike Signed-off-by: mikemikimike <13286568797@163.com> Signed-off-by: mikemikimike <13286568797@163.com> --- src/basic_memory/markdown/plugins.py | 19 +++++++-- tests/markdown/test_relation_edge_cases.py | 47 +++++++++++++++++++++- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/basic_memory/markdown/plugins.py b/src/basic_memory/markdown/plugins.py index 5dfd3536d..1a41ccc63 100644 --- a/src/basic_memory/markdown/plugins.py +++ b/src/basic_memory/markdown/plugins.py @@ -13,6 +13,15 @@ # those bracket prefixes stay ordinary content (issue #1219). _TIMESTAMP_VALUE = r"\d{1,3}:\d{2}(?::\d{2})?(?:[.,]\d{1,3})?" _TIMESTAMP_CATEGORY = re.compile(rf"^{_TIMESTAMP_VALUE}(?:\s+-\s+{_TIMESTAMP_VALUE})?$") +_LINKS_TO_DIRECTIVE = re.compile(r"\s+#bm:links_to\s*$") + + +def remove_links_to_directive(content: str) -> tuple[str, bool]: + """Remove an exact terminal ``#bm:links_to`` directive from content.""" + match = _LINKS_TO_DIRECTIVE.search(content) + if not match: + return content, False + return content[: match.start()].rstrip(), True def _is_task_marker_category(category: str) -> bool: @@ -47,6 +56,7 @@ def is_observation(token: Token) -> bool: return False # Use token.tag which contains the actual content for test tokens, fallback to content content = (token.tag or token.content).strip() + content, _ = remove_links_to_directive(content) if not content: # pragma: no cover return False # if it's a markdown_task, return false @@ -74,6 +84,7 @@ def parse_observation(token: Token) -> Dict[str, Any]: # Use token.tag which contains the actual content for test tokens, fallback to content content = (token.tag or token.content).strip() + content, _ = remove_links_to_directive(content) # Parse [category] with regex; a timestamp-shaped prefix is not a category, so a # hashtag-promoted transcript line keeps its timecode inside the content instead. @@ -325,17 +336,19 @@ def relation_rule(state: Any) -> None: # Only process inline tokens if token.type == "inline": + content = token.tag or token.content + content_without_directive, has_directive = remove_links_to_directive(content) + # Check for explicit relations in list items - if in_list_item and is_explicit_relation(token): + if in_list_item and not has_directive and is_explicit_relation(token): rel = parse_relation(token) if rel: token.meta["relations"] = [rel] # Always check for inline links in any text else: - content = token.tag or token.content if "[[" in content: - rels = parse_inline_relations(content) + rels = parse_inline_relations(content_without_directive) if rels: token.meta["relations"] = token.meta.get("relations", []) + rels diff --git a/tests/markdown/test_relation_edge_cases.py b/tests/markdown/test_relation_edge_cases.py index e38112bcc..738b26ef9 100644 --- a/tests/markdown/test_relation_edge_cases.py +++ b/tests/markdown/test_relation_edge_cases.py @@ -2,7 +2,12 @@ from markdown_it import MarkdownIt -from basic_memory.markdown.plugins import relation_plugin, parse_relation, parse_inline_relations +from basic_memory.markdown.plugins import ( + observation_plugin, + relation_plugin, + parse_relation, + parse_inline_relations, +) from basic_memory.markdown.schemas import Relation @@ -247,3 +252,43 @@ def test_bare_list_wikilink_is_inline_link_not_default_explicit_relation(): assert token.meta["relations"] == [{"type": "links_to", "target": "Target", "context": None}] assert parse_relation(token) is None + + +def test_links_to_directive_forces_implicit_relations(): + """The terminal directive disambiguates a single-token relation prefix.""" + md = MarkdownIt().use(relation_plugin) + + tokens = md.parse("- Mother [[Alice]] #bm:links_to") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"] == [ + {"type": "links_to", "target": "Alice", "context": None} + ] + + tokens = md.parse("- Mentions [[Alice]] and [[Bob]] #bm:links_to") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"] == [ + {"type": "links_to", "target": "Alice", "context": None}, + {"type": "links_to", "target": "Bob", "context": None}, + ] + + +def test_links_to_directive_is_terminal_and_explicit_relations_are_unchanged(): + """Only the exact terminal directive changes relation interpretation.""" + md = MarkdownIt().use(relation_plugin) + + tokens = md.parse("- spouse_of [[Alice]]") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"][0]["type"] == "spouse_of" + + tokens = md.parse("- Mother [[Alice]] #bm:links_to later") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"][0]["type"] == "links_to" + + +def test_links_to_directive_is_not_an_observation_tag(): + """The directive is syntax, not a note tag or indexed observation text.""" + md = MarkdownIt().use(observation_plugin).use(relation_plugin) + + tokens = md.parse("- Mother [[Alice]] #bm:links_to") + token = next(t for t in tokens if t.type == "inline") + assert "observation" not in token.meta From d465fcc36053650b3919330c8ad4d3d5ae4cda79 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Mon, 24 Aug 2026 07:43:12 +0800 Subject: [PATCH 2/7] test: cover links_to source preservation Add regression coverage for preserving the directive in source and excluding it from observation semantics. Signed-off-by: Mike Mikemikike Signed-off-by: mikemikimike <13286568797@163.com> Signed-off-by: mikemikimike <13286568797@163.com> --- tests/markdown/test_relation_edge_cases.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/markdown/test_relation_edge_cases.py b/tests/markdown/test_relation_edge_cases.py index 738b26ef9..f3a32ee59 100644 --- a/tests/markdown/test_relation_edge_cases.py +++ b/tests/markdown/test_relation_edge_cases.py @@ -8,6 +8,7 @@ parse_relation, parse_inline_relations, ) +from basic_memory.markdown.entity_parser import parse from basic_memory.markdown.schemas import Relation @@ -292,3 +293,18 @@ def test_links_to_directive_is_not_an_observation_tag(): tokens = md.parse("- Mother [[Alice]] #bm:links_to") token = next(t for t in tokens if t.type == "inline") assert "observation" not in token.meta + + +def test_links_to_directive_preserves_source_and_observation_content(): + """The directive stays in source while remaining outside indexed semantics.""" + source = "- [note] Mother [[Alice]] #bm:links_to\n" + parsed = parse(source) + + assert parsed.content == source + assert len(parsed.observations) == 1 + assert parsed.observations[0].category == "note" + assert parsed.observations[0].content == "Mother [[Alice]]" + assert parsed.observations[0].tags is None + assert len(parsed.relations) == 1 + assert parsed.relations[0].type == "links_to" + assert parsed.relations[0].target == "Alice" From 856cca25ad7bdbf27d510a8e38e2cd82e94dfce8 Mon Sep 17 00:00:00 2001 From: Paul Hernandez <60959+phernandez@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:37:14 -0500 Subject: [PATCH 3/7] fix(core): update benchmark MCP result fields (#1307) Signed-off-by: phernandez Signed-off-by: mikemikimike <13286568797@163.com> --- benchmarks/scripts/read_load_bench.py | 10 +++++----- tests/test_read_load_bench.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/benchmarks/scripts/read_load_bench.py b/benchmarks/scripts/read_load_bench.py index ad2465e7d..35bb2de45 100644 --- a/benchmarks/scripts/read_load_bench.py +++ b/benchmarks/scripts/read_load_bench.py @@ -382,7 +382,7 @@ def write_manifest( def result_payload(result: CallToolResult) -> dict[str, object]: - structured = result.structuredContent + structured = result.structured_content if isinstance(structured, dict): wrapped = structured.get("result") payload = wrapped if isinstance(wrapped, dict) else structured @@ -402,7 +402,7 @@ def result_payload(result: CallToolResult) -> dict[str, object]: def result_text(result: CallToolResult) -> str | None: """Return text from a successful text-mode tool response.""" - if result.isError: + if result.is_error: return None text_parts = [ text for item in result.content if isinstance((text := getattr(item, "text", None)), str) @@ -517,7 +517,7 @@ async def write_target( "output_format": "json", }, ) - if result.isError: + if result.is_error: raise RuntimeError(f"write_note failed while seeding {title}") payload = result_payload(result) identifier = payload.get("permalink") @@ -573,7 +573,7 @@ async def searchable_count(session: ClientSession, project: str) -> int: "output_format": "json", }, ) - if result.isError: + if result.is_error: return 0 payload = result_payload(result) total = payload.get("total") @@ -742,7 +742,7 @@ async def run(args: argparse.Namespace) -> int: "create_memory_project", {"project_name": project, "project_path": str(project_dir)}, ) - if created.isError: + if created.is_error: raise RuntimeError("could not create benchmark project") targets = await seed_corpus( diff --git a/tests/test_read_load_bench.py b/tests/test_read_load_bench.py index d5aedd2a5..d609edca7 100644 --- a/tests/test_read_load_bench.py +++ b/tests/test_read_load_bench.py @@ -10,6 +10,7 @@ from types import ModuleType import pytest +from mcp.types import CallToolResult, TextContent def load_read_load_bench() -> ModuleType: @@ -26,6 +27,21 @@ def load_read_load_bench() -> ModuleType: read_load_bench = load_read_load_bench() +def test_result_helpers_use_current_mcp_python_fields() -> None: + success = CallToolResult( + content=[TextContent(type="text", text="plain response")], + structured_content={"result": {"permalink": "notes/example"}}, + ) + failure = CallToolResult( + content=[TextContent(type="text", text="failed response")], + is_error=True, + ) + + assert read_load_bench.result_payload(success) == {"permalink": "notes/example"} + assert read_load_bench.result_text(success) == "plain response" + assert read_load_bench.result_text(failure) is None + + @dataclass class RecordingScalarResult: version: str From aaa790731f6bd7aa2873535bd17ce842bbada6dd Mon Sep 17 00:00:00 2001 From: Paul Hernandez <60959+phernandez@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:14:48 -0500 Subject: [PATCH 4/7] docs(core): cover the 2026-08-22 merges in the v0.23.0 changelog (#1309) Signed-off-by: phernandez Co-authored-by: Claude Fable 5 Signed-off-by: mikemikimike <13286568797@163.com> --- CHANGELOG.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69b9c2ae..d77292114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -229,6 +229,11 @@ search entries. - **#1073**: `edit_note` with a `memory://` URL routes to the target project instead of creating phantom notes (**#1066**), and scoped `memory://` URL paths are preserved (**#1092**). +- **#1299**: `build_context` bounds its traversal requests — primary pages + cap at 50 results, related context at 100 per page, and negative + `max_related` is rejected instead of acting as an unbounded SQL limit — + so a traversal call can no longer become a multi-megabyte vault export + (**#1296**). Callers paginate or follow returned `memory://` links. - **#1285**: Glob-filtered directory listings traverse subdirectories again: `file_name_glob` filters results instead of accidentally pruning recursion, so `*.md` with depth 2 finds files inside subdirectories. @@ -261,6 +266,15 @@ search entries. `create_time` — timestamps fall back to `update_time`, then the earliest message time, so the whole archive imports (**#1276**); undecodable import uploads return a 400 with the parse error instead of a 500. +- **#1298**: The update check no longer reports "up to date" when the + Homebrew probe fails, no longer auto-runs `brew upgrade` for an update + inferred from PyPI metadata, and names the actual Homebrew target + version it will install. +- **#1300 / #1302**: The multilingual-E5 prefix contract is documented — + asymmetric FastEmbed models need `semantic_embedding_query_prefix` / + `semantic_embedding_document_prefix`, which already drive reindex through + the embedding identity — along with custom-model selection, the installed + FastEmbed catalog, and dimensions (**#1264**). - **#1058 / #1094**: `bm doctor` never prints a blank failure message, and migrations adapt to existing event loops (**#1027**). - **#1080**: Loading config no longer recreates an empty `~/basic-memory` @@ -279,7 +293,9 @@ search entries. registration APIs instead of private PluginManager writes, preserving the lifecycle ownership that cleans them up on provider unload (**#1257**); the slash-command monkeypatch docs are split by Hermes version so modern - installs skip the legacy collector workaround (**#1278**). + installs skip the legacy collector workaround (**#1278**), and the install + docs require a Hermes release with managed manifest-v2 installation + instead of the nonexistent `--path` option (**#1280**, **#1303**). ### Maintenance @@ -292,6 +308,12 @@ search entries. - Kept Milvus as a first-party optional vector backend while removing the unused Python entry-point registry for separately packaged vector adapters. +- **#1292 / #1293**: The dev-release pipeline publishes again via PyPI + trusted publishing — its version gate imported a hardcoded module version + and had silently skipped every dev publish since the 0.22.1 bump. +- **#1304 / #1307**: The concurrent-write convergence benchmark is ported + into the canonical `/benchmarks` package, and the read-load benchmark + runs against the current MCP SDK's typed result fields. ## v0.22.1 (2026-06-12) From 81b3daba90d241c82f6ce49b77a65a4363938da8 Mon Sep 17 00:00:00 2001 From: Paul Hernandez <60959+phernandez@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:14:57 -0500 Subject: [PATCH 5/7] chore(core): remove legacy benchmark judge (#1308) Signed-off-by: phernandez Signed-off-by: mikemikimike <13286568797@163.com> --- benchmarks/AGENTS.md | 13 +- benchmarks/README.md | 25 +--- benchmarks/docs/benchmarks.md | 40 +++-- benchmarks/justfile | 29 +--- benchmarks/pyproject.toml | 5 - benchmarks/src/basic_memory_benchmarks/cli.py | 24 --- .../basic_memory_benchmarks/llm/runners.py | 4 +- .../src/basic_memory_benchmarks/models.py | 24 --- .../reporting/artifacts.py | 32 ---- .../src/basic_memory_benchmarks/runner.py | 48 ------ .../basic_memory_benchmarks/scoring/judge.py | 140 ------------------ benchmarks/tests/llm/test_runners.py | 8 + benchmarks/tests/test_cli_surface.py | 24 +++ benchmarks/tests/test_manifest_schema.py | 2 + benchmarks/uv.lock | 88 ----------- 15 files changed, 71 insertions(+), 435 deletions(-) delete mode 100644 benchmarks/src/basic_memory_benchmarks/scoring/judge.py create mode 100644 benchmarks/tests/test_cli_surface.py diff --git a/benchmarks/AGENTS.md b/benchmarks/AGENTS.md index 7827cc1be..d94b2ccdd 100644 --- a/benchmarks/AGENTS.md +++ b/benchmarks/AGENTS.md @@ -7,7 +7,7 @@ directory for comparing Basic Memory against other memory systems. Primary goals: - Deterministic retrieval benchmarks -- Optional LLM-as-a-judge benchmarks +- End-to-end QA scoring with fixed answerer and judge models - Public, reproducible artifact publication (including provenance metadata) The benchmark package keeps its own `pyproject.toml` and lockfile so benchmark @@ -16,7 +16,6 @@ dependencies do not pollute the Core product environment. ## Build / Test Commands - Install: `uv sync --group dev` -- Install judge extras: `uv sync --group dev --extra judge` - Run tests: `uv run pytest -q` - Lint: `uv run ruff check .` - Type check: `uv run pyright` @@ -37,8 +36,8 @@ Dataset and conversion: Run retrieval: - `uv run bm-bench run retrieval --providers bm-local,mem0-local --dataset-id locomo --dataset-path benchmarks/datasets/locomo/locomo10.json --corpus-dir benchmarks/generated/locomo/docs --queries-path benchmarks/generated/locomo/queries.json --output-root benchmarks/runs --allow-provider-skip` -Run judge (optional): -- `uv run bm-bench run judge --run-dir benchmarks/runs/` +Run end-to-end QA scoring: +- `uv run bm-bench run qa --run-dir benchmarks/runs/ --answerer claude:claude-haiku-4-5 --judge claude:claude-sonnet-4-6` Validate and publish: - `uv run bm-bench validate-artifacts --run-dir benchmarks/runs/` @@ -53,15 +52,15 @@ Validate and publish: - `just bench-run-bm-local` - `just bench-run-mem0-local` - `just bench-run-full` -- `just bench-judge RUN_DIR=benchmarks/runs/` -- `just bench-publish RUN_DIR=benchmarks/runs/` +- `just bench-qa benchmarks/runs/` +- `just bench-publish benchmarks/runs/` ## Repository Layout - `src/basic_memory_benchmarks/cli.py` - CLI surface - `src/basic_memory_benchmarks/runner.py` - run orchestration - `src/basic_memory_benchmarks/providers/` - provider adapters (`bm-local`, `bm-cloud`, `mem0-local`, `zep-reference`) -- `src/basic_memory_benchmarks/scoring/` - retrieval + judge scoring +- `src/basic_memory_benchmarks/scoring/` - retrieval + end-to-end QA scoring - `src/basic_memory_benchmarks/reporting/` - artifact writers / comparison helpers - `src/basic_memory_benchmarks/converters/` - dataset conversion logic - `src/basic_memory_benchmarks/datasets/` - dataset fetch/load helpers diff --git a/benchmarks/README.md b/benchmarks/README.md index 9b67ba830..b47cf9275 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,7 +7,7 @@ dependencies remain isolated from the product environment. ## Goals - Deterministic retrieval benchmarks (Recall@5/10, MRR, Precision@5, content-hit, latency) -- Optional LLM-as-judge scoring (Pydantic Evals) +- End-to-end QA scoring with fixed answerer and judge models - Public artifacts with provenance and reproducibility metadata - Clean dependency isolation from the core Basic Memory package @@ -29,12 +29,6 @@ dependencies remain isolated from the product environment. uv sync --group dev ``` -Optional judge dependencies: - -```bash -uv sync --group dev --extra judge -``` - ## Quickstart ### 1) Fetch LoCoMo dataset @@ -85,16 +79,7 @@ know") when the retrieved memories don't contain the answer, and abstention is graded correct only when the gold answer marks the question unanswerable (LoCoMo adversarial cases). -### 5) Optional retrieval-context judge (legacy) - -Scores whether the *retrieved context* contains the expected answer, without -answer generation: - -```bash -uv run bm-bench run judge --run-dir benchmarks/runs/ -``` - -### 6) Publish run artifacts +### 5) Publish run artifacts ```bash uv run bm-bench publish --run-dir benchmarks/runs/ @@ -290,8 +275,6 @@ Per run (`benchmarks/runs//`): - `retrieval-summary.json` - `per-query-qa.jsonl` (optional) - `qa-summary.json` (optional) -- `per-query-judge.jsonl` (optional) -- `judge-summary.json` (optional) - `summary.md` ## Just commands @@ -303,8 +286,8 @@ just bench-convert-locomo just bench-run-bm-local just bench-run-mem0-local just bench-run-full -just bench-judge -just bench-publish RUN_DIR=benchmarks/runs/ +just bench-qa benchmarks/runs/ +just bench-publish benchmarks/runs/ ``` ## Notes on dataset publication diff --git a/benchmarks/docs/benchmarks.md b/benchmarks/docs/benchmarks.md index 17952f07f..0fd0911ce 100644 --- a/benchmarks/docs/benchmarks.md +++ b/benchmarks/docs/benchmarks.md @@ -12,9 +12,9 @@ It covers: | Area | Status | | --- | --- | -| Single run execution (`run retrieval`, `run full`, `run judge`) | Implemented | +| Single run execution (`run retrieval`, `run full`, `run qa`) | Implemented | | Concurrent write convergence (`run concurrent-write`) | Implemented | -| `just` one-command pipelines (`bench-full`, `bench-full-judge`) | Implemented | +| `just` retrieval and QA workflows (`bench-full`, `bench-qa`) | Implemented | | Artifact generation and publish/compare commands | Implemented | | Manual BM revision comparison via worktrees + `--bm-local-path` | Implemented workflow, manual orchestration | | `bm-bench run revision-matrix` | Planned, not implemented yet | @@ -58,12 +58,6 @@ cd /path/to/basic-memory/benchmarks just sync ``` -If you plan to run judge metrics: - -```bash -just sync-judge -``` - ### Dataset assumptions LoCoMo source and converted outputs are created by: @@ -78,7 +72,7 @@ just bench-prepare-long ### `just` commands (current) - `bench-full` -- `bench-full-judge` +- `bench-qa` - `bench-concurrent-write-smoke` - `bench-concurrent-write-load` - `bench-prepare-short` @@ -86,7 +80,6 @@ just bench-prepare-long - `bench-run-short` - `bench-run-long` - `bench-run-full` -- `bench-judge` - `bench-validate` - `bench-publish` - `bench-compare` @@ -101,7 +94,9 @@ Top-level commands: - `run retrieval` - `run concurrent-write` - `run full` -- `run judge` +- `run qa` +- `run rejudge` +- `run review` - `compare` - `validate-artifacts` - `publish` @@ -120,17 +115,15 @@ This runs: 2. `just bench-prepare-long` 3. `just bench-run-full` -### One-command full retrieval + judge +### End-to-end QA scoring ```bash cd /path/to/basic-memory/benchmarks -just bench-full-judge +just bench-qa benchmarks/runs/ ``` -This runs: -1. `just sync-judge` -2. `just bench-prepare-long` -3. `just bench-run-full-judge` +This generates answers from each provider's retrieved context, applies the +same judge to every provider, and writes QA artifacts into the retrieval run. ### Short vs long workflows @@ -196,10 +189,15 @@ Required files: - `retrieval-summary.json` - `summary.md` -Optional judge files: +Optional QA files: -- `per-query-judge.jsonl` -- `judge-summary.json` +- `per-query-qa.jsonl` +- `qa-summary.json` +- `per-query-qa-rejudge.jsonl` +- `qa-rejudge-summary.json` +- `qa-rejudge-flips.json` +- `review.html` +- `qa-diagnosis.json` ### Key provenance fields @@ -483,7 +481,7 @@ Dry-run checks: just --dry-run bench-run-short just --dry-run bench-run-long just --dry-run bench-full -just --dry-run bench-full-judge +just --dry-run bench-qa benchmarks/runs/ ``` Artifact field checks: diff --git a/benchmarks/justfile b/benchmarks/justfile index 6bcff0ae9..eaef7ce99 100644 --- a/benchmarks/justfile +++ b/benchmarks/justfile @@ -18,9 +18,6 @@ longmemeval_dev_output_dir := "benchmarks/generated/longmemeval-s-dev" sync: uv sync --group dev -sync-judge: - uv sync --group dev --extra judge - test: uv run pytest -q @@ -110,13 +107,6 @@ bench-full: just bench-prepare-long just bench-run-full -# Full retrieval + judge pipeline: -# 1) sync deps (+judge extras), 2) fetch+convert long dataset, 3) run full with judge -bench-full-judge model="gpt-4o-mini": - just sync-judge - just bench-prepare-long - just bench-run-full-judge model="{{model}}" - # --- Benchmark execution --- bench-smoke: @@ -198,18 +188,6 @@ bench-run-full: {{bm_local_path_flag}} \ --allow-provider-skip -bench-run-full-judge model="gpt-4o-mini": - uv run bm-bench run full \ - --dataset-id locomo \ - --dataset-path {{locomo_dataset_path}} \ - --corpus-dir benchmarks/generated/locomo/docs \ - --queries-path benchmarks/generated/locomo/queries.json \ - --providers bm-local,mem0-local \ - {{bm_local_path_flag}} \ - --allow-provider-skip \ - --judge \ - --judge-model "{{model}}" - # --- Concurrency benchmark (basic-memory#1248) --- # Small-scale smoke: 4 writers x 25 notes; strict so divergence fails the command @@ -244,8 +222,11 @@ bench-latest-run: set -euo pipefail ls -1dt benchmarks/runs/* | head -n 1 -bench-judge run_dir model="gpt-4o-mini": - uv run bm-bench run judge --run-dir "{{run_dir}}" --model "{{model}}" +bench-qa run_dir answerer="claude:claude-haiku-4-5" judge="claude:claude-sonnet-4-6": + uv run bm-bench run qa \ + --run-dir "{{run_dir}}" \ + --answerer "{{answerer}}" \ + --judge "{{judge}}" bench-validate run_dir: uv run bm-bench validate-artifacts --run-dir "{{run_dir}}" diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml index f206ef66e..d858f0792 100644 --- a/benchmarks/pyproject.toml +++ b/benchmarks/pyproject.toml @@ -18,11 +18,6 @@ dependencies = [ "typer>=0.16.1", ] -[project.optional-dependencies] -judge = [ - "pydantic-evals>=0.4.0", -] - [project.scripts] bm-bench = "basic_memory_benchmarks.cli:main" basic-memory-benchmarks = "basic_memory_benchmarks.cli:main" diff --git a/benchmarks/src/basic_memory_benchmarks/cli.py b/benchmarks/src/basic_memory_benchmarks/cli.py index 42d33cc12..8f2739738 100644 --- a/benchmarks/src/basic_memory_benchmarks/cli.py +++ b/benchmarks/src/basic_memory_benchmarks/cli.py @@ -30,7 +30,6 @@ ) from basic_memory_benchmarks.runner import ( run_diagnose_stage, - run_judge, run_qa_stage, run_rejudge_stage, run_review_stage, @@ -430,15 +429,6 @@ def run_rejudge_command( console.print(f"Flips: [cyan]{out / 'qa-rejudge-flips.json'}[/cyan]") -@run_app.command("judge") -def run_judge_command( - run_dir: Path = typer.Option(..., "--run-dir"), - model: str = typer.Option("gpt-4o-mini", "--model"), -) -> None: - out = run_judge(run_dir=run_dir, model=model) - console.print(f"Judge run complete: [green]{out}[/green]") - - @run_app.command("full") def run_full_command( providers: str = typer.Option("bm-local,mem0-local", "--providers"), @@ -456,8 +446,6 @@ def run_full_command( bm_source: str = typer.Option("github:basicmachines-co/basic-memory@main", "--bm-source"), bm_local_path: str | None = typer.Option(None, "--bm-local-path"), allow_provider_skip: bool = typer.Option(True, "--allow-provider-skip/--strict-providers"), - judge: bool = typer.Option(False, "--judge"), - judge_model: str = typer.Option("gpt-4o-mini", "--judge-model"), ) -> None: run_retrieval_command( providers=providers, @@ -473,18 +461,6 @@ def run_full_command( allow_provider_skip=allow_provider_skip, ) - if judge: - resolved_run_id = run_id - if resolved_run_id is None: - # run_retrieval_command generated uuid when run_id is None. infer by latest dir. - run_dirs = sorted(Path(output_root).glob("*"), key=lambda path: path.stat().st_mtime) - if not run_dirs: - raise RuntimeError("Unable to locate run directory for judge step") - run_dir = run_dirs[-1] - else: - run_dir = Path(output_root) / resolved_run_id - run_judge_command(run_dir=run_dir, model=judge_model) - @app.command("compare") def compare_runs( diff --git a/benchmarks/src/basic_memory_benchmarks/llm/runners.py b/benchmarks/src/basic_memory_benchmarks/llm/runners.py index 2cf72915e..a328cc9c5 100644 --- a/benchmarks/src/basic_memory_benchmarks/llm/runners.py +++ b/benchmarks/src/basic_memory_benchmarks/llm/runners.py @@ -16,6 +16,7 @@ from __future__ import annotations import json +import os import subprocess import time from abc import ABC, abstractmethod @@ -187,7 +188,8 @@ def create_runner(spec: str, *, api_key: str | None = None) -> LLMRunner: raise ValueError( f"openai-compat spec must be 'openai-compat:@', got: {spec}" ) - return OpenAICompatRunner(model=model, base_url=base_url, api_key=api_key) + resolved_api_key = api_key if api_key is not None else os.getenv("OPENAI_API_KEY") + return OpenAICompatRunner(model=model, base_url=base_url, api_key=resolved_api_key) raise ValueError( f"Unknown runner spec '{spec}'. Expected 'claude:' or " f"'openai-compat:@'." diff --git a/benchmarks/src/basic_memory_benchmarks/models.py b/benchmarks/src/basic_memory_benchmarks/models.py index a7d88c4aa..d5fd71772 100644 --- a/benchmarks/src/basic_memory_benchmarks/models.py +++ b/benchmarks/src/basic_memory_benchmarks/models.py @@ -79,26 +79,6 @@ class RetrievalSummary(BaseModel): adversarial_breakout: RetrievalMetrics -class JudgeCaseResult(BaseModel): - provider: str - query_id: str - category: str - passed: bool - score: float - reason: str - evaluator: str - - -class JudgeSummary(BaseModel): - provider: str - evaluator: str - model: str - total_cases: int - pass_count: int - accuracy: float - skipped_reason: str | None = None - - class QACategoryMetrics(BaseModel): total: int = 0 correct: int = 0 @@ -213,8 +193,6 @@ class RunConfig(BaseModel): top_k: int = 10 bm_source: str = "github:basicmachines-co/basic-memory@main" bm_local_path: str | None = None - judge_enabled: bool = False - judge_model: str = "gpt-4o-mini" allow_provider_skip: bool = True @@ -237,8 +215,6 @@ class RunArtifacts(BaseModel): provider_status: list[ProviderStatus] retrieval_summaries: list[RetrievalSummary] retrieval_rows: list[PerQueryRetrievalResult] - judge_summaries: list[JudgeSummary] = Field(default_factory=list) - judge_rows: list[JudgeCaseResult] = Field(default_factory=list) fairness_warnings: list[str] = Field(default_factory=list) diff --git a/benchmarks/src/basic_memory_benchmarks/reporting/artifacts.py b/benchmarks/src/basic_memory_benchmarks/reporting/artifacts.py index 6ca49d22c..b673422fa 100644 --- a/benchmarks/src/basic_memory_benchmarks/reporting/artifacts.py +++ b/benchmarks/src/basic_memory_benchmarks/reporting/artifacts.py @@ -6,8 +6,6 @@ from pathlib import Path from basic_memory_benchmarks.models import ( - JudgeCaseResult, - JudgeSummary, PerQueryRetrievalResult, ProviderStatus, RetrievalSummary, @@ -35,8 +33,6 @@ def write_artifacts( retrieval_rows: list[PerQueryRetrievalResult], retrieval_summaries: list[RetrievalSummary], fairness_warnings: list[str], - judge_rows: list[JudgeCaseResult] | None = None, - judge_summaries: list[JudgeSummary] | None = None, ) -> None: run_dir.mkdir(parents=True, exist_ok=True) _write_json(run_dir / "manifest.json", manifest.model_dump(mode="json")) @@ -56,25 +52,11 @@ def write_artifacts( }, ) - if judge_rows is not None: - _write_jsonl( - run_dir / "per-query-judge.jsonl", - [row.model_dump(mode="json") for row in judge_rows], - ) - if judge_summaries is not None: - _write_json( - run_dir / "judge-summary.json", - { - "providers": [row.model_dump(mode="json") for row in judge_summaries], - }, - ) - summary_markdown = build_summary_markdown( manifest=manifest, provider_status=provider_status, retrieval_summaries=retrieval_summaries, fairness_warnings=fairness_warnings, - judge_summaries=judge_summaries or [], ) (run_dir / "summary.md").write_text(summary_markdown, encoding="utf-8") @@ -85,7 +67,6 @@ def build_summary_markdown( provider_status: list[ProviderStatus], retrieval_summaries: list[RetrievalSummary], fairness_warnings: list[str], - judge_summaries: list[JudgeSummary], ) -> str: lines: list[str] = [] lines.append(f"# Benchmark Run `{manifest.run_id}`") @@ -146,19 +127,6 @@ def build_summary_markdown( ) lines.append("") - if judge_summaries: - lines.append("## Judge Summary") - lines.append("") - lines.append("| Provider | Evaluator | Model | Cases | Accuracy | Note |") - lines.append("| --- | --- | --- | --- | --- | --- |") - for summary in judge_summaries: - lines.append( - "| " - f"{summary.provider} | {summary.evaluator} | {summary.model} | " - f"{summary.total_cases} | {summary.accuracy:.3f} | {summary.skipped_reason or ''} |" - ) - lines.append("") - if fairness_warnings: lines.append("## Fairness Warnings") lines.append("") diff --git a/benchmarks/src/basic_memory_benchmarks/runner.py b/benchmarks/src/basic_memory_benchmarks/runner.py index 2a4f63622..d769a1f06 100644 --- a/benchmarks/src/basic_memory_benchmarks/runner.py +++ b/benchmarks/src/basic_memory_benchmarks/runner.py @@ -22,7 +22,6 @@ from basic_memory_benchmarks.providers import create_provider from basic_memory_benchmarks.providers.base import BenchmarkProvider from basic_memory_benchmarks.reporting.artifacts import write_artifacts -from basic_memory_benchmarks.scoring.judge import run_optional_judge from basic_memory_benchmarks.scoring.retrieval import evaluate_query, summarize_provider from basic_memory_benchmarks.utils import ( git_sha, @@ -487,50 +486,3 @@ def run_rejudge_stage( encoding="utf-8", ) return run_dir - - -def run_judge( - *, - run_dir: Path, - model: str, -) -> Path: - retrieval_path = run_dir / "per-query-retrieval.jsonl" - if not retrieval_path.exists(): - raise FileNotFoundError(f"Missing retrieval artifact: {retrieval_path}") - - rows: list[PerQueryRetrievalResult] = [] - with retrieval_path.open("r", encoding="utf-8") as file: - for line in file: - line = line.strip() - if not line: - continue - rows.append(PerQueryRetrievalResult.model_validate(json.loads(line))) - - grouped: dict[str, list[PerQueryRetrievalResult]] = {} - for row in rows: - grouped.setdefault(row.provider, []).append(row) - - judge_rows = [] - judge_summaries = [] - for provider, provider_rows in grouped.items(): - provider_case_results, provider_summary = run_optional_judge( - provider_rows, - provider=provider, - model=model, - ) - judge_rows.extend(provider_case_results) - judge_summaries.append(provider_summary) - - judge_jsonl = run_dir / "per-query-judge.jsonl" - with judge_jsonl.open("w", encoding="utf-8") as file: - for row in judge_rows: - file.write(json.dumps(row.model_dump(mode="json"), sort_keys=True) + "\n") - - judge_summary_path = run_dir / "judge-summary.json" - judge_summary_path.write_text( - json.dumps( - {"providers": [item.model_dump(mode="json") for item in judge_summaries]}, indent=2 - ), - encoding="utf-8", - ) - return run_dir diff --git a/benchmarks/src/basic_memory_benchmarks/scoring/judge.py b/benchmarks/src/basic_memory_benchmarks/scoring/judge.py deleted file mode 100644 index 755205559..000000000 --- a/benchmarks/src/basic_memory_benchmarks/scoring/judge.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Optional judge scoring. - -Primary mode tries Pydantic Evals. If unavailable or misconfigured, it falls back to -an explicit deterministic checker and records the reason. -""" - -from __future__ import annotations - -import os -from typing import Any - -from basic_memory_benchmarks.models import JudgeCaseResult, JudgeSummary, PerQueryRetrievalResult - - -def _deterministic_contains_eval( - rows: list[PerQueryRetrievalResult], provider: str -) -> tuple[list[JudgeCaseResult], JudgeSummary]: - case_results: list[JudgeCaseResult] = [] - for row in rows: - expected = (row.expected_answer or "").strip().lower() - context = row.retrieved_context.strip().lower() - passed = bool(expected and expected in context) - case_results.append( - JudgeCaseResult( - provider=provider, - query_id=row.query_id, - category=row.category, - passed=passed, - score=1.0 if passed else 0.0, - reason="Deterministic contains check", - evaluator="deterministic-fallback", - ) - ) - - pass_count = sum(1 for item in case_results if item.passed) - total = len(case_results) - summary = JudgeSummary( - provider=provider, - evaluator="deterministic-fallback", - model="none", - total_cases=total, - pass_count=pass_count, - accuracy=(pass_count / total) if total else 0.0, - ) - return case_results, summary - - -def run_optional_judge( - rows: list[PerQueryRetrievalResult], - provider: str, - model: str, -) -> tuple[list[JudgeCaseResult], JudgeSummary]: - """Run optional judge scoring. - - If pydantic-evals + OpenAI credentials are available, we attempt to use it. - Otherwise we return deterministic fallback scores. - """ - relevant_rows = [row for row in rows if row.expected_answer] - if not relevant_rows: - return [], JudgeSummary( - provider=provider, - evaluator="none", - model=model, - total_cases=0, - pass_count=0, - accuracy=0.0, - skipped_reason="No expected answers available for judge scoring", - ) - - if not os.getenv("OPENAI_API_KEY"): - case_results, summary = _deterministic_contains_eval(relevant_rows, provider) - summary.skipped_reason = "OPENAI_API_KEY missing; used deterministic fallback" - return case_results, summary - - try: - from pydantic_evals import Case, Dataset # type: ignore - from pydantic_evals.evaluators import LLMJudge # type: ignore - except Exception: - case_results, summary = _deterministic_contains_eval(relevant_rows, provider) - summary.skipped_reason = "pydantic-evals not installed; used deterministic fallback" - return case_results, summary - - # Best-effort pydantic-evals integration. - # Trigger: runtime has judge deps + API key - # Why: align with competitor methodology - # Outcome: if API shape changes, fallback remains deterministic and explicit - try: - cases: list[Any] = [] - for row in relevant_rows: - rubric = ( - "PASS if the candidate response contains the same core factual answer as expected. " - "FAIL if key facts are missing or contradictory." - ) - candidate = row.retrieved_context - expected = row.expected_answer or "" - cases.append( - Case( - name=row.query_id, - inputs=f"question: {row.query_text}\nexpected: {expected}\ncandidate: {candidate}", - evaluators=[LLMJudge(rubric=rubric, include_input=True, model=model)], - metadata={"provider": provider, "category": row.category}, - ) - ) - - dataset = Dataset(cases=cases) - report = dataset.evaluate_sync(lambda value: value) - - case_results: list[JudgeCaseResult] = [] - pass_count = 0 - for idx, row in enumerate(relevant_rows): - case_report = report.cases[idx] - passed = bool(getattr(case_report, "passed", False)) - pass_count += 1 if passed else 0 - reason = str(getattr(case_report, "reason", "")) or "LLM judge result" - case_results.append( - JudgeCaseResult( - provider=provider, - query_id=row.query_id, - category=row.category, - passed=passed, - score=1.0 if passed else 0.0, - reason=reason, - evaluator="pydantic-evals-llmjudge", - ) - ) - - total = len(case_results) - summary = JudgeSummary( - provider=provider, - evaluator="pydantic-evals-llmjudge", - model=model, - total_cases=total, - pass_count=pass_count, - accuracy=(pass_count / total) if total else 0.0, - ) - return case_results, summary - except Exception: - case_results, summary = _deterministic_contains_eval(relevant_rows, provider) - summary.skipped_reason = "pydantic-evals execution failed; used deterministic fallback" - return case_results, summary diff --git a/benchmarks/tests/llm/test_runners.py b/benchmarks/tests/llm/test_runners.py index 55d74a829..d1a29685d 100644 --- a/benchmarks/tests/llm/test_runners.py +++ b/benchmarks/tests/llm/test_runners.py @@ -28,6 +28,14 @@ def test_openai_compat_spec(self): assert runner.model == "llama3.1" assert runner.base_url == "http://localhost:11434/v1" + def test_openai_compat_uses_openai_api_key(self, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + + runner = create_runner("openai-compat:gpt-4o-mini@https://api.openai.com/v1") + + assert isinstance(runner, OpenAICompatRunner) + assert runner._api_key == "test-key" + def test_openai_compat_spec_requires_base_url(self): with pytest.raises(ValueError): create_runner("openai-compat:llama3.1") diff --git a/benchmarks/tests/test_cli_surface.py b/benchmarks/tests/test_cli_surface.py new file mode 100644 index 000000000..52fb92709 --- /dev/null +++ b/benchmarks/tests/test_cli_surface.py @@ -0,0 +1,24 @@ +from typer.testing import CliRunner + +from basic_memory_benchmarks.cli import app + + +runner = CliRunner() + + +def test_modern_qa_replaces_legacy_judge_command() -> None: + qa_help = runner.invoke(app, ["run", "qa", "--help"]) + legacy_judge = runner.invoke(app, ["run", "judge", "--help"]) + + assert qa_help.exit_code == 0 + assert "--answerer" in qa_help.output + assert "--judge" in qa_help.output + assert legacy_judge.exit_code != 0 + assert "No such command 'judge'" in legacy_judge.output + + +def test_full_command_has_no_legacy_judge_options() -> None: + result = runner.invoke(app, ["run", "full", "--help"]) + + assert result.exit_code == 0 + assert "--judge-model" not in result.output diff --git a/benchmarks/tests/test_manifest_schema.py b/benchmarks/tests/test_manifest_schema.py index 0c16441bd..88d5d3a1c 100644 --- a/benchmarks/tests/test_manifest_schema.py +++ b/benchmarks/tests/test_manifest_schema.py @@ -30,3 +30,5 @@ def test_manifest_schema_roundtrip() -> None: payload = manifest.model_dump(mode="json") assert payload["run_id"] == "run1" assert payload["config"]["providers"] == ["bm-local"] + assert "judge_enabled" not in payload["config"] + assert "judge_model" not in payload["config"] diff --git a/benchmarks/uv.lock b/benchmarks/uv.lock index 4948a715f..da9f5c6eb 100644 --- a/benchmarks/uv.lock +++ b/benchmarks/uv.lock @@ -216,11 +216,6 @@ dependencies = [ { name = "typer" }, ] -[package.optional-dependencies] -judge = [ - { name = "pydantic-evals" }, -] - [package.dev-dependencies] dev = [ { name = "pyright" }, @@ -236,12 +231,10 @@ requires-dist = [ { name = "mcp", specifier = ">=1.23.1" }, { name = "mem0ai", extras = ["nlp"], specifier = "==2.0.5" }, { name = "pydantic", specifier = ">=2.12.0" }, - { name = "pydantic-evals", marker = "extra == 'judge'", specifier = ">=0.4.0" }, { name = "python-frontmatter", specifier = ">=1.1.0" }, { name = "rich", specifier = ">=13.9.4" }, { name = "typer", specifier = ">=0.16.1" }, ] -provides-extras = ["judge"] [package.metadata.requires-dev] dev = [ @@ -898,19 +891,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl", hash = "sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437", size = 202505, upload-time = "2026-02-05T21:50:51.819Z" }, ] -[[package]] -name = "genai-prices" -version = "0.0.54" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/20/d64d04c7870db01232f8f8e36dec2072281b1f594b924200aa017778eec2/genai_prices-0.0.54.tar.gz", hash = "sha256:9d985affc19d055be16613b0c5e49d182d4c2164cf1587129f9996dfb9a3cb8d", size = 59588, upload-time = "2026-02-17T20:26:06.849Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/71/d2a941b0ca01186912fedb096d8eef7b3e1680c86fdcf8fe3dc84e76d5a9/genai_prices-0.0.54-py3-none-any.whl", hash = "sha256:5b45012b2981b7d4d42c49c8614ee95420fec244c87542542045786b36fc2235", size = 62198, upload-time = "2026-02-17T20:26:05.186Z" }, -] - [[package]] name = "greenlet" version = "3.3.2" @@ -954,15 +934,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/4b/45d90626aef8e65336bed690106d1382f7a43665e2249017e9527df8823b/greenlet-3.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c04c5e06ec3e022cbfe2cd4a846e1d4e50087444f875ff6d2c2ad8445495cf1a", size = 237086, upload-time = "2026-02-20T20:20:45.786Z" }, ] -[[package]] -name = "griffelib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ad/06/eccbd311c9e2b3ca45dbc063b93134c57a1ccc7607c5e545264ad092c4a9/griffelib-2.0.0.tar.gz", hash = "sha256:e504d637a089f5cab9b5daf18f7645970509bf4f53eda8d79ed71cce8bd97934", size = 166312, upload-time = "2026-03-23T21:06:55.954Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/51/c936033e16d12b627ea334aaaaf42229c37620d0f15593456ab69ab48161/griffelib-2.0.0-py3-none-any.whl", hash = "sha256:01284878c966508b6d6f1dbff9b6fa607bc062d8261c5c7253cb285b06422a7f", size = 142004, upload-time = "2026-02-09T19:09:40.561Z" }, -] - [[package]] name = "grpcio" version = "1.78.1" @@ -1387,15 +1358,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] -[[package]] -name = "logfire-api" -version = "4.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/5c/026cec30d85394aec8f5f12d70edbe2d706837bc9a411bd71a542cedae50/logfire_api-4.25.0.tar.gz", hash = "sha256:7562d5adfe3987291039dddb21947c86cb9d832d068c87d9aa23db86ef07095b", size = 75853, upload-time = "2026-02-19T15:27:29.518Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/39/83414c0fadb4f11f90e6b80b631aa79f62a605664f0c4693e2ebc7ee73f3/logfire_api-4.25.0-py3-none-any.whl", hash = "sha256:0d607eb09ef5426e26f376ff277a8d401bc5b7b4178ea66db404e13c368494cf", size = 120473, upload-time = "2026-02-19T15:27:25.832Z" }, -] - [[package]] name = "loguru" version = "0.7.3" @@ -2204,24 +2166,6 @@ timezone = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -[[package]] -name = "pydantic-ai-slim" -version = "1.63.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "genai-prices" }, - { name = "griffelib" }, - { name = "httpx" }, - { name = "opentelemetry-api" }, - { name = "pydantic" }, - { name = "pydantic-graph" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/6d/2b5c0c60b42e6af49830f6a09b5d38fecdb1f20d9659152691eba95613b4/pydantic_ai_slim-1.63.0.tar.gz", hash = "sha256:9377afecdfe4bc17f5c9ed72c758e460703ac5876931aa2f18ace8ac0e69312a", size = 426862, upload-time = "2026-02-23T17:56:36.215Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/ca/c4e39eec1cff5a294b64313a8a959b38d326819e0f0a41f48e61ce019a22/pydantic_ai_slim-1.63.0-py3-none-any.whl", hash = "sha256:ed393b0f871b748171f65bec5191c3025b5abb8a4fc616afee17eb9dc2dfa15d", size = 554190, upload-time = "2026-02-23T17:56:29.533Z" }, -] - [[package]] name = "pydantic-core" version = "2.41.5" @@ -2293,23 +2237,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, ] -[[package]] -name = "pydantic-evals" -version = "1.63.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "logfire-api" }, - { name = "pydantic" }, - { name = "pydantic-ai-slim" }, - { name = "pyyaml" }, - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/99/43/21b6ddf65b56f7401c344f98e4e6258a02d2868c8a52a8b79c0e0e701029/pydantic_evals-1.63.0.tar.gz", hash = "sha256:eed56a7192e07c8be8cf16e53bb2ef652b4f7f7b8527650ac45fde865a4ecf9d", size = 56365, upload-time = "2026-02-23T17:56:37.71Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/f2/7174ad6abca2457e35a1b902ca4fa78aa8ee72e4ec2e9cd5dc8904014ec9/pydantic_evals-1.63.0-py3-none-any.whl", hash = "sha256:2e92a3af579a5670b2babf2044081d0ef99ab5a9ef141972616d71fd7e5bfd0e", size = 67279, upload-time = "2026-02-23T17:56:31.008Z" }, -] - [[package]] name = "pydantic-extra-types" version = "2.11.0" @@ -2323,21 +2250,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/17/fabd56da47096d240dd45ba627bead0333b0cf0ee8ada9bec579287dadf3/pydantic_extra_types-2.11.0-py3-none-any.whl", hash = "sha256:84b864d250a0fc62535b7ec591e36f2c5b4d1325fa0017eb8cda9aeb63b374a6", size = 74296, upload-time = "2025-12-31T16:18:26.38Z" }, ] -[[package]] -name = "pydantic-graph" -version = "1.63.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "logfire-api" }, - { name = "pydantic" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/c8/aa3cb56552562b799f31e9de291c8bd88306308cfc9647d220dfff2bea18/pydantic_graph-1.63.0.tar.gz", hash = "sha256:5fd98bb22fa6181f0357a6ffad38a3214af12868bd46492d6456c5db434466b4", size = 58528, upload-time = "2026-02-23T17:56:39.118Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/1c/8dcae24c824dd2690fbe7375083b369b10ed1ad773e2b9d1122bb6c0fcdc/pydantic_graph-1.63.0-py3-none-any.whl", hash = "sha256:d9b7a387116f358d470c042b07aa08125cadfcfa8c08ef01769746a489aef0d5", size = 72353, upload-time = "2026-02-23T17:56:32.304Z" }, -] - [[package]] name = "pydantic-settings" version = "2.13.1" From 698bbc74f5aa7d82de915843026d340dc0c0ab95 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Mon, 24 Aug 2026 10:04:45 +0800 Subject: [PATCH 6/7] docs: document links_to directive Signed-off-by: mikemikimike <13286568797@163.com> --- docs/NOTE-FORMAT.md | 11 +++++++++++ src/basic_memory/mcp/tools/write_note.py | 3 +++ 2 files changed, 14 insertions(+) diff --git a/docs/NOTE-FORMAT.md b/docs/NOTE-FORMAT.md index be1ebb718..2d666f08a 100644 --- a/docs/NOTE-FORMAT.md +++ b/docs/NOTE-FORMAT.md @@ -166,6 +166,17 @@ Explicit relations: - 'in response to' [[Incident Review]] ``` +When the text before `[[` is ordinary prose rather than a relation type, add +`#bm:links_to` to force the reference to use the implicit `links_to` relation: + +```markdown +- Mother [[Alice]] #bm:links_to +``` + +The directive is parser metadata and is not included in the observation or +relation context. It is especially useful when a single-token prose prefix +would otherwise be interpreted as an explicit relation type. + Bare wiki links and prose list items create implicit `links_to` relations: ```markdown diff --git a/src/basic_memory/mcp/tools/write_note.py b/src/basic_memory/mcp/tools/write_note.py index edd823635..7b521ccc8 100644 --- a/src/basic_memory/mcp/tools/write_note.py +++ b/src/basic_memory/mcp/tools/write_note.py @@ -109,12 +109,15 @@ async def write_note( - Explicit: `- relation_type [[Entity]] (optional context)` - Quoted: `- "multi word relation type" [[Entity]] (optional context)` - Quoted: `- 'multi word relation type' [[Entity]] (optional context)` + - Disambiguation: Add `#bm:links_to` when prose before `[[Entity]]` + must not be treated as a single-token relation type - Inline: Any other `[[Entity]]` reference creates a `links_to` relation Examples: `- depends_on [[Content Parser]] (Need for semantic extraction)` `- "based on" [[Design Notes]]` `- 'in response to' [[Incident Review]]` + `- Mother [[Alice]] #bm:links_to` `- implements [[Search Spec]] (Initial implementation)` `- This feature extends [[Base Design]] and uses [[Core Utils]]` From f57ea66a73f26faab89a89cd1f4e3b5548e663d5 Mon Sep 17 00:00:00 2001 From: Paul Hernandez <60959+phernandez@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:05:55 -0500 Subject: [PATCH 7/7] fix(cli): route Team WebDAV transfers to the control-plane base (#1310) Signed-off-by: phernandez Co-authored-by: Claude Fable 5 Signed-off-by: mikemikimike <13286568797@163.com> --- .../cli/commands/cloud/webdav_transfer.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/cli/commands/cloud/webdav_transfer.py b/src/basic_memory/cli/commands/cloud/webdav_transfer.py index 0c824f1c3..26cfd22f7 100644 --- a/src/basic_memory/cli/commands/cloud/webdav_transfer.py +++ b/src/basic_memory/cli/commands/cloud/webdav_transfer.py @@ -56,7 +56,7 @@ upload_file, ) from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path -from basic_memory.mcp.async_client import get_cloud_proxy_client +from basic_memory.mcp.async_client import get_cloud_control_plane_client console = Console() @@ -123,7 +123,13 @@ async def webdav_project_diff( Raises: WebdavError: If the project cannot be listed. """ - cm_factory = client_cm_factory or partial(get_cloud_proxy_client, workspace=workspace_id) + # The tenant WebDAV surface is mounted at the cloud app root (/webdav), + # not behind /proxy: the proxy catch-alls forward to the per-tenant core + # instance, which serves no WebDAV routes, so the proxy client 404s on + # every listing (cloud#1816). + cm_factory = client_cm_factory or partial( + get_cloud_control_plane_client, workspace=workspace_id + ) async with cm_factory() as client: remote_files = await list_project_files(client, project) @@ -188,7 +194,11 @@ async def webdav_project_transfer( console.print(f" [dim]{transfer.describe()}[/dim]") return - cm_factory = client_cm_factory or partial(get_cloud_proxy_client, workspace=workspace_id) + # Root-mounted /webdav needs the control-plane base — see the note in + # webdav_project_diff (cloud#1816). + cm_factory = client_cm_factory or partial( + get_cloud_control_plane_client, workspace=workspace_id + ) async with cm_factory() as client: if direction == "push": transfers, appeared = await _drop_appeared_on_cloud(client, project, transfers)