From 51904337ab98e42a935b5d62fe238f83bbc102fb Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark" Date: Sun, 2 Aug 2026 06:36:43 -0500 Subject: [PATCH] =?UTF-8?q?feat(graph):=20rank=20the=20frontier=20?= =?UTF-8?q?=E2=80=94=20what=20to=20consolidate=20next?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every existing graph op answers a structural yes/no question: is this note connected, is that link broken, how big is the graph. None answered the ranking question — of everything in the vault, what should be worked on next? score = (out_degree - in_degree) * 0.5 ** (days_since_updated / 30) A high score means the note points at many things, few point at it, and it was touched recently: memory is accumulating there and consolidation has not caught up. A negative score is a hub the vault has already absorbed. `orphans` finds notes that are *disconnected*; a frontier note is connected but **unabsorbed**, which is a different and more actionable state. This complements `omind consolidate`, which finds merge candidates by similarity. Frontier finds them by structure, catching "this note has sprawled outward for weeks and nothing pulled it together" rather than "these two notes say the same thing". Journals, worklogs, and checkpoints are excluded by default — they link outward at everything by construction and would otherwise fill the whole ranking, the same reason searchindex._weight_generated de-prioritises them. Read-only, with no write path, following the source design's instinct. Costs no new scan and no new state: the links table was already built. The one addition is surfacing `mtime_ns` from the index's notes table, which the recency decay reads. Closes #197. Co-Authored-By: Claude Opus 5 --- BACKLOG.md | 5 +- CHANGELOG.md | 28 ++++++++++ README.md | 5 ++ pyproject.toml | 2 +- src/omind/__init__.py | 2 +- src/omind/cli.py | 38 ++++++++++++-- src/omind/graph.py | 107 ++++++++++++++++++++++++++++++++++++--- src/omind/searchindex.py | 6 ++- src/omind/server.py | 23 ++++++++- tests/test_graph.py | 66 ++++++++++++++++++++++++ tests/test_server.py | 7 +++ uv.lock | 2 +- 12 files changed, 274 insertions(+), 17 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index c83afa1..f6aba49 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -88,7 +88,10 @@ five places their design is genuinely better and the idea transfers._ where no automated verifier exists. omind's `doctor` checks are hand-written per concern with no declaration of what each surface may touch, and nothing fails when code and declaration drift. Natural home for the #190 `serve` risk model. -- [ ] **Frontier/boundary scoring to rank what to consolidate next** ([#197](https://github.com/CryptoJones/omind/issues/197)) — _enhancement (efficiency)_ — +- [x] **Frontier/boundary scoring to rank what to consolidate next** ([#197](https://github.com/CryptoJones/omind/issues/197)) — _enhancement (efficiency)_ — + shipped as `omind graph frontier` and `graph(op="frontier")`: + `(out - in) * 0.5 ** (days/30)`, generated notes excluded by default, read-only, + no new scan or state. Original description follows. `(out_degree - in_degree) * recency_weight` finds notes that point outward, are pointed at by few, and were touched recently. Every `omind graph` op answers a structural yes/no question; none rank what to work on next. Complements diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f61a54..7523e36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.5.0] - 2026-08-02 + +### Added +- **Frontier scoring: `omind graph frontier` and `graph(op="frontier")`** + ([#197](https://github.com/CryptoJones/omind/issues/197)). Every existing graph + op answers a structural yes/no question — is this note connected, is that link + broken, how big is the graph. None answered the *ranking* question: of + everything in the vault, what should be worked on next? + + `(out_degree - in_degree) * 0.5 ** (days_since_updated / 30)`. A high score + means the note points at many things, few point at it, and it was touched + recently — memory is accumulating there and consolidation has not caught up. A + negative score is a hub the vault has already absorbed. An *orphan* is + disconnected; a *frontier* note is connected but **unabsorbed**, which is the + more actionable state. + + Complements `omind consolidate`, which finds merge candidates by similarity: + this finds them by structure, catching "this note has sprawled outward for + weeks" rather than "these two notes say the same thing". Read-only, with no + write path by design. Machine-written notes (journals, worklogs, checkpoints) + are excluded by default — they link outward at everything by construction and + would otherwise fill the entire ranking — with `--include-generated` to see + them. Costs no new scan or state: the `links` table was already built. + +### Changed +- The search index's note rows carry `mtime_ns` (already stored in the `notes` + table, previously not surfaced), which is what the recency decay reads. + ## [6.4.0] - 2026-08-02 ### Added diff --git a/README.md b/README.md index c25e0c1..3527029 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ OMI/Obsidian memory tooling for AI agents: reproduce the integration on any mach *`omind graph` over an OMI vault — every note a node **coloured by its OKF `type`** (and sized by link degree), every `[[wikilink]]` an edge. Rendered from `omind graph export` (see [docs/graph-demo](docs/graph-demo/)).* +`omind graph frontier` ranks that same graph by what to work on next: notes that +reach out to many others, that few reach back to, and that were touched recently +— memory is accumulating there and nothing has pulled it together yet. An +*orphan* is disconnected; a *frontier* note is connected but unabsorbed. + ## What it does **OMI** ("Open Mind Interface") is a folder of Markdown notes that an AI agent diff --git a/pyproject.toml b/pyproject.toml index 6b21af3..8103fd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "omind" -version = "6.4.0" +version = "6.5.0" description = "Reproduce the OMI/Obsidian memory integration for AI agents, plus a local web app to view, edit, and add memory entries." readme = "README.md" requires-python = ">=3.10" diff --git a/src/omind/__init__.py b/src/omind/__init__.py index 2d1d449..ad3054c 100644 --- a/src/omind/__init__.py +++ b/src/omind/__init__.py @@ -2,4 +2,4 @@ # Copyright 2026 Aaron K. Clark """omind — OMI/Obsidian memory tooling for AI agents.""" -__version__ = "6.4.0" +__version__ = "6.5.0" diff --git a/src/omind/cli.py b/src/omind/cli.py index 8d6c6c9..5e90bbe 100644 --- a/src/omind/cli.py +++ b/src/omind/cli.py @@ -418,11 +418,11 @@ def build_parser() -> argparse.ArgumentParser: graph_p = sub.add_parser( "graph", help="query the [[wikilink]] knowledge graph: neighbors, path, orphans, " - "dangling links, stats, export", + "dangling links, stats, frontier, export", ) gsub = graph_p.add_subparsers( dest="graph_command", - metavar="{neighbors,path,orphans,dangling,stats,export}", + metavar="{neighbors,path,orphans,dangling,stats,frontier,export}", required=True, ) g_neighbors = gsub.add_parser("neighbors", help="notes within N hops of a note") @@ -442,6 +442,20 @@ def build_parser() -> argparse.ArgumentParser: g_orphans = gsub.add_parser("orphans", help="notes with no inbound or outbound links") g_dangling = gsub.add_parser("dangling", help="wikilinks pointing at no existing note") g_stats = gsub.add_parser("stats", help="counts: notes, links, orphans, dangling") + g_frontier = gsub.add_parser( + "frontier", + help="rank notes by how far they reach out beyond what reaches back (what to " + "consolidate next)", + ) + g_frontier.add_argument( + "--limit", type=int, default=10, help="how many to show (default: 10; 0 = all)" + ) + g_frontier.add_argument( + "--include-generated", + action="store_true", + help="include journals/worklogs, which link outward at everything by construction", + ) + g_frontier.add_argument("--json", action="store_true", help="emit JSON instead of a table") g_export = gsub.add_parser("export", help="dump the whole graph for visualization") g_export.add_argument( "--format", @@ -449,7 +463,7 @@ def build_parser() -> argparse.ArgumentParser: default="json", help="output format (default: json)", ) - for gp in (g_neighbors, g_path, g_orphans, g_dangling, g_stats, g_export): + for gp in (g_neighbors, g_path, g_orphans, g_dangling, g_stats, g_frontier, g_export): _add_vault_args(gp) checkpoint = sub.add_parser( @@ -1125,6 +1139,7 @@ def _run_lint(args: argparse.Namespace) -> int: def _run_graph(args: argparse.Namespace) -> int: import json + from dataclasses import asdict from omind import graph as graphmod @@ -1167,6 +1182,23 @@ def _run_graph(args: argparse.Namespace) -> int: if cmd == "stats": print(json.dumps(graphmod.stats(g), indent=2)) return 0 + if cmd == "frontier": + ranked = graphmod.frontier( + g, limit=args.limit, include_generated=args.include_generated + ) + if args.json: + print(json.dumps([asdict(entry) for entry in ranked], indent=2)) + return 0 + if not ranked: + print("no notes to rank") + return 0 + print("score\tout\tin\tdays\tnote") + for entry in ranked: + print( + f"{entry.score:+.2f}\t{entry.out_degree}\t{entry.in_degree}\t" + f"{entry.days_since_updated:.0f}\t{entry.filename}" + ) + return 0 # cmd == "export" if args.format == "dot": print(graphmod.to_dot(g)) diff --git a/src/omind/graph.py b/src/omind/graph.py index 896b00d..8797c48 100644 --- a/src/omind/graph.py +++ b/src/omind/graph.py @@ -6,7 +6,8 @@ store already answers the inbound question (``backlinks``) and ``lint`` already flags orphans and broken links one note at a time — this module assembles the *whole-graph* view those leave out: forward links, multi-hop neighborhoods, -the shortest link path between two notes, and a JSON/Graphviz-DOT export. +the shortest link path between two notes, a frontier ranking of what to +consolidate next, and a JSON/Graphviz-DOT export. Resolution mirrors :meth:`OmiStore.backlinks` and ``lint``: a link ``[[Target]]`` (its ``|alias`` and ``#heading`` stripped) resolves to a note when its lowercased @@ -43,6 +44,7 @@ class GraphNode: okf_type: str = "" # the note's OKF ``type`` — for grouping/colouring the graph out: set[str] = field(default_factory=set) # filenames this note links to inn: set[str] = field(default_factory=set) # filenames that link to this note + mtime_ns: int = 0 # last write; only :func:`frontier` reads it @dataclass @@ -102,7 +104,12 @@ def _from_index(omi_dir: Path | str) -> Graph | None: if row.title.strip(): id_to_file[row.title.strip().lower()] = filename nodes = { - filename: GraphNode(filename=filename, title=row.title.strip(), okf_type=row.okf_type) + filename: GraphNode( + filename=filename, + title=row.title.strip(), + okf_type=row.okf_type, + mtime_ns=row.mtime_ns, + ) for filename, row in live.items() } dangling: list[tuple[str, str]] = [] @@ -123,7 +130,7 @@ def _from_disk(omi_dir: Path | str) -> Graph: omi = Path(omi_dir) # (filename, title, raw outbound targets) for each live note, plus an index # from every linkable identifier (stem + title, lowercased) to its filename. - parsed: list[tuple[str, str, str, set[str]]] = [] + parsed: list[tuple[str, str, str, set[str], int]] = [] id_to_file: dict[str, str] = {} if omi.is_dir(): for path in sorted(omi.glob("*.md")): @@ -141,17 +148,21 @@ def _from_disk(omi_dir: Path | str) -> Graph: # same rule render_fields uses — so every node carries a non-empty type. okf_type = fields.okf_type.strip() or derive_okf_type(fields.tags) targets = {t for t in (_link_target(m) for m in _WIKILINK_RE.findall(text)) if t} - parsed.append((path.name, title, okf_type, targets)) + try: + mtime_ns = path.stat().st_mtime_ns + except OSError: + mtime_ns = 0 + parsed.append((path.name, title, okf_type, targets, mtime_ns)) id_to_file[path.stem.strip().lower()] = path.name if title: id_to_file[title.lower()] = path.name nodes = { - fn: GraphNode(filename=fn, title=title, okf_type=okf_type) - for fn, title, okf_type, _ in parsed + fn: GraphNode(filename=fn, title=title, okf_type=okf_type, mtime_ns=mtime_ns) + for fn, title, okf_type, _, mtime_ns in parsed } dangling: list[tuple[str, str]] = [] - for src, _title, _type, targets in parsed: + for src, _title, _type, targets, _mtime in parsed: for target in sorted(targets): dest = id_to_file.get(target.lower()) if dest is None: @@ -233,6 +244,88 @@ def dangling_links(graph: Graph) -> list[tuple[str, str]]: return sorted(graph.dangling) +#: Half-life of the recency weight, in days: a note untouched for this long +#: counts half as much as one touched today, and half again after another. +FRONTIER_HALFLIFE_DAYS = 30.0 + +#: OKF types written by machines rather than curated. They link outward at +#: everything by construction, so they would monopolise a frontier ranking. +#: Mirrors the de-prioritisation in ``searchindex._weight_generated``. +_GENERATED_TYPES = frozenset({"journal", "worklog", "checkpoint", "rollup"}) + + +def _is_generated(node: GraphNode) -> bool: + """Whether a node is machine-written (auto-journal, worklog, checkpoint).""" + stem = node.filename[:-3].lower() if node.filename.endswith(".md") else node.filename.lower() + return ( + node.okf_type.strip().lower() in _GENERATED_TYPES + or stem.startswith("session journal") + or stem.startswith("worklog ") + ) + + +@dataclass(frozen=True) +class FrontierEntry: + """One note's frontier score and the components behind it.""" + + filename: str + title: str + score: float + out_degree: int + in_degree: int + days_since_updated: float + + +def frontier( + graph: Graph, + limit: int = 10, + *, + include_generated: bool = False, + now: float | None = None, +) -> list[FrontierEntry]: + """Rank notes by how far they reach out beyond what reaches back at them. + + ``(out_degree - in_degree) * 0.5 ** (days_since_updated / half-life)``. + + A high score means the note points at many things, few things point at it, + and it was touched recently — memory is actively accumulating there and + nothing has pulled it together yet. A low or negative score means a hub: an + absorbed note the rest of the vault already refers to. + + This is the ranking question the other graph ops don't answer. ``orphans`` + finds notes that are *disconnected*; a frontier note is connected but + **unabsorbed**, which is a different and more actionable state. It + complements ``omind consolidate``, which finds merge candidates by + similarity — this finds them by structure, catching "this note has sprawled + outward for weeks" rather than "these two notes say the same thing". + + Machine-written notes (journals, worklogs, checkpoints) are excluded by + default: they link outward at everything by construction and would fill the + whole ranking. Read-only, like every other op here. + """ + import time + + reference = time.time() if now is None else now + entries: list[FrontierEntry] = [] + for node in graph.nodes.values(): + if not include_generated and _is_generated(node): + continue + days = max(0.0, (reference - node.mtime_ns / 1_000_000_000) / 86_400.0) + weight = 0.5 ** (days / FRONTIER_HALFLIFE_DAYS) + entries.append( + FrontierEntry( + filename=node.filename, + title=node.title, + score=(len(node.out) - len(node.inn)) * weight, + out_degree=len(node.out), + in_degree=len(node.inn), + days_since_updated=days, + ) + ) + entries.sort(key=lambda e: (-e.score, e.filename.lower())) + return entries[:limit] if limit > 0 else entries + + def stats(graph: Graph) -> dict[str, int]: """Whole-graph counts: notes, directed links, orphans, and dangling links.""" return { diff --git a/src/omind/searchindex.py b/src/omind/searchindex.py index 1ced40b..45f35ad 100644 --- a/src/omind/searchindex.py +++ b/src/omind/searchindex.py @@ -201,6 +201,9 @@ class _NoteRow: has_title: bool = True tags: list[str] = field(default_factory=list) disabled: bool = False + #: Last write time recorded at ingest. Only the graph's frontier scoring + #: reads it; retrieval ranks on content and the `created` field. + mtime_ns: int = 0 _SCHEMA = """ @@ -1242,10 +1245,11 @@ def _notes(db: sqlite3.Connection) -> list[_NoteRow]: has_title=bool(r["has_title"]), tags=tags.get(str(r["filename"]), []), disabled=bool(r["disabled"]), + mtime_ns=int(r["mtime_ns"]), ) for r in db.execute( "SELECT filename, title, created, okf_type, supersedes, superseded_by," - " has_title, disabled FROM notes" + " has_title, disabled, mtime_ns FROM notes" ) ] diff --git a/src/omind/server.py b/src/omind/server.py index 55bb5b0..4c138ad 100644 --- a/src/omind/server.py +++ b/src/omind/server.py @@ -457,13 +457,32 @@ def _graph_query( return _page(rows, limit, offset) if operation == "stats": return dict(graph.stats(graph_for())) - raise ValueError("graph op must be one of: path, orphans, dangling, stats") + if operation == "frontier": + # Paged like every other list-shaped result (invariant 7): the caller + # asks for a page, the ranking is computed over the whole graph. + ranked = [ + { + "filename": entry.filename, + "title": entry.title, + "score": round(entry.score, 4), + "out_degree": entry.out_degree, + "in_degree": entry.in_degree, + "days_since_updated": round(entry.days_since_updated, 1), + } + for entry in graph.frontier(graph_for(), limit=0) + ] + return _page(ranked, limit, offset) + raise ValueError( + "graph op must be one of: path, orphans, dangling, stats, frontier" + ) @mcp.tool( name="graph", description=( "Graph audit/query selected by op: path (requires source + target), " - "orphans, dangling, or stats. Orphan/dangling results are paged." + "orphans, dangling, stats, or frontier (notes that reach out further " + "than anything reaches back — what to consolidate next). List-shaped " + "results are paged." ), ) def graph_tool( diff --git a/tests/test_graph.py b/tests/test_graph.py index 42c8678..0539dbb 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -5,6 +5,8 @@ from __future__ import annotations import json +import os +import time from pathlib import Path import pytest @@ -103,6 +105,70 @@ def test_stats_counts(store: OmiStore) -> None: assert graph.stats(g) == {"notes": 5, "links": 3, "orphans": 1, "dangling": 1} +def test_frontier_ranks_outward_reaching_notes_above_hubs(store: OmiStore) -> None: + """A note that points at many and is pointed at by few is the frontier.""" + store.create_note(NoteFields(title="Hub", summary="everyone links here")) + store.create_note(NoteFields(title="Edge", summary="see [[Hub]], [[P1]], [[P2]]")) + store.create_note(NoteFields(title="P1", summary="see [[Hub]]")) + store.create_note(NoteFields(title="P2", summary="see [[Hub]]")) + g = graph.build_graph(store.omi_dir) + + ranked = graph.frontier(g) + by_name = {entry.filename: entry for entry in ranked} + # Edge: 3 out, 0 in. Hub: 0 out, 3 in. + assert ranked[0].filename == "Edge.md" + assert by_name["Edge.md"].out_degree == 3 and by_name["Edge.md"].in_degree == 0 + assert by_name["Hub.md"].score < 0 # an absorbed hub, not a frontier + assert ranked[-1].filename == "Hub.md" + + +def test_frontier_decays_with_staleness(store: OmiStore) -> None: + """Two identically-shaped notes rank by when they were last touched.""" + store.create_note(NoteFields(title="Fresh", summary="see [[T1]], [[T2]]")) + store.create_note(NoteFields(title="Stale", summary="see [[T1]], [[T2]]")) + store.create_note(NoteFields(title="T1", summary="target")) + store.create_note(NoteFields(title="T2", summary="target")) + stale = store.omi_dir / "Stale.md" + old = time.time() - graph.FRONTIER_HALFLIFE_DAYS * 86_400 # exactly one half-life + os.utime(stale, (old, old)) + + g = graph._from_disk(store.omi_dir) # mtime, not the index's ingest snapshot + by_name = {entry.filename: entry for entry in graph.frontier(g)} + assert by_name["Fresh.md"].score > by_name["Stale.md"].score + # One half-life back is worth half as much, within a tolerance for the + # seconds that elapsed between writing the note and reading the clock. + assert by_name["Stale.md"].score == pytest.approx( + by_name["Fresh.md"].score / 2, rel=0.01 + ) + + +def test_frontier_excludes_machine_written_notes_by_default(store: OmiStore) -> None: + """Journals link outward at everything and would own the whole ranking.""" + store.create_note(NoteFields(title="Real", summary="see [[T1]]")) + store.create_note(NoteFields(title="T1", summary="target")) + store.create_note( + NoteFields( + title="Worklog 2026-08-02", + summary="see [[Real]], [[T1]]", + okf_type="worklog", + ) + ) + g = graph.build_graph(store.omi_dir) + + assert "Worklog 2026-08-02.md" not in {e.filename for e in graph.frontier(g)} + included = {e.filename for e in graph.frontier(g, include_generated=True)} + assert "Worklog 2026-08-02.md" in included + + +def test_frontier_limit_zero_returns_everything_ranked(store: OmiStore) -> None: + _chain(store) + g = graph.build_graph(store.omi_dir) + assert len(graph.frontier(g, limit=0)) == 5 + assert len(graph.frontier(g, limit=2)) == 2 + scores = [entry.score for entry in graph.frontier(g, limit=0)] + assert scores == sorted(scores, reverse=True) + + def test_to_json_shape(store: OmiStore) -> None: _chain(store) g = graph.build_graph(store.omi_dir) diff --git a/tests/test_server.py b/tests/test_server.py index 613a587..60f72aa 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -192,6 +192,7 @@ def test_every_list_tool_is_bounded(server: MCPServer) -> None: ("graph-neighbors", {"name": "Linker", "limit": 1}), ("graph", {"op": "orphans", "limit": 1}), ("graph", {"op": "dangling", "limit": 1}), + ("graph", {"op": "frontier", "limit": 1}), ): page = call(server, tool, args) assert set(page) >= {"result", "count", "offset", "total", "has_more"}, tool @@ -284,6 +285,12 @@ def test_graph_tools(server: MCPServer) -> None: ] assert call(server, "graph", {"op": "stats"})["notes"] == 4 + ranked = call(server, "graph", {"op": "frontier"})["result"] + # A links B, B links C, and nothing links A: A is the frontier, C the sink. + assert ranked[0]["filename"] == "A.md" + assert ranked[0]["out_degree"] == 1 and ranked[0]["in_degree"] == 0 + assert ranked[-1]["filename"] == "C.md" + def test_unified_graph_validates_operation_and_path_arguments(server: MCPServer) -> None: with pytest.raises(ToolError, match="one of"): diff --git a/uv.lock b/uv.lock index 3503ba2..0415a4b 100644 --- a/uv.lock +++ b/uv.lock @@ -2354,7 +2354,7 @@ wheels = [ [[package]] name = "omind" -version = "6.4.0" +version = "6.5.0" source = { editable = "." } dependencies = [ { name = "cryptography" },