diff --git a/BACKLOG.md b/BACKLOG.md index c83afa1..f6aba49 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -88,7 +88,10 @@ five places their design is genuinely better and the idea transfers._ where no automated verifier exists. omind's `doctor` checks are hand-written per concern with no declaration of what each surface may touch, and nothing fails when code and declaration drift. Natural home for the #190 `serve` risk model. -- [ ] **Frontier/boundary scoring to rank what to consolidate next** ([#197](https://github.com/CryptoJones/omind/issues/197)) — _enhancement (efficiency)_ — +- [x] **Frontier/boundary scoring to rank what to consolidate next** ([#197](https://github.com/CryptoJones/omind/issues/197)) — _enhancement (efficiency)_ — + shipped as `omind graph frontier` and `graph(op="frontier")`: + `(out - in) * 0.5 ** (days/30)`, generated notes excluded by default, read-only, + no new scan or state. Original description follows. `(out_degree - in_degree) * recency_weight` finds notes that point outward, are pointed at by few, and were touched recently. Every `omind graph` op answers a structural yes/no question; none rank what to work on next. Complements diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f61a54..7523e36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.5.0] - 2026-08-02 + +### Added +- **Frontier scoring: `omind graph frontier` and `graph(op="frontier")`** + ([#197](https://github.com/CryptoJones/omind/issues/197)). Every existing graph + op answers a structural yes/no question — is this note connected, is that link + broken, how big is the graph. None answered the *ranking* question: of + everything in the vault, what should be worked on next? + + `(out_degree - in_degree) * 0.5 ** (days_since_updated / 30)`. A high score + means the note points at many things, few point at it, and it was touched + recently — memory is accumulating there and consolidation has not caught up. A + negative score is a hub the vault has already absorbed. An *orphan* is + disconnected; a *frontier* note is connected but **unabsorbed**, which is the + more actionable state. + + Complements `omind consolidate`, which finds merge candidates by similarity: + this finds them by structure, catching "this note has sprawled outward for + weeks" rather than "these two notes say the same thing". Read-only, with no + write path by design. Machine-written notes (journals, worklogs, checkpoints) + are excluded by default — they link outward at everything by construction and + would otherwise fill the entire ranking — with `--include-generated` to see + them. Costs no new scan or state: the `links` table was already built. + +### Changed +- The search index's note rows carry `mtime_ns` (already stored in the `notes` + table, previously not surfaced), which is what the recency decay reads. + ## [6.4.0] - 2026-08-02 ### Added diff --git a/README.md b/README.md index c25e0c1..3527029 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ OMI/Obsidian memory tooling for AI agents: reproduce the integration on any mach *`omind graph` over an OMI vault — every note a node **coloured by its OKF `type`** (and sized by link degree), every `[[wikilink]]` an edge. Rendered from `omind graph export` (see [docs/graph-demo](docs/graph-demo/)).* +`omind graph frontier` ranks that same graph by what to work on next: notes that +reach out to many others, that few reach back to, and that were touched recently +— memory is accumulating there and nothing has pulled it together yet. An +*orphan* is disconnected; a *frontier* note is connected but unabsorbed. + ## What it does **OMI** ("Open Mind Interface") is a folder of Markdown notes that an AI agent diff --git a/pyproject.toml b/pyproject.toml index 6b21af3..8103fd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "omind" -version = "6.4.0" +version = "6.5.0" description = "Reproduce the OMI/Obsidian memory integration for AI agents, plus a local web app to view, edit, and add memory entries." readme = "README.md" requires-python = ">=3.10" diff --git a/src/omind/__init__.py b/src/omind/__init__.py index 2d1d449..ad3054c 100644 --- a/src/omind/__init__.py +++ b/src/omind/__init__.py @@ -2,4 +2,4 @@ # Copyright 2026 Aaron K. Clark """omind — OMI/Obsidian memory tooling for AI agents.""" -__version__ = "6.4.0" +__version__ = "6.5.0" diff --git a/src/omind/cli.py b/src/omind/cli.py index 8d6c6c9..5e90bbe 100644 --- a/src/omind/cli.py +++ b/src/omind/cli.py @@ -418,11 +418,11 @@ def build_parser() -> argparse.ArgumentParser: graph_p = sub.add_parser( "graph", help="query the [[wikilink]] knowledge graph: neighbors, path, orphans, " - "dangling links, stats, export", + "dangling links, stats, frontier, export", ) gsub = graph_p.add_subparsers( dest="graph_command", - metavar="{neighbors,path,orphans,dangling,stats,export}", + metavar="{neighbors,path,orphans,dangling,stats,frontier,export}", required=True, ) g_neighbors = gsub.add_parser("neighbors", help="notes within N hops of a note") @@ -442,6 +442,20 @@ def build_parser() -> argparse.ArgumentParser: g_orphans = gsub.add_parser("orphans", help="notes with no inbound or outbound links") g_dangling = gsub.add_parser("dangling", help="wikilinks pointing at no existing note") g_stats = gsub.add_parser("stats", help="counts: notes, links, orphans, dangling") + g_frontier = gsub.add_parser( + "frontier", + help="rank notes by how far they reach out beyond what reaches back (what to " + "consolidate next)", + ) + g_frontier.add_argument( + "--limit", type=int, default=10, help="how many to show (default: 10; 0 = all)" + ) + g_frontier.add_argument( + "--include-generated", + action="store_true", + help="include journals/worklogs, which link outward at everything by construction", + ) + g_frontier.add_argument("--json", action="store_true", help="emit JSON instead of a table") g_export = gsub.add_parser("export", help="dump the whole graph for visualization") g_export.add_argument( "--format", @@ -449,7 +463,7 @@ def build_parser() -> argparse.ArgumentParser: default="json", help="output format (default: json)", ) - for gp in (g_neighbors, g_path, g_orphans, g_dangling, g_stats, g_export): + for gp in (g_neighbors, g_path, g_orphans, g_dangling, g_stats, g_frontier, g_export): _add_vault_args(gp) checkpoint = sub.add_parser( @@ -1125,6 +1139,7 @@ def _run_lint(args: argparse.Namespace) -> int: def _run_graph(args: argparse.Namespace) -> int: import json + from dataclasses import asdict from omind import graph as graphmod @@ -1167,6 +1182,23 @@ def _run_graph(args: argparse.Namespace) -> int: if cmd == "stats": print(json.dumps(graphmod.stats(g), indent=2)) return 0 + if cmd == "frontier": + ranked = graphmod.frontier( + g, limit=args.limit, include_generated=args.include_generated + ) + if args.json: + print(json.dumps([asdict(entry) for entry in ranked], indent=2)) + return 0 + if not ranked: + print("no notes to rank") + return 0 + print("score\tout\tin\tdays\tnote") + for entry in ranked: + print( + f"{entry.score:+.2f}\t{entry.out_degree}\t{entry.in_degree}\t" + f"{entry.days_since_updated:.0f}\t{entry.filename}" + ) + return 0 # cmd == "export" if args.format == "dot": print(graphmod.to_dot(g)) diff --git a/src/omind/graph.py b/src/omind/graph.py index 896b00d..8797c48 100644 --- a/src/omind/graph.py +++ b/src/omind/graph.py @@ -6,7 +6,8 @@ store already answers the inbound question (``backlinks``) and ``lint`` already flags orphans and broken links one note at a time — this module assembles the *whole-graph* view those leave out: forward links, multi-hop neighborhoods, -the shortest link path between two notes, and a JSON/Graphviz-DOT export. +the shortest link path between two notes, a frontier ranking of what to +consolidate next, and a JSON/Graphviz-DOT export. Resolution mirrors :meth:`OmiStore.backlinks` and ``lint``: a link ``[[Target]]`` (its ``|alias`` and ``#heading`` stripped) resolves to a note when its lowercased @@ -43,6 +44,7 @@ class GraphNode: okf_type: str = "" # the note's OKF ``type`` — for grouping/colouring the graph out: set[str] = field(default_factory=set) # filenames this note links to inn: set[str] = field(default_factory=set) # filenames that link to this note + mtime_ns: int = 0 # last write; only :func:`frontier` reads it @dataclass @@ -102,7 +104,12 @@ def _from_index(omi_dir: Path | str) -> Graph | None: if row.title.strip(): id_to_file[row.title.strip().lower()] = filename nodes = { - filename: GraphNode(filename=filename, title=row.title.strip(), okf_type=row.okf_type) + filename: GraphNode( + filename=filename, + title=row.title.strip(), + okf_type=row.okf_type, + mtime_ns=row.mtime_ns, + ) for filename, row in live.items() } dangling: list[tuple[str, str]] = [] @@ -123,7 +130,7 @@ def _from_disk(omi_dir: Path | str) -> Graph: omi = Path(omi_dir) # (filename, title, raw outbound targets) for each live note, plus an index # from every linkable identifier (stem + title, lowercased) to its filename. - parsed: list[tuple[str, str, str, set[str]]] = [] + parsed: list[tuple[str, str, str, set[str], int]] = [] id_to_file: dict[str, str] = {} if omi.is_dir(): for path in sorted(omi.glob("*.md")): @@ -141,17 +148,21 @@ def _from_disk(omi_dir: Path | str) -> Graph: # same rule render_fields uses — so every node carries a non-empty type. okf_type = fields.okf_type.strip() or derive_okf_type(fields.tags) targets = {t for t in (_link_target(m) for m in _WIKILINK_RE.findall(text)) if t} - parsed.append((path.name, title, okf_type, targets)) + try: + mtime_ns = path.stat().st_mtime_ns + except OSError: + mtime_ns = 0 + parsed.append((path.name, title, okf_type, targets, mtime_ns)) id_to_file[path.stem.strip().lower()] = path.name if title: id_to_file[title.lower()] = path.name nodes = { - fn: GraphNode(filename=fn, title=title, okf_type=okf_type) - for fn, title, okf_type, _ in parsed + fn: GraphNode(filename=fn, title=title, okf_type=okf_type, mtime_ns=mtime_ns) + for fn, title, okf_type, _, mtime_ns in parsed } dangling: list[tuple[str, str]] = [] - for src, _title, _type, targets in parsed: + for src, _title, _type, targets, _mtime in parsed: for target in sorted(targets): dest = id_to_file.get(target.lower()) if dest is None: @@ -233,6 +244,88 @@ def dangling_links(graph: Graph) -> list[tuple[str, str]]: return sorted(graph.dangling) +#: Half-life of the recency weight, in days: a note untouched for this long +#: counts half as much as one touched today, and half again after another. +FRONTIER_HALFLIFE_DAYS = 30.0 + +#: OKF types written by machines rather than curated. They link outward at +#: everything by construction, so they would monopolise a frontier ranking. +#: Mirrors the de-prioritisation in ``searchindex._weight_generated``. +_GENERATED_TYPES = frozenset({"journal", "worklog", "checkpoint", "rollup"}) + + +def _is_generated(node: GraphNode) -> bool: + """Whether a node is machine-written (auto-journal, worklog, checkpoint).""" + stem = node.filename[:-3].lower() if node.filename.endswith(".md") else node.filename.lower() + return ( + node.okf_type.strip().lower() in _GENERATED_TYPES + or stem.startswith("session journal") + or stem.startswith("worklog ") + ) + + +@dataclass(frozen=True) +class FrontierEntry: + """One note's frontier score and the components behind it.""" + + filename: str + title: str + score: float + out_degree: int + in_degree: int + days_since_updated: float + + +def frontier( + graph: Graph, + limit: int = 10, + *, + include_generated: bool = False, + now: float | None = None, +) -> list[FrontierEntry]: + """Rank notes by how far they reach out beyond what reaches back at them. + + ``(out_degree - in_degree) * 0.5 ** (days_since_updated / half-life)``. + + A high score means the note points at many things, few things point at it, + and it was touched recently — memory is actively accumulating there and + nothing has pulled it together yet. A low or negative score means a hub: an + absorbed note the rest of the vault already refers to. + + This is the ranking question the other graph ops don't answer. ``orphans`` + finds notes that are *disconnected*; a frontier note is connected but + **unabsorbed**, which is a different and more actionable state. It + complements ``omind consolidate``, which finds merge candidates by + similarity — this finds them by structure, catching "this note has sprawled + outward for weeks" rather than "these two notes say the same thing". + + Machine-written notes (journals, worklogs, checkpoints) are excluded by + default: they link outward at everything by construction and would fill the + whole ranking. Read-only, like every other op here. + """ + import time + + reference = time.time() if now is None else now + entries: list[FrontierEntry] = [] + for node in graph.nodes.values(): + if not include_generated and _is_generated(node): + continue + days = max(0.0, (reference - node.mtime_ns / 1_000_000_000) / 86_400.0) + weight = 0.5 ** (days / FRONTIER_HALFLIFE_DAYS) + entries.append( + FrontierEntry( + filename=node.filename, + title=node.title, + score=(len(node.out) - len(node.inn)) * weight, + out_degree=len(node.out), + in_degree=len(node.inn), + days_since_updated=days, + ) + ) + entries.sort(key=lambda e: (-e.score, e.filename.lower())) + return entries[:limit] if limit > 0 else entries + + def stats(graph: Graph) -> dict[str, int]: """Whole-graph counts: notes, directed links, orphans, and dangling links.""" return { diff --git a/src/omind/searchindex.py b/src/omind/searchindex.py index 1ced40b..45f35ad 100644 --- a/src/omind/searchindex.py +++ b/src/omind/searchindex.py @@ -201,6 +201,9 @@ class _NoteRow: has_title: bool = True tags: list[str] = field(default_factory=list) disabled: bool = False + #: Last write time recorded at ingest. Only the graph's frontier scoring + #: reads it; retrieval ranks on content and the `created` field. + mtime_ns: int = 0 _SCHEMA = """ @@ -1242,10 +1245,11 @@ def _notes(db: sqlite3.Connection) -> list[_NoteRow]: has_title=bool(r["has_title"]), tags=tags.get(str(r["filename"]), []), disabled=bool(r["disabled"]), + mtime_ns=int(r["mtime_ns"]), ) for r in db.execute( "SELECT filename, title, created, okf_type, supersedes, superseded_by," - " has_title, disabled FROM notes" + " has_title, disabled, mtime_ns FROM notes" ) ] diff --git a/src/omind/server.py b/src/omind/server.py index 55bb5b0..4c138ad 100644 --- a/src/omind/server.py +++ b/src/omind/server.py @@ -457,13 +457,32 @@ def _graph_query( return _page(rows, limit, offset) if operation == "stats": return dict(graph.stats(graph_for())) - raise ValueError("graph op must be one of: path, orphans, dangling, stats") + if operation == "frontier": + # Paged like every other list-shaped result (invariant 7): the caller + # asks for a page, the ranking is computed over the whole graph. + ranked = [ + { + "filename": entry.filename, + "title": entry.title, + "score": round(entry.score, 4), + "out_degree": entry.out_degree, + "in_degree": entry.in_degree, + "days_since_updated": round(entry.days_since_updated, 1), + } + for entry in graph.frontier(graph_for(), limit=0) + ] + return _page(ranked, limit, offset) + raise ValueError( + "graph op must be one of: path, orphans, dangling, stats, frontier" + ) @mcp.tool( name="graph", description=( "Graph audit/query selected by op: path (requires source + target), " - "orphans, dangling, or stats. Orphan/dangling results are paged." + "orphans, dangling, stats, or frontier (notes that reach out further " + "than anything reaches back — what to consolidate next). List-shaped " + "results are paged." ), ) def graph_tool( diff --git a/tests/test_graph.py b/tests/test_graph.py index 42c8678..0539dbb 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -5,6 +5,8 @@ from __future__ import annotations import json +import os +import time from pathlib import Path import pytest @@ -103,6 +105,70 @@ def test_stats_counts(store: OmiStore) -> None: assert graph.stats(g) == {"notes": 5, "links": 3, "orphans": 1, "dangling": 1} +def test_frontier_ranks_outward_reaching_notes_above_hubs(store: OmiStore) -> None: + """A note that points at many and is pointed at by few is the frontier.""" + store.create_note(NoteFields(title="Hub", summary="everyone links here")) + store.create_note(NoteFields(title="Edge", summary="see [[Hub]], [[P1]], [[P2]]")) + store.create_note(NoteFields(title="P1", summary="see [[Hub]]")) + store.create_note(NoteFields(title="P2", summary="see [[Hub]]")) + g = graph.build_graph(store.omi_dir) + + ranked = graph.frontier(g) + by_name = {entry.filename: entry for entry in ranked} + # Edge: 3 out, 0 in. Hub: 0 out, 3 in. + assert ranked[0].filename == "Edge.md" + assert by_name["Edge.md"].out_degree == 3 and by_name["Edge.md"].in_degree == 0 + assert by_name["Hub.md"].score < 0 # an absorbed hub, not a frontier + assert ranked[-1].filename == "Hub.md" + + +def test_frontier_decays_with_staleness(store: OmiStore) -> None: + """Two identically-shaped notes rank by when they were last touched.""" + store.create_note(NoteFields(title="Fresh", summary="see [[T1]], [[T2]]")) + store.create_note(NoteFields(title="Stale", summary="see [[T1]], [[T2]]")) + store.create_note(NoteFields(title="T1", summary="target")) + store.create_note(NoteFields(title="T2", summary="target")) + stale = store.omi_dir / "Stale.md" + old = time.time() - graph.FRONTIER_HALFLIFE_DAYS * 86_400 # exactly one half-life + os.utime(stale, (old, old)) + + g = graph._from_disk(store.omi_dir) # mtime, not the index's ingest snapshot + by_name = {entry.filename: entry for entry in graph.frontier(g)} + assert by_name["Fresh.md"].score > by_name["Stale.md"].score + # One half-life back is worth half as much, within a tolerance for the + # seconds that elapsed between writing the note and reading the clock. + assert by_name["Stale.md"].score == pytest.approx( + by_name["Fresh.md"].score / 2, rel=0.01 + ) + + +def test_frontier_excludes_machine_written_notes_by_default(store: OmiStore) -> None: + """Journals link outward at everything and would own the whole ranking.""" + store.create_note(NoteFields(title="Real", summary="see [[T1]]")) + store.create_note(NoteFields(title="T1", summary="target")) + store.create_note( + NoteFields( + title="Worklog 2026-08-02", + summary="see [[Real]], [[T1]]", + okf_type="worklog", + ) + ) + g = graph.build_graph(store.omi_dir) + + assert "Worklog 2026-08-02.md" not in {e.filename for e in graph.frontier(g)} + included = {e.filename for e in graph.frontier(g, include_generated=True)} + assert "Worklog 2026-08-02.md" in included + + +def test_frontier_limit_zero_returns_everything_ranked(store: OmiStore) -> None: + _chain(store) + g = graph.build_graph(store.omi_dir) + assert len(graph.frontier(g, limit=0)) == 5 + assert len(graph.frontier(g, limit=2)) == 2 + scores = [entry.score for entry in graph.frontier(g, limit=0)] + assert scores == sorted(scores, reverse=True) + + def test_to_json_shape(store: OmiStore) -> None: _chain(store) g = graph.build_graph(store.omi_dir) diff --git a/tests/test_server.py b/tests/test_server.py index 613a587..60f72aa 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -192,6 +192,7 @@ def test_every_list_tool_is_bounded(server: MCPServer) -> None: ("graph-neighbors", {"name": "Linker", "limit": 1}), ("graph", {"op": "orphans", "limit": 1}), ("graph", {"op": "dangling", "limit": 1}), + ("graph", {"op": "frontier", "limit": 1}), ): page = call(server, tool, args) assert set(page) >= {"result", "count", "offset", "total", "has_more"}, tool @@ -284,6 +285,12 @@ def test_graph_tools(server: MCPServer) -> None: ] assert call(server, "graph", {"op": "stats"})["notes"] == 4 + ranked = call(server, "graph", {"op": "frontier"})["result"] + # A links B, B links C, and nothing links A: A is the frontier, C the sink. + assert ranked[0]["filename"] == "A.md" + assert ranked[0]["out_degree"] == 1 and ranked[0]["in_degree"] == 0 + assert ranked[-1]["filename"] == "C.md" + def test_unified_graph_validates_operation_and_path_arguments(server: MCPServer) -> None: with pytest.raises(ToolError, match="one of"): diff --git a/uv.lock b/uv.lock index 3503ba2..0415a4b 100644 --- a/uv.lock +++ b/uv.lock @@ -2354,7 +2354,7 @@ wheels = [ [[package]] name = "omind" -version = "6.4.0" +version = "6.5.0" source = { editable = "." } dependencies = [ { name = "cryptography" },