From 4505ef3bcbe847e5fedfa0c61e508f8d85d5ee40 Mon Sep 17 00:00:00 2001 From: nmdra Date: Wed, 12 Aug 2026 13:48:51 +0530 Subject: [PATCH 01/18] feat(parser): extract note references (attachments + external links) --- internal/parser/attachments.go | 156 ++++++++++++++++++++++++++++ internal/parser/attachments_test.go | 119 +++++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 internal/parser/attachments.go create mode 100644 internal/parser/attachments_test.go diff --git a/internal/parser/attachments.go b/internal/parser/attachments.go new file mode 100644 index 0000000..fb2a272 --- /dev/null +++ b/internal/parser/attachments.go @@ -0,0 +1,156 @@ +// Copyright © 2026 nmdra. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package parser + +import ( + "path/filepath" + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "go.abhg.dev/goldmark/wikilink" +) + +// AttachmentKind classifies a resolved reference. +type AttachmentKind string + +const ( + KindImage AttachmentKind = "image" + KindPDF AttachmentKind = "pdf" + KindOther AttachmentKind = "other" + KindExternalLinks AttachmentKind = "external-links" +) + +// AttachmentRef is one attachment reference extracted from a note body. +// Target is cleaned for wiki refs (alias/anchor stripped) and the raw +// destination for markdown refs; resolution happens in the caller. +type AttachmentRef struct { + Target string + Kind AttachmentKind +} + +// ExtractedRefs holds the references collected from a note body. +type ExtractedRefs struct { + Attachments []AttachmentRef + External []string +} + +// ExtractReferences walks a note body's AST and collects direct references: +// local attachments (wiki and markdown syntax) and external http(s) website +// links. URLs and content inside code fences never match. Results are deduped +// (attachments by cleaned target, external by exact URL) in first-occurrence +// document order. +func ExtractReferences(body string) ExtractedRefs { + src := []byte(body) + doc := mdParser.Parser().Parse(text.NewReader(src)) + + var refs ExtractedRefs + seenAttachments := make(map[string]struct{}) + seenExternal := make(map[string]struct{}) + + addAttachment := func(ref AttachmentRef) { + if _, ok := seenAttachments[ref.Target]; ok { + return + } + seenAttachments[ref.Target] = struct{}{} + refs.Attachments = append(refs.Attachments, ref) + } + addExternal := func(url string) { + if _, ok := seenExternal[url]; ok { + return + } + seenExternal[url] = struct{}{} + refs.External = append(refs.External, url) + } + + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch nTyped := n.(type) { + case *wikilink.Node: + target := string(nTyped.Target) + if target == "" { + break + } + if isHTTPScheme(target) { + addExternal(target) + break + } + cleaned := cleanWikiTarget(target) + if kind, ok := classifyAttachmentKind(cleaned); ok { + addAttachment(AttachmentRef{Target: cleaned, Kind: kind}) + } + case *ast.Link: + handleMarkdownReference(string(nTyped.Destination), addAttachment, addExternal) + case *ast.Image: + handleMarkdownReference(string(nTyped.Destination), addAttachment, addExternal) + case *ast.AutoLink: + if nTyped.AutoLinkType != ast.AutoLinkURL { + break + } + // URL() assembles the scheme for <...> autolinks; linkify nodes + // already carry it in the value. Either way only http(s) counts. + if url := string(nTyped.URL(src)); isHTTPScheme(url) { + addExternal(url) + } + } + return ast.WalkContinue, nil + }) + + return refs +} + +// handleMarkdownReference classifies a markdown link/image destination as +// either an external http(s) URL or a local attachment. +func handleMarkdownReference(destination string, addAttachment func(AttachmentRef), addExternal func(string)) { + if isHTTPScheme(destination) { + addExternal(destination) + return + } + if kind, ok := classifyAttachmentKind(destination); ok { + addAttachment(AttachmentRef{Target: destination, Kind: kind}) + } +} + +// isHTTPScheme reports whether s starts with an http:// or https:// scheme. +func isHTTPScheme(s string) bool { + lower := strings.ToLower(s) + return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") +} + +// cleanWikiTarget strips the alias (|...) and heading fragment (#...) from a +// wiki link target so the file target itself is classified and resolved. +func cleanWikiTarget(target string) string { + if idx := strings.Index(target, "|"); idx != -1 { + target = target[:idx] + } + if idx := strings.Index(target, "#"); idx != -1 { + target = target[:idx] + } + return strings.TrimSpace(target) +} + +// classifyAttachmentKind returns the kind of an attachment reference for a +// cleaned target (extensions only), or false when the target is not a known +// attachment (e.g. other notes). Unlike IsAttachmentLink, PDFs classify here +// because refs list them as file references even though ingestion treats +// them as notes. +func classifyAttachmentKind(target string) (AttachmentKind, bool) { + if idx := strings.LastIndex(target, "#"); idx != -1 { + target = target[:idx] + } + ext := strings.ToLower(filepath.Ext(target)) + if _, ok := imageExts[ext]; ok { + return KindImage, true + } + if ext == ".pdf" { + return KindPDF, true + } + if _, ok := attachmentExts[ext]; ok { + return KindOther, true + } + return "", false +} diff --git a/internal/parser/attachments_test.go b/internal/parser/attachments_test.go new file mode 100644 index 0000000..5340456 --- /dev/null +++ b/internal/parser/attachments_test.go @@ -0,0 +1,119 @@ +// Copyright © 2026 nmdra. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package parser + +import ( + "reflect" + "testing" +) + +func TestExtractReferences_WikiAttachments(t *testing.T) { + tests := []struct { + name string + body string + want []AttachmentRef + }{ + {name: "image embed with size", body: "![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "pdf plain link", body: "[[doc.pdf]]", want: []AttachmentRef{{Target: "doc.pdf", Kind: KindPDF}}}, + {name: "image embed with alias", body: "![[img.png|alt text]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "image with heading anchor", body: "[[img.png#anchor]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "subfolder image embed", body: "![[sub/img.png]]", want: []AttachmentRef{{Target: "sub/img.png", Kind: KindImage}}}, + {name: "relative dot prefix", body: "[[./local.png]]", want: []AttachmentRef{{Target: "./local.png", Kind: KindImage}}}, + {name: "archive attachment", body: "[[bundle.zip]]", want: []AttachmentRef{{Target: "bundle.zip", Kind: KindOther}}}, + {name: "canvas attachment", body: "[[diagram.canvas]]", want: []AttachmentRef{{Target: "diagram.canvas", Kind: KindOther}}}, + {name: "unknown extension is not an attachment", body: "[[archive.xyz]]", want: nil}, + {name: "dotted note name is not an attachment", body: "[[Note 1.2.3]]", want: nil}, + {name: "plain note link is not an attachment", body: "[[Other Note]]", want: nil}, + {name: "uppercase extension is case-insensitive", body: "![[PHOTO.PNG]]", want: []AttachmentRef{{Target: "PHOTO.PNG", Kind: KindImage}}}, + {name: "duplicate embeds dedupe", body: "![[img.png]]\n\n![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "code fence contents ignored", body: "```\n![[x.png]]\n[[secret.pdf]]\n```", want: nil}, + {name: "inline code ignored", body: "`![[x.png]]`", want: nil}, + {name: "empty target ignored", body: "[[#heading]]", want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExtractReferences(tt.body) + if !reflect.DeepEqual(got.Attachments, tt.want) { + t.Errorf("ExtractReferences(%q).Attachments = %v, want %v", tt.body, got.Attachments, tt.want) + } + if len(got.External) != 0 { + t.Errorf("ExtractReferences(%q).External = %v, want none", tt.body, got.External) + } + }) + } +} + +func TestExtractReferences_MarkdownAttachments(t *testing.T) { + tests := []struct { + name string + body string + want []AttachmentRef + }{ + {name: "image", body: "![alt](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "pdf in subfolder", body: "[doc](sub/file.pdf)", want: []AttachmentRef{{Target: "sub/file.pdf", Kind: KindPDF}}}, + {name: "percent-encoded destination kept raw", body: "![alt](Router%20Modes.webp)", want: []AttachmentRef{{Target: "Router%20Modes.webp", Kind: KindImage}}}, + {name: "parent traversal kept raw", body: "[x](../up.pdf)", want: []AttachmentRef{{Target: "../up.pdf", Kind: KindPDF}}}, + {name: "pdf with page fragment", body: "[x](STP.pdf#page=5)", want: []AttachmentRef{{Target: "STP.pdf#page=5", Kind: KindPDF}}}, + {name: "external link is not an attachment", body: "[text](https://example.com)", want: nil}, + {name: "relative note link without extension", body: "[rel](../other-note)", want: nil}, + {name: "anchor-only link", body: "[x](#anchor)", want: nil}, + {name: "code fence contents ignored", body: "```\n![x](img.png)\n```", want: nil}, + {name: "duplicate destinations dedupe", body: "![a](img.png) and ![b](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExtractReferences(tt.body) + if !reflect.DeepEqual(got.Attachments, tt.want) { + t.Errorf("ExtractReferences(%q).Attachments = %v, want %v", tt.body, got.Attachments, tt.want) + } + }) + } +} + +func TestExtractReferences_External(t *testing.T) { + tests := []struct { + name string + body string + want []string + }{ + {name: "markdown link", body: "[text](https://example.com/a)", want: []string{"https://example.com/a"}}, + {name: "markdown image embed", body: "![alt](https://example.com/i.png)", want: []string{"https://example.com/i.png"}}, + {name: "bare url", body: "see https://example.com here", want: []string{"https://example.com"}}, + {name: "angle url", body: "", want: []string{"https://example.com"}}, + {name: "bare www url gains http protocol", body: "visit www.example.com", want: []string{"http://www.example.com"}}, + {name: "wikilink to external url", body: "[[https://example.com]]", want: []string{"https://example.com"}}, + {name: "wikilink to external image url", body: "[[https://example.com/img.png]]", want: []string{"https://example.com/img.png"}}, + {name: "multiple urls keep first occurrence order", body: "[b](https://b.org)\n\n[a](https://a.org) and https://b.org", want: []string{"https://b.org", "https://a.org"}}, + {name: "code fence contents ignored", body: "```\nhttps://example.com\n```", want: nil}, + {name: "excluded schemes", body: "[mail](mailto:a@b.c)\n\n[ftp](ftp://x.y/z)\n\nemail me at a@b.c", want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExtractReferences(tt.body) + if !reflect.DeepEqual(got.External, tt.want) { + t.Errorf("ExtractReferences(%q).External = %v, want %v", tt.body, got.External, tt.want) + } + if len(got.Attachments) != 0 { + t.Errorf("ExtractReferences(%q).Attachments = %v, want none", tt.body, got.Attachments) + } + }) + } +} + +func TestExtractReferences_Mixed(t *testing.T) { + body := "![[cover.png]] and [doc](manual.pdf) and https://example.com and [[https://links.example.com]]" + got := ExtractReferences(body) + wantAttach := []AttachmentRef{ + {Target: "cover.png", Kind: KindImage}, + {Target: "manual.pdf", Kind: KindPDF}, + } + if !reflect.DeepEqual(got.Attachments, wantAttach) { + t.Errorf("Attachments = %v, want %v", got.Attachments, wantAttach) + } + wantExt := []string{"https://example.com", "https://links.example.com"} + if !reflect.DeepEqual(got.External, wantExt) { + t.Errorf("External = %v, want %v", got.External, wantExt) + } +} From c0e9f2839f4378a13cfe5ddc26243cd940513a9c Mon Sep 17 00:00:00 2001 From: nmdra Date: Wed, 12 Aug 2026 13:55:34 +0530 Subject: [PATCH 02/18] feat(ingest): expose configured Obsidian attachment folder - add LoadAttachmentFolderPath reading .obsidian/app.json - share the app.json read with LoadExcludedPaths via readObsidianAppConfig --- internal/ingest/ignore.go | 35 +++++++++++++++++++++------ internal/ingest/ignore_test.go | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/internal/ingest/ignore.go b/internal/ingest/ignore.go index d2906d6..c37f6d7 100644 --- a/internal/ingest/ignore.go +++ b/internal/ingest/ignore.go @@ -18,23 +18,44 @@ type ObsidianAppConfig struct { // LoadExcludedPaths reads the userIgnoreFilters and attachmentFolderPath from .obsidian/app.json. // Returns nil if the file is absent or unreadable. func LoadExcludedPaths(vaultPath string) []string { + config, err := readObsidianAppConfig(vaultPath) + if err != nil { + return nil + } + filters := config.UserIgnoreFilters + if config.AttachmentFolderPath != "" { + filters = append(filters, config.AttachmentFolderPath) + } + return filters +} + +// LoadAttachmentFolderPath returns the Obsidian attachment folder configured +// in .obsidian/app.json, or "" when the file is absent, unreadable, or the +// setting is unset. +func LoadAttachmentFolderPath(vaultPath string) string { + config, err := readObsidianAppConfig(vaultPath) + if err != nil { + return "" + } + return config.AttachmentFolderPath +} + +// readObsidianAppConfig reads and parses .obsidian/app.json. The caller +// decides what to return when the file is absent or malformed. +func readObsidianAppConfig(vaultPath string) (*ObsidianAppConfig, error) { data, err := os.ReadFile(filepath.Join(vaultPath, ".obsidian", "app.json")) if err != nil { if !errors.Is(err, os.ErrNotExist) { slog.Warn("failed to read .obsidian/app.json", "vault_path", vaultPath, "err", err) } - return nil + return nil, err } var config ObsidianAppConfig if err := json.Unmarshal(data, &config); err != nil { slog.Warn("failed to parse .obsidian/app.json", "vault_path", vaultPath, "err", err) - return nil + return nil, err } - filters := config.UserIgnoreFilters - if config.AttachmentFolderPath != "" { - filters = append(filters, config.AttachmentFolderPath) - } - return filters + return &config, nil } // IsExcluded checks if the relative path matches any ignore filters. diff --git a/internal/ingest/ignore_test.go b/internal/ingest/ignore_test.go index 8719ab1..e31f486 100644 --- a/internal/ingest/ignore_test.go +++ b/internal/ingest/ignore_test.go @@ -45,6 +45,50 @@ func TestLoadExcludedPaths_MissingFile(t *testing.T) { } } +func TestLoadAttachmentFolderPath(t *testing.T) { + vaultDir := t.TempDir() + obsidianDir := filepath.Join(vaultDir, ".obsidian") + if err := os.MkdirAll(obsidianDir, 0755); err != nil { + t.Fatalf("failed to create .obsidian dir: %v", err) + } + + appJSONPath := filepath.Join(obsidianDir, "app.json") + content := []byte(`{ + "userIgnoreFilters": ["Archive"], + "attachmentFolderPath": "99.Storage-Shed/Attachments" + }`) + if err := os.WriteFile(appJSONPath, content, 0644); err != nil { + t.Fatalf("failed to write app.json: %v", err) + } + + if got := ingest.LoadAttachmentFolderPath(vaultDir); got != "99.Storage-Shed/Attachments" { + t.Errorf("LoadAttachmentFolderPath = %q, want %q", got, "99.Storage-Shed/Attachments") + } +} + +func TestLoadAttachmentFolderPath_MissingFile(t *testing.T) { + vaultDir := t.TempDir() + if got := ingest.LoadAttachmentFolderPath(vaultDir); got != "" { + t.Errorf("expected empty attachment folder for missing app.json, got %q", got) + } +} + +func TestLoadAttachmentFolderPath_EmptyValue(t *testing.T) { + vaultDir := t.TempDir() + obsidianDir := filepath.Join(vaultDir, ".obsidian") + if err := os.MkdirAll(obsidianDir, 0755); err != nil { + t.Fatalf("failed to create .obsidian dir: %v", err) + } + content := []byte(`{"userIgnoreFilters": ["Archive"]}`) + if err := os.WriteFile(filepath.Join(obsidianDir, "app.json"), content, 0644); err != nil { + t.Fatalf("failed to write app.json: %v", err) + } + + if got := ingest.LoadAttachmentFolderPath(vaultDir); got != "" { + t.Errorf("expected empty attachment folder when unset, got %q", got) + } +} + func TestIsExcluded(t *testing.T) { filters := []string{ "Archive", From e222dc5406f85dd2d64627cbc8350c5f4f3974a3 Mon Sep 17 00:00:00 2001 From: nmdra Date: Wed, 12 Aug 2026 14:07:57 +0530 Subject: [PATCH 03/18] feat(parser): track reference syntax source for resolution --- internal/parser/attachments.go | 14 +++++++++-- internal/parser/attachments_test.go | 36 ++++++++++++++--------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/internal/parser/attachments.go b/internal/parser/attachments.go index fb2a272..1e6904b 100644 --- a/internal/parser/attachments.go +++ b/internal/parser/attachments.go @@ -23,12 +23,22 @@ const ( KindExternalLinks AttachmentKind = "external-links" ) +// AttachmentSource records which markdown syntax produced a reference, so +// resolution can apply Obsidian semantics (wiki vs note-folder-relative). +type AttachmentSource string + +const ( + SrcWiki AttachmentSource = "wiki" + SrcMarkdown AttachmentSource = "markdown" +) + // AttachmentRef is one attachment reference extracted from a note body. // Target is cleaned for wiki refs (alias/anchor stripped) and the raw // destination for markdown refs; resolution happens in the caller. type AttachmentRef struct { Target string Kind AttachmentKind + Source AttachmentSource } // ExtractedRefs holds the references collected from a note body. @@ -81,7 +91,7 @@ func ExtractReferences(body string) ExtractedRefs { } cleaned := cleanWikiTarget(target) if kind, ok := classifyAttachmentKind(cleaned); ok { - addAttachment(AttachmentRef{Target: cleaned, Kind: kind}) + addAttachment(AttachmentRef{Target: cleaned, Kind: kind, Source: SrcWiki}) } case *ast.Link: handleMarkdownReference(string(nTyped.Destination), addAttachment, addExternal) @@ -111,7 +121,7 @@ func handleMarkdownReference(destination string, addAttachment func(AttachmentRe return } if kind, ok := classifyAttachmentKind(destination); ok { - addAttachment(AttachmentRef{Target: destination, Kind: kind}) + addAttachment(AttachmentRef{Target: destination, Kind: kind, Source: SrcMarkdown}) } } diff --git a/internal/parser/attachments_test.go b/internal/parser/attachments_test.go index 5340456..1b2f8d5 100644 --- a/internal/parser/attachments_test.go +++ b/internal/parser/attachments_test.go @@ -15,19 +15,19 @@ func TestExtractReferences_WikiAttachments(t *testing.T) { body string want []AttachmentRef }{ - {name: "image embed with size", body: "![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, - {name: "pdf plain link", body: "[[doc.pdf]]", want: []AttachmentRef{{Target: "doc.pdf", Kind: KindPDF}}}, - {name: "image embed with alias", body: "![[img.png|alt text]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, - {name: "image with heading anchor", body: "[[img.png#anchor]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, - {name: "subfolder image embed", body: "![[sub/img.png]]", want: []AttachmentRef{{Target: "sub/img.png", Kind: KindImage}}}, - {name: "relative dot prefix", body: "[[./local.png]]", want: []AttachmentRef{{Target: "./local.png", Kind: KindImage}}}, - {name: "archive attachment", body: "[[bundle.zip]]", want: []AttachmentRef{{Target: "bundle.zip", Kind: KindOther}}}, - {name: "canvas attachment", body: "[[diagram.canvas]]", want: []AttachmentRef{{Target: "diagram.canvas", Kind: KindOther}}}, + {name: "image embed with size", body: "![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "pdf plain link", body: "[[doc.pdf]]", want: []AttachmentRef{{Target: "doc.pdf", Kind: KindPDF, Source: SrcWiki}}}, + {name: "image embed with alias", body: "![[img.png|alt text]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "image with heading anchor", body: "[[img.png#anchor]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "subfolder image embed", body: "![[sub/img.png]]", want: []AttachmentRef{{Target: "sub/img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "relative dot prefix", body: "[[./local.png]]", want: []AttachmentRef{{Target: "./local.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "archive attachment", body: "[[bundle.zip]]", want: []AttachmentRef{{Target: "bundle.zip", Kind: KindOther, Source: SrcWiki}}}, + {name: "canvas attachment", body: "[[diagram.canvas]]", want: []AttachmentRef{{Target: "diagram.canvas", Kind: KindOther, Source: SrcWiki}}}, {name: "unknown extension is not an attachment", body: "[[archive.xyz]]", want: nil}, {name: "dotted note name is not an attachment", body: "[[Note 1.2.3]]", want: nil}, {name: "plain note link is not an attachment", body: "[[Other Note]]", want: nil}, - {name: "uppercase extension is case-insensitive", body: "![[PHOTO.PNG]]", want: []AttachmentRef{{Target: "PHOTO.PNG", Kind: KindImage}}}, - {name: "duplicate embeds dedupe", body: "![[img.png]]\n\n![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "uppercase extension is case-insensitive", body: "![[PHOTO.PNG]]", want: []AttachmentRef{{Target: "PHOTO.PNG", Kind: KindImage, Source: SrcWiki}}}, + {name: "duplicate embeds dedupe", body: "![[img.png]]\n\n![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, {name: "code fence contents ignored", body: "```\n![[x.png]]\n[[secret.pdf]]\n```", want: nil}, {name: "inline code ignored", body: "`![[x.png]]`", want: nil}, {name: "empty target ignored", body: "[[#heading]]", want: nil}, @@ -51,16 +51,16 @@ func TestExtractReferences_MarkdownAttachments(t *testing.T) { body string want []AttachmentRef }{ - {name: "image", body: "![alt](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, - {name: "pdf in subfolder", body: "[doc](sub/file.pdf)", want: []AttachmentRef{{Target: "sub/file.pdf", Kind: KindPDF}}}, - {name: "percent-encoded destination kept raw", body: "![alt](Router%20Modes.webp)", want: []AttachmentRef{{Target: "Router%20Modes.webp", Kind: KindImage}}}, - {name: "parent traversal kept raw", body: "[x](../up.pdf)", want: []AttachmentRef{{Target: "../up.pdf", Kind: KindPDF}}}, - {name: "pdf with page fragment", body: "[x](STP.pdf#page=5)", want: []AttachmentRef{{Target: "STP.pdf#page=5", Kind: KindPDF}}}, + {name: "image", body: "![alt](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcMarkdown}}}, + {name: "pdf in subfolder", body: "[doc](sub/file.pdf)", want: []AttachmentRef{{Target: "sub/file.pdf", Kind: KindPDF, Source: SrcMarkdown}}}, + {name: "percent-encoded destination kept raw", body: "![alt](Router%20Modes.webp)", want: []AttachmentRef{{Target: "Router%20Modes.webp", Kind: KindImage, Source: SrcMarkdown}}}, + {name: "parent traversal kept raw", body: "[x](../up.pdf)", want: []AttachmentRef{{Target: "../up.pdf", Kind: KindPDF, Source: SrcMarkdown}}}, + {name: "pdf with page fragment", body: "[x](STP.pdf#page=5)", want: []AttachmentRef{{Target: "STP.pdf#page=5", Kind: KindPDF, Source: SrcMarkdown}}}, {name: "external link is not an attachment", body: "[text](https://example.com)", want: nil}, {name: "relative note link without extension", body: "[rel](../other-note)", want: nil}, {name: "anchor-only link", body: "[x](#anchor)", want: nil}, {name: "code fence contents ignored", body: "```\n![x](img.png)\n```", want: nil}, - {name: "duplicate destinations dedupe", body: "![a](img.png) and ![b](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage}}}, + {name: "duplicate destinations dedupe", body: "![a](img.png) and ![b](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcMarkdown}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -106,8 +106,8 @@ func TestExtractReferences_Mixed(t *testing.T) { body := "![[cover.png]] and [doc](manual.pdf) and https://example.com and [[https://links.example.com]]" got := ExtractReferences(body) wantAttach := []AttachmentRef{ - {Target: "cover.png", Kind: KindImage}, - {Target: "manual.pdf", Kind: KindPDF}, + {Target: "cover.png", Kind: KindImage, Source: SrcWiki}, + {Target: "manual.pdf", Kind: KindPDF, Source: SrcMarkdown}, } if !reflect.DeepEqual(got.Attachments, wantAttach) { t.Errorf("Attachments = %v, want %v", got.Attachments, wantAttach) From d7ee594b1d751a140b18361da594e98ddf0c1a7c Mon Sep 17 00:00:00 2001 From: nmdra Date: Wed, 12 Aug 2026 14:15:23 +0530 Subject: [PATCH 04/18] feat(cmd): list a note's attachments and external links - add refs command resolving wiki and markdown attachment references - resolve wiki refs via Obsidian search order (note folder, vault root, attachment folder) and markdown refs note-folder-relative - hide broken links by default, surface them with --include-missing - filter by kind with --images/--pdf/--other/--external-links (OR) - render text/JSON/TSV output with jsonpath support --- cmd/cli.go | 3 + cmd/refs.go | 300 ++++++++++++++++++++++++++ cmd/refs_test.go | 464 ++++++++++++++++++++++++++++++++++++++++ cmd/testhelpers_test.go | 26 ++- 4 files changed, 782 insertions(+), 11 deletions(-) create mode 100644 cmd/refs.go create mode 100644 cmd/refs_test.go diff --git a/cmd/cli.go b/cmd/cli.go index accfd1c..a4f1acb 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -40,6 +40,7 @@ const ( groupTags = "tags" groupReset = "reset" groupGet = "get" + groupRefs = "refs" ) // helpGroups returns the titled flag groups shown in --help output. Flags in @@ -57,6 +58,7 @@ func helpGroups() []kong.Group { {Key: groupTags, Title: "Tags Flags"}, {Key: groupReset, Title: "Reset Flags"}, {Key: groupGet, Title: "Get Flags"}, + {Key: groupRefs, Title: "Refs Flags"}, } } @@ -98,6 +100,7 @@ type CLI struct { Boosted BoostedCmd `cmd:"" help:"Semantic search boosted by wikilink graph proximity"` Stats StatsCmd `cmd:"" help:"Show collection statistics"` Get GetCmd `cmd:"" help:"Retrieve the full text of an indexed note"` + Refs RefsCmd `cmd:"" help:"List a note's attachments and external links"` Reset ResetCmd `cmd:"" help:"Delete all indexed data and reset the database"` Doctor DoctorCmd `cmd:"" help:"Run diagnostics to check system dependencies and configurations"` DoctorProbe DoctorProbeCmd `cmd:"" hidden:"" help:"internal: verify the database can be opened (used by doctor)"` diff --git a/cmd/refs.go b/cmd/refs.go new file mode 100644 index 0000000..11ed74c --- /dev/null +++ b/cmd/refs.go @@ -0,0 +1,300 @@ +/* +Copyright © 2026 nmdra + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/nmdra/notebrain-cli/v2/internal/ingest" + "github.com/nmdra/notebrain-cli/v2/internal/parser" +) + +// refs kinds. They mirror parser.AttachmentKind but are plain strings so the +// output layer stays independent of the parser package. +const ( + kindImage = "image" + kindPDF = "pdf" + kindOther = "other" + kindExternal = "external-links" +) + +type RefsCmd struct { + Note string `arg:"" help:"note slug, title, or file path (auto-resolved)" completion-predictor:"note-slug"` + Images bool `group:"refs" help:"include image attachments" default:"false"` + PDF bool `group:"refs" help:"include PDF attachments" default:"false"` + Other bool `group:"refs" help:"include other attachments (video, audio, archives, office docs)" default:"false"` + ExternalLinks bool `group:"refs" name:"external-links" help:"include external website links (URLs)" default:"false"` + IncludeMissing bool `group:"refs" name:"include-missing" help:"include references whose file is missing from the vault" default:"false"` +} + +// refEntry is one resolved reference row. Path is absolute for attachments and +// the URL for external links; external rows never carry relative_path. +type refEntry struct { + Path string `json:"path"` + RelativePath string `json:"relative_path,omitempty"` + Kind string `json:"kind"` + Missing bool `json:"missing"` +} + +// refsEnvelope is the machine-readable shape of "refs" output. +type refsEnvelope struct { + Command string `json:"command"` + NoteSlug string `json:"note_slug"` + Title string `json:"title"` + Total int `json:"total"` + Refs []refEntry `json:"refs"` +} + +func (c *RefsCmd) Run(globals *Globals) error { + ctx := globals.Ctx + vaultPath := globals.VaultPath + if vaultPath == "" { + return &UsageError{Err: fmt.Errorf("--vault-path flag or config file setting must be specified — run 'notebrain init' to create a config")} + } + if strings.TrimSpace(c.Note) == "" { + return &UsageError{Err: fmt.Errorf("%s requires a note slug, title, or file path", groupRefs)} + } + + st, err := openStore(ctx, globals) + if err != nil { + return err + } + defer func() { _ = st.Close() }() + + meta, err := st.GetNoteMeta(ctx, c.Note) + if err != nil { + return err + } + if strings.HasSuffix(strings.ToLower(meta.FilePath), ".pdf") { + return fmt.Errorf("note %q is a PDF; refs are listed for markdown notes", c.Note) + } + + absPath := filepath.Join(vaultPath, meta.FilePath) + body, err := os.ReadFile(absPath) + if err != nil { + return fmt.Errorf("read note file %q: %w (is --vault-path pointing at the right vault?)", absPath, err) + } + + entries := c.resolveRefs(vaultPath, meta.FilePath, parser.ExtractReferences(string(body))) + if !c.IncludeMissing { + entries = filterExistingRefs(entries) + } + entries = filterRefKinds(entries, c) + + env := refsEnvelope{ + Command: groupRefs, + NoteSlug: meta.NoteSlug, + Title: meta.Title, + Total: len(entries), + Refs: entries, + } + return printRefsFormatted(env, globals) +} + +// resolveRefs turns extracted references into resolved entries, deduped by +// resolved absolute path (or exact URL) in first-occurrence order. +func (c *RefsCmd) resolveRefs(vaultPath, noteFilePath string, extracted parser.ExtractedRefs) []refEntry { + noteDir := filepath.Dir(filepath.Join(vaultPath, filepath.FromSlash(noteFilePath))) + attachmentFolder := ingest.LoadAttachmentFolderPath(vaultPath) + + var entries []refEntry + seen := make(map[string]struct{}) + add := func(e refEntry) { + if _, ok := seen[e.Path]; ok { + return + } + seen[e.Path] = struct{}{} + entries = append(entries, e) + } + + for _, ref := range extracted.Attachments { + add(c.resolveAttachment(vaultPath, noteDir, attachmentFolder, ref)) + } + for _, link := range extracted.External { + add(refEntry{Path: link, Kind: kindExternal}) + } + return entries +} + +// resolveAttachment finds the on-disk path of one attachment reference. Wiki +// refs follow Obsidian's search order (note folder, vault root, attachment +// folder); markdown refs resolve note-folder-relative with percent-decoding. +// A reference that escapes the vault or matches no existing file resolves to +// its first candidate marked missing. +func (c *RefsCmd) resolveAttachment(vaultPath, noteDir, attachmentFolder string, ref parser.AttachmentRef) refEntry { + var candidates []string + switch ref.Source { + case parser.SrcMarkdown: + candidates = []string{resolveMarkdownDestination(noteDir, ref.Target)} + default: + candidates = wikiCandidates(vaultPath, noteDir, attachmentFolder, ref.Target) + } + first := candidates[0] + for _, cand := range candidates { + if !insideVault(vaultPath, cand) { + continue + } + if _, err := os.Stat(cand); err == nil { + return refEntry{Path: cand, RelativePath: vaultRelativePath(vaultPath, cand), Kind: string(ref.Kind)} + } + } + return refEntry{Path: first, RelativePath: vaultRelativePath(vaultPath, first), Kind: string(ref.Kind), Missing: true} +} + +// wikiCandidates returns candidate paths for a wiki target in Obsidian +// resolution order. Targets containing "/" start at the vault root; "./" +// resolves relative to the note's folder; bare names search the note folder, +// then the vault root, then the configured attachment folder. +func wikiCandidates(vaultPath, noteDir, attachmentFolder, target string) []string { + switch { + case strings.HasPrefix(target, "./"): + return []string{filepath.Join(noteDir, filepath.FromSlash(strings.TrimPrefix(target, "./")))} + case strings.Contains(target, "/"): + return []string{filepath.Join(vaultPath, filepath.FromSlash(target))} + } + candidates := []string{filepath.Join(noteDir, filepath.FromSlash(target))} + if vaultPath != noteDir { + candidates = append(candidates, filepath.Join(vaultPath, filepath.FromSlash(target))) + } + if attachmentFolder != "" { + candidates = append(candidates, filepath.Join(vaultPath, filepath.FromSlash(attachmentFolder), filepath.FromSlash(target))) + } + return candidates +} + +// resolveMarkdownDestination decodes a markdown link destination (fragment +// stripped, percent-encoding unescaped) and joins it to the note's folder. +func resolveMarkdownDestination(noteDir, target string) string { + decoded := target + if before, _, ok := strings.Cut(target, "#"); ok { + decoded = before + } + if unescaped, err := url.PathUnescape(decoded); err == nil { + decoded = unescaped + } + return filepath.Join(noteDir, filepath.FromSlash(decoded)) +} + +// insideVault reports whether abs stays within the vault, rejecting `..` +// traversal escapes via filepath.Rel. +func insideVault(vaultPath, abs string) bool { + rel, err := filepath.Rel(vaultPath, abs) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// vaultRelativePath renders abs as a slash-separated vault-relative path. +func vaultRelativePath(vaultPath, abs string) string { + rel, err := filepath.Rel(vaultPath, abs) + if err != nil { + return "" + } + return filepath.ToSlash(rel) +} + +// filterExistingRefs drops missing rows (broken links are hidden by default). +func filterExistingRefs(entries []refEntry) []refEntry { + kept := make([]refEntry, 0, len(entries)) + for _, e := range entries { + if !e.Missing { + kept = append(kept, e) + } + } + return kept +} + +// filterRefKinds keeps rows matching any selected kind flag; no flags select +// every kind. +func filterRefKinds(entries []refEntry, c *RefsCmd) []refEntry { + if !c.Images && !c.PDF && !c.Other && !c.ExternalLinks { + return entries + } + kept := make([]refEntry, 0, len(entries)) + for _, e := range entries { + keep := false + switch e.Kind { + case kindImage: + keep = c.Images + case kindPDF: + keep = c.PDF + case kindOther: + keep = c.Other + case kindExternal: + keep = c.ExternalLinks + } + if keep { + kept = append(kept, e) + } + } + return kept +} + +// printRefsFormatted renders a refs envelope to stdout based on the requested +// format. JSONPath extraction applies to the envelope when requested. +func printRefsFormatted(env refsEnvelope, globals *Globals) error { + return printRefsFormattedToWriter(os.Stdout, env, globals) +} + +func printRefsFormattedToWriter(w io.Writer, env refsEnvelope, globals *Globals) error { + if globals.JSONPath != "" { + return printJSONPathResultToWriter(w, env, globals.JSONPath) + } + + switch globals.Format { + case formatJSON: + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(env) + case formatTSV: + _, _ = fmt.Fprintln(w, "path\tkind\tmissing\trelative_path") + for _, r := range env.Refs { + missing := strconv.FormatBool(r.Missing) + if r.Kind == kindExternal { + missing = "" + } + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", tsvEscape(r.Path), r.Kind, missing, tsvEscape(r.RelativePath)) + } + return nil + default: // "text" + if len(env.Refs) == 0 { + _, _ = fmt.Fprintln(w, "No references found") + return nil + } + for _, r := range env.Refs { + marker := "" + if r.Missing { + marker = " (missing)" + } + _, _ = fmt.Fprintf(w, "[%s] %s%s\n", r.Kind, r.Path, marker) + } + return nil + } +} diff --git a/cmd/refs_test.go b/cmd/refs_test.go new file mode 100644 index 0000000..baed4f4 --- /dev/null +++ b/cmd/refs_test.go @@ -0,0 +1,464 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nmdra/notebrain-cli/v2/internal/store" +) + +// writeRefsTestVault creates a temp vault with the standard fixture note and +// returns the vault root. The note is Notes/router.md: +// +// ![[cover.png]] [[assets/arch.png]] [[./local.png]] [modes](Router%20Modes.webp) +// [[att.pdf]] [[a.png]] [a](a.png) [broken](broken.png) [ext](https://example.com/docs) +// [[https://links.example.com]] +// +// It advertises the Obsidian attachment folder 99.Storage-Shed/Attachments. +func writeRefsTestVault(t *testing.T) string { + t.Helper() + vaultDir := t.TempDir() + files := map[string]string{ + ".obsidian/app.json": `{"attachmentFolderPath": "99.Storage-Shed/Attachments"}`, + "Notes/router.md": "![[cover.png]] [[assets/arch.png]] [[./local.png]] [modes](Router%20Modes.webp)\n[[att.pdf]] [[a.png]] [a](a.png) [broken](broken.png) [ext](https://example.com/docs)\n[[https://links.example.com]]", + "Notes/cover.png": "png", + "Notes/Router Modes.webp": "webp", + "Notes/local.png": "png", + "Notes/a.png": "png", + "assets/arch.png": "png", + "99.Storage-Shed/Attachments/att.pdf": "pdf", + } + for path, content := range files { + full := filepath.Join(vaultDir, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return vaultDir +} + +func refsTestGlobals(vaultDir string) *Globals { + return &Globals{Ctx: context.Background(), VaultPath: vaultDir} +} + +func TestRefsText(t *testing.T) { + vaultDir := writeRefsTestVault(t) + fs := &fakeStore{noteMeta: &store.NoteContent{ + NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md", + }} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "router"}).Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + + wantLines := []string{ + "[image] " + filepath.Join(vaultDir, "Notes", "cover.png"), + "[image] " + filepath.Join(vaultDir, "assets", "arch.png"), + "[image] " + filepath.Join(vaultDir, "Notes", "local.png"), + "[image] " + filepath.Join(vaultDir, "Notes", "Router Modes.webp"), + "[pdf] " + filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf"), + "[image] " + filepath.Join(vaultDir, "Notes", "a.png"), + "[external-links] https://example.com/docs", + "[external-links] https://links.example.com", + } + for _, want := range wantLines { + if !strings.Contains(out, want) { + t.Errorf("text output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "broken.png") { + t.Errorf("missing reference shown without --include-missing:\n%s", out) + } + idxs := make([]int, len(wantLines)) + for i, want := range wantLines { + idxs[i] = strings.Index(out, want) + } + for i := 1; i < len(idxs); i++ { + if idxs[i] < idxs[i-1] { + t.Errorf("output not in first-occurrence order: %q before %q", wantLines[i-1], wantLines[i]) + } + } +} + +func TestRefsFilters(t *testing.T) { + vaultDir := writeRefsTestVault(t) + fs := &fakeStore{noteMeta: &store.NoteContent{ + NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md", + }} + withFakeStore(t, fs) + + tests := []struct { + name string + cmd RefsCmd + include []string + exclude []string + }{ + { + name: "images only", + cmd: RefsCmd{Note: "router", Images: true}, + include: []string{filepath.Join(vaultDir, "Notes", "cover.png")}, + exclude: []string{"att.pdf", "https://example.com/docs", "[external-links]"}, + }, + { + name: "pdf only", + cmd: RefsCmd{Note: "router", PDF: true}, + include: []string{filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, + exclude: []string{"cover.png", "https://example.com/docs"}, + }, + { + name: "external links only", + cmd: RefsCmd{Note: "router", ExternalLinks: true}, + include: []string{"[external-links] https://example.com/docs", "[external-links] https://links.example.com"}, + exclude: []string{"cover.png", "att.pdf", "localhost", ".png"}, + }, + { + name: "combined or", + cmd: RefsCmd{Note: "router", Images: true, PDF: true}, + include: []string{filepath.Join(vaultDir, "Notes", "cover.png"), filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, + exclude: []string{"https://example.com"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := captureStdout(t, func() { + if err := tt.cmd.Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + for _, want := range tt.include { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + for _, notWant := range tt.exclude { + if strings.Contains(out, notWant) { + t.Errorf("output contains excluded %q:\n%s", notWant, out) + } + } + }) + } +} + +func TestRefsMissingHiddenUnlessFlagged(t *testing.T) { + vaultDir := writeRefsTestVault(t) + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md"}} + withFakeStore(t, fs) + + missingPath := filepath.Join(vaultDir, "Notes", "broken.png") + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "router"}).Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + if strings.Contains(out, brokenPNG) { + t.Errorf("missing reference visible by default:\n%s", out) + } + + out = captureStdout(t, func() { + if err := (&RefsCmd{Note: "router", IncludeMissing: true}).Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + if !strings.Contains(out, missingPath) || !strings.Contains(out, "(missing)") { + t.Errorf("--include-missing should show the broken reference marked missing:\n%s", out) + } +} + +// brokenPNG names the reference that does not exist on disk in the fixture. +const brokenPNG = "broken.png" + +func TestRefsMarkdownTraversalEscapeIsMissing(t *testing.T) { + vaultDir := t.TempDir() + noteDir := filepath.Join(vaultDir, "Notes") + if err := os.MkdirAll(noteDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(noteDir, "escape.md"), + []byte("[x](../../secret.pdf)"), 0o644); err != nil { + t.Fatal(err) + } + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "escape", Title: "Escape", FilePath: "Notes/escape.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "escape", IncludeMissing: true}).Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + if !strings.Contains(out, "(missing)") { + t.Errorf("traversal escape should be rejected and marked missing:\n%s", out) + } +} + +func TestRefsCrossSyntaxDedupe(t *testing.T) { + vaultDir := t.TempDir() + noteDir := filepath.Join(vaultDir, "Notes") + if err := os.MkdirAll(noteDir, 0o755); err != nil { + t.Fatal(err) + } + body := "[[a.png]] and [x](a.png) and https://example.com and [y](https://example.com)" + if err := os.WriteFile(filepath.Join(noteDir, "dedupe.md"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(noteDir, "a.png"), []byte("png"), 0o644); err != nil { + t.Fatal(err) + } + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "dedupe", Title: "Dedupe", FilePath: "Notes/dedupe.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + err := (&RefsCmd{Note: "dedupe"}).Run(&Globals{Ctx: context.Background(), VaultPath: vaultDir, Format: formatJSON}) + if err != nil { + t.Errorf("Run: %v", err) + } + }) + if want := `"total": 2`; !strings.Contains(out, want) { + t.Errorf("expected deduped total 2, got:\n%s", out) + } + if n := strings.Count(out, filepath.Join(vaultDir, "Notes", "a.png")); n != 1 { + t.Errorf("a.png listed %d times, want 1:\n%s", n, out) + } + if n := strings.Count(out, "https://example.com"); n != 1 { + t.Errorf("url listed %d times, want 1 (deduped):\n%s", n, out) + } +} + +func TestRefsJSON(t *testing.T) { + vaultDir := writeRefsTestVault(t) + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "router"}).Run(&Globals{Ctx: context.Background(), VaultPath: vaultDir, Format: formatJSON}); err != nil { + t.Errorf("Run: %v", err) + } + }) + for _, want := range []string{ + `"command": "refs"`, + `"note_slug": "router"`, + `"title": "Router"`, + `"total": 8`, + `"relative_path": "Notes/cover.png"`, + `"kind": "external-links"`, + `"missing": false`, + } { + if !strings.Contains(out, want) { + t.Errorf("json output missing %s:\n%s", want, out) + } + } + if strings.Contains(out, `"relative_path": "https://`) { + t.Errorf("external URL must not carry relative_path:\n%s", out) + } +} + +func TestRefsTSV(t *testing.T) { + vaultDir := writeRefsTestVault(t) + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "router"}).Run(&Globals{Ctx: context.Background(), VaultPath: vaultDir, Format: formatTSV}); err != nil { + t.Errorf("Run: %v", err) + } + }) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if lines[0] != "path\tkind\tmissing\trelative_path" { + t.Errorf("tsv header = %q", lines[0]) + } + first := filepath.Join(vaultDir, "Notes", "cover.png") + if want := fmt.Sprintf("%s\timage\tfalse\tNotes/cover.png", first); lines[1] != want { + t.Errorf("tsv row = %q, want %q", lines[1], want) + } + last := lines[len(lines)-1] + if want := "https://links.example.com\texternal-links\t\t"; last != want { + t.Errorf("external tsv row = %q, want %q", last, want) + } +} + +func TestRefsJSONPath(t *testing.T) { + vaultDir := writeRefsTestVault(t) + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "router"}).Run(&Globals{Ctx: context.Background(), VaultPath: vaultDir, JSONPath: "$.refs[*].path"}); err != nil { + t.Errorf("Run: %v", err) + } + }) + want := filepath.Join(vaultDir, "Notes", "cover.png") + if !strings.Contains(out, want) { + t.Errorf("jsonpath output missing %q:\n%s", want, out) + } + if !strings.Contains(out, "https://example.com/docs") { + t.Errorf("jsonpath output missing external url:\n%s", out) + } +} + +func TestRefsEmptyResult(t *testing.T) { + vaultDir := t.TempDir() + noteDir := filepath.Join(vaultDir, "Notes") + if err := os.MkdirAll(noteDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(noteDir, "plain.md"), []byte("just a note with no references"), 0o644); err != nil { + t.Fatal(err) + } + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "plain", Title: "Plain", FilePath: "Notes/plain.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "plain"}).Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + if !strings.Contains(out, "No references found") { + t.Errorf("text mode should report no references:\n%s", out) + } +} + +func TestRefsPDFNoteError(t *testing.T) { + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "guide", Title: "Guide", FilePath: "Notes/guide.pdf"}} + withFakeStore(t, fs) + + vaultDir := t.TempDir() + err := (&RefsCmd{Note: "guide"}).Run(refsTestGlobals(vaultDir)) + if err == nil || !strings.Contains(err.Error(), "is a PDF") { + t.Errorf("expected PDF-note error, got %v", err) + } +} + +func TestRefsNoteFileMissingError(t *testing.T) { + vaultDir := t.TempDir() + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "ghost", Title: "Ghost", FilePath: "Notes/ghost.md"}} + withFakeStore(t, fs) + + err := (&RefsCmd{Note: "ghost"}).Run(refsTestGlobals(vaultDir)) + if err == nil || !strings.Contains(err.Error(), "--vault-path") { + t.Errorf("expected missing-file error hinting --vault-path, got %v", err) + } +} + +func TestRefsUsageErrors(t *testing.T) { + withFakeStore(t, &fakeStore{}) + + t.Run("missing vault path", func(t *testing.T) { + err := (&RefsCmd{Note: "router"}).Run(&Globals{Ctx: context.Background()}) + var usage *UsageError + if !errors.As(err, &usage) { + t.Errorf("expected UsageError, got %T: %v", err, err) + } + }) + t.Run("missing note", func(t *testing.T) { + err := (&RefsCmd{}).Run(refsTestGlobals(t.TempDir())) + var usage *UsageError + if !errors.As(err, &usage) { + t.Errorf("expected UsageError, got %T: %v", err, err) + } + }) +} + +func TestRefsNoteNotFoundPassthrough(t *testing.T) { + fs := &fakeStore{noteMetaErr: errors.New("note not found: nope")} + withFakeStore(t, fs) + + err := (&RefsCmd{Note: "nope"}).Run(refsTestGlobals(t.TempDir())) + if err == nil || err.Error() != "note not found: nope" { + t.Errorf("store error must pass through untouched, got %v", err) + } +} + +func TestPrintRefsFormattedToWriter(t *testing.T) { + env := refsEnvelope{ + Command: "refs", + NoteSlug: "router", + Title: "Router", + Total: 2, + Refs: []refEntry{ + {Path: "/vault/Notes/cover.png", RelativePath: "Notes/cover.png", Kind: kindImage, Missing: false}, + {Path: "https://example.com", Kind: kindExternal, Missing: false}, + }, + } + globals := &Globals{} + + t.Run("text", func(t *testing.T) { + var sb strings.Builder + if err := printRefsFormattedToWriter(&sb, env, globals); err != nil { + t.Fatal(err) + } + want := "[image] /vault/Notes/cover.png\n[external-links] https://example.com\n" + if sb.String() != want { + t.Errorf("text = %q, want %q", sb.String(), want) + } + }) + + t.Run("text marks missing", func(t *testing.T) { + envMissing := env + envMissing.Refs = append([]refEntry(nil), env.Refs...) + envMissing.Refs[0].Missing = true + var sb strings.Builder + if err := printRefsFormattedToWriter(&sb, envMissing, globals); err != nil { + t.Fatal(err) + } + if !strings.Contains(sb.String(), "(missing)") { + t.Errorf("missing marker absent: %q", sb.String()) + } + }) + + t.Run("json", func(t *testing.T) { + var sb strings.Builder + if err := printRefsFormattedToWriter(&sb, env, &Globals{Format: formatJSON}); err != nil { + t.Fatal(err) + } + out := sb.String() + for _, want := range []string{`"command": "refs"`, `"total": 2`, `"relative_path": "Notes/cover.png"`, `"missing": false`} { + if !strings.Contains(out, want) { + t.Errorf("json missing %s:\n%s", want, out) + } + } + }) + + t.Run("tsv", func(t *testing.T) { + var sb strings.Builder + if err := printRefsFormattedToWriter(&sb, env, &Globals{Format: formatTSV}); err != nil { + t.Fatal(err) + } + want := "path\tkind\tmissing\trelative_path\n/vault/Notes/cover.png\timage\tfalse\tNotes/cover.png\nhttps://example.com\texternal-links\t\t\n" + if sb.String() != want { + t.Errorf("tsv = %q, want %q", sb.String(), want) + } + }) + + t.Run("jsonpath", func(t *testing.T) { + var sb strings.Builder + if err := printRefsFormattedToWriter(&sb, env, &Globals{JSONPath: "$.refs[*].path"}); err != nil { + t.Fatal(err) + } + want := "/vault/Notes/cover.png\nhttps://example.com\n" + if sb.String() != want { + t.Errorf("jsonpath = %q, want %q", sb.String(), want) + } + }) + + t.Run("empty text", func(t *testing.T) { + empty := refsEnvelope{Command: "refs", NoteSlug: "x", Refs: nil} + var sb strings.Builder + if err := printRefsFormattedToWriter(&sb, empty, globals); err != nil { + t.Fatal(err) + } + if !strings.Contains(sb.String(), "No references found") { + t.Errorf("empty text should print a notice, got %q", sb.String()) + } + }) +} diff --git a/cmd/testhelpers_test.go b/cmd/testhelpers_test.go index 827d463..d837b1f 100644 --- a/cmd/testhelpers_test.go +++ b/cmd/testhelpers_test.go @@ -13,17 +13,18 @@ import ( // mutation calls and returns zero values for queries; tests can extend it // with the behavior they need. type fakeStore struct { - mu sync.Mutex - resetCalls int - resetErr error - tags []store.TagCount - suggest []string - semantic []store.Result - lexical []store.Result - metaCalls int - headCalls int - noteMeta *store.NoteContent - noteHead *store.NoteContent + mu sync.Mutex + resetCalls int + resetErr error + tags []store.TagCount + suggest []string + semantic []store.Result + lexical []store.Result + metaCalls int + headCalls int + noteMeta *store.NoteContent + noteMetaErr error + noteHead *store.NoteContent } func (f *fakeStore) Close() error { return nil } @@ -103,6 +104,9 @@ func (f *fakeStore) GetNoteMeta(context.Context, string) (*store.NoteContent, er f.mu.Lock() defer f.mu.Unlock() f.metaCalls++ + if f.noteMetaErr != nil { + return nil, f.noteMetaErr + } return f.noteMeta, nil } From d46ff1090d70523a4b62438a4a8d8f2f1d5055e8 Mon Sep 17 00:00:00 2001 From: nmdra Date: Wed, 12 Aug 2026 14:16:36 +0530 Subject: [PATCH 05/18] docs: document the refs command - README: feature bullet and chaining example - wiki/Commands.md: full refs section with flags, examples, JSON shape - AGENTS.md: structure tree entry and CLI flag standards - CHANGELOG.md: unreleased entry for the refs command --- .agents/AGENTS.md | 3 +- CHANGELOG.md | 5 ++++ README.md | 6 +++- wiki/Commands.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 5969788..166d6d6 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -58,6 +58,7 @@ notebrain-cli/ ├── ingest.go ├── search.go ├── backlinks.go + ├── refs.go ├── connections.go ├── hidden.go ├── tags.go @@ -75,7 +76,7 @@ notebrain-cli/ - Name test files `*_test.go` alongside the source file. - **Go Vendoring:** This repository uses Go vendoring (`vendor/`). Whenever dependencies in `go.mod` or `go.sum` are added, removed, or updated, you MUST run `go mod vendor` before running tests or builds. - **Strict Non-Regression Guardrails:** When refactoring or removing features, always add explicit assertion tests across `internal/configfile/` and `internal/store/` to verify that existing core functions, default settings, TOML key resolution, and database initialization do not regress or depend on removed parameters. -- **CLI Testing & Flag Standards:** When executing CLI commands or writing automated tests/scripts for NoteBrain, strictly use the exact flag names `--vault-path` and `--chroma-path` (never `--vault` or `--db`). For graph and note commands (`backlinks`, `connections`, `hidden`, `tags`, `get`), pass exactly one positional argument: the note slug (``). For `boosted` search, always provide the required `--seed=` flag. When testing `hidden` connection discovery where already-linked notes should be included, pass `--include-linked`. Note that `backlinks` and `connections` canonicalize link targets by stripping `#heading` anchors and matching exact vault subfolders. Use `--show-tags` to show tags in CLI output. Debug logging is enabled via `--debug`. When testing `reset` in automated scripts, pipe confirmation via stdin (`echo yes | ./notebrain reset`). To avoid contextual empty-result hints in automated scripts, always request machine formats (`--format=json`, `tsv`, or `--jsonpath`). When testing LLM-based PDF ingestion, use `--llm-model` and provide the required API key via environment variables (`DEEPSEEK_API_KEY`, or `OPENROUTER_API_KEY`). +- **CLI Testing & Flag Standards:** When executing CLI commands or writing automated tests/scripts for NoteBrain, strictly use the exact flag names `--vault-path` and `--chroma-path` (never `--vault` or `--db`). For graph and note commands (`backlinks`, `connections`, `hidden`, `tags`, `get`, `refs`), pass exactly one positional argument: the note slug (``). For `refs`, use `--images`/`--pdf`/`--other`/`--external-links` to filter by kind and `--include-missing` to surface broken attachment links. For `boosted` search, always provide the required `--seed=` flag. When testing `hidden` connection discovery where already-linked notes should be included, pass `--include-linked`. Note that `backlinks` and `connections` canonicalize link targets by stripping `#heading` anchors and matching exact vault subfolders. Use `--show-tags` to show tags in CLI output. Debug logging is enabled via `--debug`. When testing `reset` in automated scripts, pipe confirmation via stdin (`echo yes | ./notebrain reset`). To avoid contextual empty-result hints in automated scripts, always request machine formats (`--format=json`, `tsv`, or `--jsonpath`). When testing LLM-based PDF ingestion, use `--llm-model` and provide the required API key via environment variables (`DEEPSEEK_API_KEY`, or `OPENROUTER_API_KEY`). ## Coding Conventions diff --git a/CHANGELOG.md b/CHANGELOG.md index f6cb072..002cb86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- **Reference Listing**: `notebrain refs ` lists a note's local attachments (images, PDFs, archives, …) and external website links, filterable with `--images`, `--pdf`, `--other`, and `--external-links`. Broken links are hidden by default and surfaced with `--include-missing` (`feat(cmd)`). + ## [v2.12.0] - 2026-08-02 ### Added diff --git a/README.md b/README.md index 4897015..fb5313b 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ NoteBrain includes an [AI agent skill](wiki/Skill_Usage.md) and an [OpenCode Age - **Graph-Boosted Ranking**: Combine semantic similarity with graph relationships for better search results. - **Advanced Filtering**: Filter results by sections, tags, code blocks, tasks, and other metadata. - **Full Note Retrieval**: Get the complete note from the indexed content. +- **Reference Listing**: List a note's local attachments (images, PDFs, archives) and external website links with `notebrain refs`, filterable by kind — `--images`, `--pdf`, or `--external-links`. - **Structured Output**: Export results as JSON or TSV. Use built-in JSONPath queries for automation. - **AI Agent Integration**: NoteBrain has a built-in AI agent skill for autonomous knowledge retrieval. - **Terminal Hyperlinks**: Use OSC 8 hyperlinks to open notes from supported terminals. @@ -128,7 +129,7 @@ notebrain search "how do message brokers work?" --limit 2 --top-k 1 --format=jso -**6. Chain commands to retrieve full notes:** +**6. Chain commands to retrieve full notes and their references:** ```bash # Extract slug from top search result @@ -136,6 +137,9 @@ SLUG=$(notebrain search "message broker" --limit 1 --jsonpath="$.results[0].note # Retrieve complete reconstructed note text notebrain get "$SLUG" --jsonpath="$.text" + +# Fetch absolute paths of the note's image attachments +notebrain refs "$SLUG" --images --jsonpath='$.refs[*].path' ``` **7. Automate indexing:** Set a cron job or systemd timer to keep your index current. Read [Scheduled Ingestion](wiki/Scheduled_Ingestion.md). diff --git a/wiki/Commands.md b/wiki/Commands.md index 13ad284..447b3c6 100644 --- a/wiki/Commands.md +++ b/wiki/Commands.md @@ -279,6 +279,80 @@ notebrain get "kubernetes-native-applications" --format json --- +### `refs` + +This command lists the direct references of a note: local attachment files (images, PDFs, archives, …) and external website links. It parses the note file fresh from the vault, so it sees everything, including embedded images that the index intentionally skips. References are listed in first-occurrence order, deduplicated by resolved path (or exact URL). + +The command resolves the note exactly like the other note commands (slug, title, filename, or partial path), then prints absolute file paths and URLs, filterable by kind. + +Reference resolution follows Obsidian semantics: + +- **Wiki links** (`[[file.png]]`): targets containing a folder path start at the vault root; `./` targets resolve relative to the note's folder; bare names search the note's folder, then the vault root, then the configured `attachmentFolderPath` from `.obsidian/app.json`. +- **Markdown links** (`[doc](file.pdf)`, `![alt](img.png)`): resolve relative to the note's folder, with percent-decoding (`Router%20Modes.webp` → `Router Modes.webp`) and traversal protection (`..` escapes are rejected). + +Broken links are hidden by default; pass `--include-missing` to list them marked `missing: true`. External links are never verified (the tool is offline by design), so URL rows always report `missing: false`. + +#### Usage + +```bash +notebrain refs [flags] +``` + +#### Arguments + +- `` (required): The note slug, title, or file path (auto-resolved). + +#### Command-Specific Flags + +| Flag | Description | +| --- | --- | +| `--images` | Include image attachments only | +| `--pdf` | Include PDF attachments only | +| `--other` | Include other attachments (video, audio, archives, office docs) | +| `--external-links` | Include external website links (URLs) only | +| `--include-missing` | Include references whose file is missing from the vault (marked `missing: true`) | + +Filters combine with OR semantics; with no filter flags every kind is listed. + +#### Examples + +```bash +# List every reference of a note +notebrain refs "kubernetes-notes" + +# List image attachments as machine-readable JSON +notebrain refs "kubernetes-notes" --images --format=json + +# List PDF attachments as TSV +notebrain refs "kubernetes-notes" --pdf --format=tsv + +# List external website links +notebrain refs "kubernetes-notes" --external-links --format=json + +# Feed attachment paths straight into a script +notebrain refs "$SLUG" --images --jsonpath='$.refs[*].path' +``` + +#### JSON shape + +```json +{ + "command": "refs", + "note_slug": "kubernetes-notes", + "title": "Kubernetes Notes", + "total": 3, + "refs": [ + {"path": "/vault/assets/arch.png", "relative_path": "assets/arch.png", "kind": "image", "missing": false}, + {"path": "/vault/99.Storage-Shed/Attachments/guide.pdf", "relative_path": "99.Storage-Shed/Attachments/guide.pdf", "kind": "pdf", "missing": false}, + {"path": "https://example.com/docs", "kind": "external-links", "missing": false} + ] +} +``` + +External rows omit `relative_path`. TSV output uses the header `path\tkind\tmissing\trelative_path`; external rows leave `missing` and `relative_path` empty. + +--- + ### `backlinks` This command finds all the notes that link to the target note. It uses the local Wikilink graph. The link target resolution is fully canonicalized. This means that the tool removes `#anchor` headings and resolves subfolders against canonical paths. This makes sure that the tool finds connections across deeply nested vault hierarchies. From 0536f6c1bc7df8acaf8e100a11897e6e3799628b Mon Sep 17 00:00:00 2001 From: nmdra Date: Thu, 13 Aug 2026 03:59:04 +0530 Subject: [PATCH 06/18] fix(cmd): keep refs order and reject traversal escapes - iterate extracted refs in document order so external links interleave with attachments (first-occurrence order) - drop attachment candidates that escape the vault instead of listing them as missing Refs: #code-review (standards + spec findings on refs branch) --- cmd/refs.go | 27 +++++--- cmd/refs_test.go | 53 ++++++++++++-- internal/parser/attachments.go | 55 +++++++-------- internal/parser/attachments_test.go | 103 +++++++++++++++------------- 4 files changed, 147 insertions(+), 91 deletions(-) diff --git a/cmd/refs.go b/cmd/refs.go index 11ed74c..6ec0a59 100644 --- a/cmd/refs.go +++ b/cmd/refs.go @@ -133,11 +133,14 @@ func (c *RefsCmd) resolveRefs(vaultPath, noteFilePath string, extracted parser.E entries = append(entries, e) } - for _, ref := range extracted.Attachments { - add(c.resolveAttachment(vaultPath, noteDir, attachmentFolder, ref)) - } - for _, link := range extracted.External { - add(refEntry{Path: link, Kind: kindExternal}) + for _, ref := range extracted.Refs { + if ref.Kind == parser.KindExternalLinks { + add(refEntry{Path: ref.Target, Kind: kindExternal}) + continue + } + if entry, ok := c.resolveAttachment(vaultPath, noteDir, attachmentFolder, ref); ok { + add(entry) + } } return entries } @@ -145,9 +148,10 @@ func (c *RefsCmd) resolveRefs(vaultPath, noteFilePath string, extracted parser.E // resolveAttachment finds the on-disk path of one attachment reference. Wiki // refs follow Obsidian's search order (note folder, vault root, attachment // folder); markdown refs resolve note-folder-relative with percent-decoding. -// A reference that escapes the vault or matches no existing file resolves to -// its first candidate marked missing. -func (c *RefsCmd) resolveAttachment(vaultPath, noteDir, attachmentFolder string, ref parser.AttachmentRef) refEntry { +// A reference that escapes the vault is rejected outright (dropped, never +// even listed as missing); one that matches no existing file resolves to its +// first candidate marked missing. +func (c *RefsCmd) resolveAttachment(vaultPath, noteDir, attachmentFolder string, ref parser.Ref) (refEntry, bool) { var candidates []string switch ref.Source { case parser.SrcMarkdown: @@ -156,15 +160,18 @@ func (c *RefsCmd) resolveAttachment(vaultPath, noteDir, attachmentFolder string, candidates = wikiCandidates(vaultPath, noteDir, attachmentFolder, ref.Target) } first := candidates[0] + if !insideVault(vaultPath, first) { + return refEntry{}, false + } for _, cand := range candidates { if !insideVault(vaultPath, cand) { continue } if _, err := os.Stat(cand); err == nil { - return refEntry{Path: cand, RelativePath: vaultRelativePath(vaultPath, cand), Kind: string(ref.Kind)} + return refEntry{Path: cand, RelativePath: vaultRelativePath(vaultPath, cand), Kind: string(ref.Kind)}, true } } - return refEntry{Path: first, RelativePath: vaultRelativePath(vaultPath, first), Kind: string(ref.Kind), Missing: true} + return refEntry{Path: first, RelativePath: vaultRelativePath(vaultPath, first), Kind: string(ref.Kind), Missing: true}, true } // wikiCandidates returns candidate paths for a wiki target in Obsidian diff --git a/cmd/refs_test.go b/cmd/refs_test.go index baed4f4..1978407 100644 --- a/cmd/refs_test.go +++ b/cmd/refs_test.go @@ -179,14 +179,14 @@ func TestRefsMissingHiddenUnlessFlagged(t *testing.T) { // brokenPNG names the reference that does not exist on disk in the fixture. const brokenPNG = "broken.png" -func TestRefsMarkdownTraversalEscapeIsMissing(t *testing.T) { +func TestRefsTraversalEscapeDropped(t *testing.T) { vaultDir := t.TempDir() noteDir := filepath.Join(vaultDir, "Notes") if err := os.MkdirAll(noteDir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(noteDir, "escape.md"), - []byte("[x](../../secret.pdf)"), 0o644); err != nil { + []byte("[x](../../secret.pdf)\n\n[[../../secret.png]]"), 0o644); err != nil { t.Fatal(err) } fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "escape", Title: "Escape", FilePath: "Notes/escape.md"}} @@ -197,8 +197,53 @@ func TestRefsMarkdownTraversalEscapeIsMissing(t *testing.T) { t.Errorf("Run: %v", err) } }) - if !strings.Contains(out, "(missing)") { - t.Errorf("traversal escape should be rejected and marked missing:\n%s", out) + for _, forbidden := range []string{"secret.pdf", "secret.png", "(missing)"} { + if strings.Contains(out, forbidden) { + t.Errorf("traversal escape must be dropped entirely, got %q:\n%s", forbidden, out) + } + } +} + +func TestRefsCrossKindOrder(t *testing.T) { + vaultDir := t.TempDir() + noteDir := filepath.Join(vaultDir, "Notes") + if err := os.MkdirAll(noteDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(noteDir, "order.md"), + []byte("[ext](https://example.com/docs)\n\n![[cover.png]]\n\n[more](https://links.example.com)\n\n![[second.png]]"), 0o644); err != nil { + t.Fatal(err) + } + for _, f := range []string{"cover.png", "second.png"} { + if err := os.WriteFile(filepath.Join(noteDir, f), []byte("png"), 0o644); err != nil { + t.Fatal(err) + } + } + fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "order", Title: "Order", FilePath: "Notes/order.md"}} + withFakeStore(t, fs) + + out := captureStdout(t, func() { + if err := (&RefsCmd{Note: "order"}).Run(refsTestGlobals(vaultDir)); err != nil { + t.Errorf("Run: %v", err) + } + }) + wantLines := []string{ + "[external-links] https://example.com/docs", + "[image] " + filepath.Join(vaultDir, "Notes", "cover.png"), + "[external-links] https://links.example.com", + "[image] " + filepath.Join(vaultDir, "Notes", "second.png"), + } + idxs := make([]int, len(wantLines)) + for i, want := range wantLines { + idxs[i] = strings.Index(out, want) + if idxs[i] == -1 { + t.Fatalf("output missing %q:\n%s", want, out) + } + } + for i := 1; i < len(idxs); i++ { + if idxs[i] < idxs[i-1] { + t.Errorf("output not in first-occurrence order: %q before %q:\n%s", wantLines[i-1], wantLines[i], out) + } } } diff --git a/internal/parser/attachments.go b/internal/parser/attachments.go index 1e6904b..faf9c7b 100644 --- a/internal/parser/attachments.go +++ b/internal/parser/attachments.go @@ -32,47 +32,42 @@ const ( SrcMarkdown AttachmentSource = "markdown" ) -// AttachmentRef is one attachment reference extracted from a note body. -// Target is cleaned for wiki refs (alias/anchor stripped) and the raw -// destination for markdown refs; resolution happens in the caller. -type AttachmentRef struct { +// Ref is one reference extracted from a note body. Target holds the cleaned +// wiki target, the raw markdown destination, or the full URL for external +// links; Kind classifies it; Source records which markdown syntax produced it +// so resolution can apply Obsidian semantics (wiki vs note-folder-relative). +// External links carry no Source: they are never resolved against the vault. +type Ref struct { Target string Kind AttachmentKind Source AttachmentSource } -// ExtractedRefs holds the references collected from a note body. +// ExtractedRefs holds the references collected from a note body, deduped (by +// cleaned target or exact URL) in first-occurrence document order across all +// kinds. type ExtractedRefs struct { - Attachments []AttachmentRef - External []string + Refs []Ref } // ExtractReferences walks a note body's AST and collects direct references: // local attachments (wiki and markdown syntax) and external http(s) website // links. URLs and content inside code fences never match. Results are deduped -// (attachments by cleaned target, external by exact URL) in first-occurrence -// document order. +// by target (or exact URL) in first-occurrence document order, so an external +// link that appears before an image is reported before it. func ExtractReferences(body string) ExtractedRefs { src := []byte(body) doc := mdParser.Parser().Parse(text.NewReader(src)) var refs ExtractedRefs - seenAttachments := make(map[string]struct{}) - seenExternal := make(map[string]struct{}) + seen := make(map[string]struct{}) - addAttachment := func(ref AttachmentRef) { - if _, ok := seenAttachments[ref.Target]; ok { + add := func(ref Ref) { + if _, ok := seen[ref.Target]; ok { return } - seenAttachments[ref.Target] = struct{}{} - refs.Attachments = append(refs.Attachments, ref) - } - addExternal := func(url string) { - if _, ok := seenExternal[url]; ok { - return - } - seenExternal[url] = struct{}{} - refs.External = append(refs.External, url) + seen[ref.Target] = struct{}{} + refs.Refs = append(refs.Refs, ref) } _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { @@ -86,17 +81,17 @@ func ExtractReferences(body string) ExtractedRefs { break } if isHTTPScheme(target) { - addExternal(target) + add(Ref{Target: target, Kind: KindExternalLinks}) break } cleaned := cleanWikiTarget(target) if kind, ok := classifyAttachmentKind(cleaned); ok { - addAttachment(AttachmentRef{Target: cleaned, Kind: kind, Source: SrcWiki}) + add(Ref{Target: cleaned, Kind: kind, Source: SrcWiki}) } case *ast.Link: - handleMarkdownReference(string(nTyped.Destination), addAttachment, addExternal) + handleMarkdownReference(string(nTyped.Destination), add) case *ast.Image: - handleMarkdownReference(string(nTyped.Destination), addAttachment, addExternal) + handleMarkdownReference(string(nTyped.Destination), add) case *ast.AutoLink: if nTyped.AutoLinkType != ast.AutoLinkURL { break @@ -104,7 +99,7 @@ func ExtractReferences(body string) ExtractedRefs { // URL() assembles the scheme for <...> autolinks; linkify nodes // already carry it in the value. Either way only http(s) counts. if url := string(nTyped.URL(src)); isHTTPScheme(url) { - addExternal(url) + add(Ref{Target: url, Kind: KindExternalLinks}) } } return ast.WalkContinue, nil @@ -115,13 +110,13 @@ func ExtractReferences(body string) ExtractedRefs { // handleMarkdownReference classifies a markdown link/image destination as // either an external http(s) URL or a local attachment. -func handleMarkdownReference(destination string, addAttachment func(AttachmentRef), addExternal func(string)) { +func handleMarkdownReference(destination string, add func(Ref)) { if isHTTPScheme(destination) { - addExternal(destination) + add(Ref{Target: destination, Kind: KindExternalLinks}) return } if kind, ok := classifyAttachmentKind(destination); ok { - addAttachment(AttachmentRef{Target: destination, Kind: kind, Source: SrcMarkdown}) + add(Ref{Target: destination, Kind: kind, Source: SrcMarkdown}) } } diff --git a/internal/parser/attachments_test.go b/internal/parser/attachments_test.go index 1b2f8d5..211a5ad 100644 --- a/internal/parser/attachments_test.go +++ b/internal/parser/attachments_test.go @@ -13,21 +13,21 @@ func TestExtractReferences_WikiAttachments(t *testing.T) { tests := []struct { name string body string - want []AttachmentRef + want []Ref }{ - {name: "image embed with size", body: "![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, - {name: "pdf plain link", body: "[[doc.pdf]]", want: []AttachmentRef{{Target: "doc.pdf", Kind: KindPDF, Source: SrcWiki}}}, - {name: "image embed with alias", body: "![[img.png|alt text]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, - {name: "image with heading anchor", body: "[[img.png#anchor]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, - {name: "subfolder image embed", body: "![[sub/img.png]]", want: []AttachmentRef{{Target: "sub/img.png", Kind: KindImage, Source: SrcWiki}}}, - {name: "relative dot prefix", body: "[[./local.png]]", want: []AttachmentRef{{Target: "./local.png", Kind: KindImage, Source: SrcWiki}}}, - {name: "archive attachment", body: "[[bundle.zip]]", want: []AttachmentRef{{Target: "bundle.zip", Kind: KindOther, Source: SrcWiki}}}, - {name: "canvas attachment", body: "[[diagram.canvas]]", want: []AttachmentRef{{Target: "diagram.canvas", Kind: KindOther, Source: SrcWiki}}}, + {name: "image embed with size", body: "![[img.png|200]]", want: []Ref{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "pdf plain link", body: "[[doc.pdf]]", want: []Ref{{Target: "doc.pdf", Kind: KindPDF, Source: SrcWiki}}}, + {name: "image embed with alias", body: "![[img.png|alt text]]", want: []Ref{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "image with heading anchor", body: "[[img.png#anchor]]", want: []Ref{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "subfolder image embed", body: "![[sub/img.png]]", want: []Ref{{Target: "sub/img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "relative dot prefix", body: "[[./local.png]]", want: []Ref{{Target: "./local.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "archive attachment", body: "[[bundle.zip]]", want: []Ref{{Target: "bundle.zip", Kind: KindOther, Source: SrcWiki}}}, + {name: "canvas attachment", body: "[[diagram.canvas]]", want: []Ref{{Target: "diagram.canvas", Kind: KindOther, Source: SrcWiki}}}, {name: "unknown extension is not an attachment", body: "[[archive.xyz]]", want: nil}, {name: "dotted note name is not an attachment", body: "[[Note 1.2.3]]", want: nil}, {name: "plain note link is not an attachment", body: "[[Other Note]]", want: nil}, - {name: "uppercase extension is case-insensitive", body: "![[PHOTO.PNG]]", want: []AttachmentRef{{Target: "PHOTO.PNG", Kind: KindImage, Source: SrcWiki}}}, - {name: "duplicate embeds dedupe", body: "![[img.png]]\n\n![[img.png|200]]", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, + {name: "uppercase extension is case-insensitive", body: "![[PHOTO.PNG]]", want: []Ref{{Target: "PHOTO.PNG", Kind: KindImage, Source: SrcWiki}}}, + {name: "duplicate embeds dedupe", body: "![[img.png]]\n\n![[img.png|200]]", want: []Ref{{Target: "img.png", Kind: KindImage, Source: SrcWiki}}}, {name: "code fence contents ignored", body: "```\n![[x.png]]\n[[secret.pdf]]\n```", want: nil}, {name: "inline code ignored", body: "`![[x.png]]`", want: nil}, {name: "empty target ignored", body: "[[#heading]]", want: nil}, @@ -35,11 +35,8 @@ func TestExtractReferences_WikiAttachments(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := ExtractReferences(tt.body) - if !reflect.DeepEqual(got.Attachments, tt.want) { - t.Errorf("ExtractReferences(%q).Attachments = %v, want %v", tt.body, got.Attachments, tt.want) - } - if len(got.External) != 0 { - t.Errorf("ExtractReferences(%q).External = %v, want none", tt.body, got.External) + if !reflect.DeepEqual(got.Refs, tt.want) { + t.Errorf("ExtractReferences(%q).Refs = %v, want %v", tt.body, got.Refs, tt.want) } }) } @@ -49,24 +46,24 @@ func TestExtractReferences_MarkdownAttachments(t *testing.T) { tests := []struct { name string body string - want []AttachmentRef + want []Ref }{ - {name: "image", body: "![alt](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcMarkdown}}}, - {name: "pdf in subfolder", body: "[doc](sub/file.pdf)", want: []AttachmentRef{{Target: "sub/file.pdf", Kind: KindPDF, Source: SrcMarkdown}}}, - {name: "percent-encoded destination kept raw", body: "![alt](Router%20Modes.webp)", want: []AttachmentRef{{Target: "Router%20Modes.webp", Kind: KindImage, Source: SrcMarkdown}}}, - {name: "parent traversal kept raw", body: "[x](../up.pdf)", want: []AttachmentRef{{Target: "../up.pdf", Kind: KindPDF, Source: SrcMarkdown}}}, - {name: "pdf with page fragment", body: "[x](STP.pdf#page=5)", want: []AttachmentRef{{Target: "STP.pdf#page=5", Kind: KindPDF, Source: SrcMarkdown}}}, - {name: "external link is not an attachment", body: "[text](https://example.com)", want: nil}, + {name: "image", body: "![alt](img.png)", want: []Ref{{Target: "img.png", Kind: KindImage, Source: SrcMarkdown}}}, + {name: "pdf in subfolder", body: "[doc](sub/file.pdf)", want: []Ref{{Target: "sub/file.pdf", Kind: KindPDF, Source: SrcMarkdown}}}, + {name: "percent-encoded destination kept raw", body: "![alt](Router%20Modes.webp)", want: []Ref{{Target: "Router%20Modes.webp", Kind: KindImage, Source: SrcMarkdown}}}, + {name: "parent traversal kept raw", body: "[x](../up.pdf)", want: []Ref{{Target: "../up.pdf", Kind: KindPDF, Source: SrcMarkdown}}}, + {name: "pdf with page fragment", body: "[x](STP.pdf#page=5)", want: []Ref{{Target: "STP.pdf#page=5", Kind: KindPDF, Source: SrcMarkdown}}}, + {name: "external link is not an attachment", body: "[text](https://example.com)", want: []Ref{{Target: "https://example.com", Kind: KindExternalLinks}}}, {name: "relative note link without extension", body: "[rel](../other-note)", want: nil}, {name: "anchor-only link", body: "[x](#anchor)", want: nil}, {name: "code fence contents ignored", body: "```\n![x](img.png)\n```", want: nil}, - {name: "duplicate destinations dedupe", body: "![a](img.png) and ![b](img.png)", want: []AttachmentRef{{Target: "img.png", Kind: KindImage, Source: SrcMarkdown}}}, + {name: "duplicate destinations dedupe", body: "![a](img.png) and ![b](img.png)", want: []Ref{{Target: "img.png", Kind: KindImage, Source: SrcMarkdown}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := ExtractReferences(tt.body) - if !reflect.DeepEqual(got.Attachments, tt.want) { - t.Errorf("ExtractReferences(%q).Attachments = %v, want %v", tt.body, got.Attachments, tt.want) + if !reflect.DeepEqual(got.Refs, tt.want) { + t.Errorf("ExtractReferences(%q).Refs = %v, want %v", tt.body, got.Refs, tt.want) } }) } @@ -76,44 +73,56 @@ func TestExtractReferences_External(t *testing.T) { tests := []struct { name string body string - want []string + want []Ref }{ - {name: "markdown link", body: "[text](https://example.com/a)", want: []string{"https://example.com/a"}}, - {name: "markdown image embed", body: "![alt](https://example.com/i.png)", want: []string{"https://example.com/i.png"}}, - {name: "bare url", body: "see https://example.com here", want: []string{"https://example.com"}}, - {name: "angle url", body: "", want: []string{"https://example.com"}}, - {name: "bare www url gains http protocol", body: "visit www.example.com", want: []string{"http://www.example.com"}}, - {name: "wikilink to external url", body: "[[https://example.com]]", want: []string{"https://example.com"}}, - {name: "wikilink to external image url", body: "[[https://example.com/img.png]]", want: []string{"https://example.com/img.png"}}, - {name: "multiple urls keep first occurrence order", body: "[b](https://b.org)\n\n[a](https://a.org) and https://b.org", want: []string{"https://b.org", "https://a.org"}}, + {name: "markdown link", body: "[text](https://example.com/a)", want: []Ref{{Target: "https://example.com/a", Kind: KindExternalLinks}}}, + {name: "markdown image embed", body: "![alt](https://example.com/i.png)", want: []Ref{{Target: "https://example.com/i.png", Kind: KindExternalLinks}}}, + {name: "bare url", body: "see https://example.com here", want: []Ref{{Target: "https://example.com", Kind: KindExternalLinks}}}, + {name: "angle url", body: "", want: []Ref{{Target: "https://example.com", Kind: KindExternalLinks}}}, + {name: "bare www url gains http protocol", body: "visit www.example.com", want: []Ref{{Target: "http://www.example.com", Kind: KindExternalLinks}}}, + {name: "wikilink to external url", body: "[[https://example.com]]", want: []Ref{{Target: "https://example.com", Kind: KindExternalLinks}}}, + {name: "wikilink to external image url", body: "[[https://example.com/img.png]]", want: []Ref{{Target: "https://example.com/img.png", Kind: KindExternalLinks}}}, + {name: "multiple urls keep first occurrence order", body: "[b](https://b.org)\n\n[a](https://a.org) and https://b.org", want: []Ref{ + {Target: "https://b.org", Kind: KindExternalLinks}, + {Target: "https://a.org", Kind: KindExternalLinks}, + }}, {name: "code fence contents ignored", body: "```\nhttps://example.com\n```", want: nil}, {name: "excluded schemes", body: "[mail](mailto:a@b.c)\n\n[ftp](ftp://x.y/z)\n\nemail me at a@b.c", want: nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := ExtractReferences(tt.body) - if !reflect.DeepEqual(got.External, tt.want) { - t.Errorf("ExtractReferences(%q).External = %v, want %v", tt.body, got.External, tt.want) - } - if len(got.Attachments) != 0 { - t.Errorf("ExtractReferences(%q).Attachments = %v, want none", tt.body, got.Attachments) + if !reflect.DeepEqual(got.Refs, tt.want) { + t.Errorf("ExtractReferences(%q).Refs = %v, want %v", tt.body, got.Refs, tt.want) } }) } } +func TestExtractReferences_CrossKindFirstOccurrenceOrder(t *testing.T) { + body := "![[cover.png]] and [ext1](https://example.com) and ![[second.png]] and [ext2](https://links.example.com)" + got := ExtractReferences(body) + want := []Ref{ + {Target: "cover.png", Kind: KindImage, Source: SrcWiki}, + {Target: "https://example.com", Kind: KindExternalLinks}, + {Target: "second.png", Kind: KindImage, Source: SrcWiki}, + {Target: "https://links.example.com", Kind: KindExternalLinks}, + } + if !reflect.DeepEqual(got.Refs, want) { + t.Errorf("Refs = %v, want %v (cross-kind first-occurrence order)", got.Refs, want) + } +} + func TestExtractReferences_Mixed(t *testing.T) { body := "![[cover.png]] and [doc](manual.pdf) and https://example.com and [[https://links.example.com]]" got := ExtractReferences(body) - wantAttach := []AttachmentRef{ + want := []Ref{ {Target: "cover.png", Kind: KindImage, Source: SrcWiki}, {Target: "manual.pdf", Kind: KindPDF, Source: SrcMarkdown}, + {Target: "https://example.com", Kind: KindExternalLinks}, + {Target: "https://links.example.com", Kind: KindExternalLinks}, } - if !reflect.DeepEqual(got.Attachments, wantAttach) { - t.Errorf("Attachments = %v, want %v", got.Attachments, wantAttach) - } - wantExt := []string{"https://example.com", "https://links.example.com"} - if !reflect.DeepEqual(got.External, wantExt) { - t.Errorf("External = %v, want %v", got.External, wantExt) + if !reflect.DeepEqual(got.Refs, want) { + t.Errorf("Refs = %v, want %v", got.Refs, want) } } From 9f5feffcb8eba864ac70aa09b4757a8bc99a4ade Mon Sep 17 00:00:00 2001 From: nmdra Date: Thu, 13 Aug 2026 03:59:19 +0530 Subject: [PATCH 07/18] docs(skill): document the refs command in notebrain skill --- .agents/skills/notebrain/SKILL.md | 11 ++-- .../skills/notebrain/references/example.md | 6 +++ .agents/skills/notebrain/references/flags.md | 14 ++++- .agents/skills/notebrain/references/schema.md | 53 +++++++++++++++++++ 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/.agents/skills/notebrain/SKILL.md b/.agents/skills/notebrain/SKILL.md index b6cf0a3..54ed1ec 100644 --- a/.agents/skills/notebrain/SKILL.md +++ b/.agents/skills/notebrain/SKILL.md @@ -1,6 +1,6 @@ --- name: notebrain-assistant -description: Search and explore an Obsidian vault through the NoteBrain CLI (semantic search, tags, backlinks, connections, hidden links, boosted retrieval). Use it whenever the user mentions their notes, knowledge base, Obsidian vault, semantic search, finding connections or unlinked notes, or asks exploratory questions like "what do I know about X", "find notes related to Y", "what connects to Z", or "summarize my notes on W" — even when they never say NoteBrain, vector search, or ChromaDB. +description: Search and explore an Obsidian vault through the NoteBrain CLI (semantic search, tags, backlinks, connections, hidden links, boosted retrieval). Use it whenever the user mentions their notes, knowledge base, Obsidian vault, semantic search, finding connections or unlinked notes, or asks exploratory questions like "what do I know about X", "find notes related to Y", "what connects to Z", "summarize my notes on W", "what does this note reference or embed", or "are any links broken" — even when they never say NoteBrain, vector search, or ChromaDB. license: MIT compatibility: Requires the `notebrain` binary on PATH. allowed-tools: Bash(notebrain:*), Bash(./notebrain:*) @@ -11,7 +11,7 @@ allowed-tools: Bash(notebrain:*), Bash(./notebrain:*) NoteBrain indexes an Obsidian vault into local ChromaDB and answers read-only questions about it: semantic search, tag queries, graph structure, and note retrieval. It never mutates the vault — for writes, use standard file tools or obsidian-cli and keep NoteBrain for the discovery step. References, read on demand: -- [references/example.md](references/example.md) — 16 worked scenarios, exact commands, verified pitfalls. +- [references/example.md](references/example.md) — 18 worked scenarios, exact commands, verified pitfalls. - [references/flags.md](references/flags.md) — every flag, default, and config override. - [references/schema.md](references/schema.md) — JSON/TSV output shape, `--jsonpath` use. @@ -60,6 +60,7 @@ Only when the task needs **graph structure** or **related-but-unlinked** notes, | ------ | ------- | ------------- | | Reading / metadata only | `get` | `--meta` (header, no body) or `--head N` (first N chunks) — full `get` only on direct demand | | What links to a note | `backlinks` | exactly the slug | +| What a note references / embeds | `refs` | kind filters; `--include-missing` for broken links | | What's graph-neighbour | `connections` | `--hops 1–2` (exponential blow-up beyond) | | Meaning-related but NOT linked | `hidden` | `--deep` for section-level matches | | Related **including** linked | `hidden` | `--include-linked` | @@ -73,9 +74,13 @@ Only when the task needs **graph structure** or **related-but-unlinked** notes, Semantic search returns zero results or nothing above `--min-score`, so `search` automatically falls back to a token scan over titles/paths/tags/text. Rows arrive `"lexical": true`, `score: 0`; the header prints `Lexical Search (no semantic matches)`. So a short word like `Lecture` can still hit. When even that returns nothing, lengthen the query into a descriptive phrase or switch to a `tags` query if the word is a heading/tag keyword. No fallback for `boosted` or `hidden`. +### Refs: what a note references + +`refs` lists the note's attachments (images, PDFs, other) and external http(s) links, in first-occurrence order, read fresh from the note file on disk — no index staleness. It does NOT list links to other notes (that is `backlinks`/`connections`). Broken references are hidden by default; `--include-missing` surfaces them as `"missing": true`. External links are never missing and never touched over the network. No kind flags = every kind; the filters are pure kind selectors, no scores. + ## Slug discipline -Slugs are the handle; titles are not. For graph and `get` commands, pass the exact `note_slug` returned by a prior `search`/`tags` — never a bare title, titles are ambiguous. Since the deterministic-resolution fix, a missing note is an **error** (`note not found: "" ...`), not a silently guessed phantom slug. A "no indexed chunks" / "note not found" failure is normally a breadth-resolution problem, not a missing note. Slugs also go stale mid-conversation on schedule (cron re-ingest): if a slug that worked earlier now 404s, re-resolve via `search` before retrying. +Slugs are the handle; titles are not. For graph, `get`, and `refs` commands, pass the exact `note_slug` returned by a prior `search`/`tags` — never a bare title, titles are ambiguous. Since the deterministic-resolution fix, a missing note is an **error** (`note not found: "" ...`), not a silently guessed phantom slug. A "no indexed chunks" / "note not found" failure is normally a breadth-resolution problem, not a missing note. Slugs also go stale mid-conversation on schedule (cron re-ingest): if a slug that worked earlier now 404s, re-resolve via `search` before retrying. ## Tag discovery diff --git a/.agents/skills/notebrain/references/example.md b/.agents/skills/notebrain/references/example.md index 17fd2e0..2ce80cf 100644 --- a/.agents/skills/notebrain/references/example.md +++ b/.agents/skills/notebrain/references/example.md @@ -24,6 +24,8 @@ Quick reference: the major scenarios with the proven command sequence. Pair with | 14 | Metadata-only extraction | `--jsonpath`, `--format tsv`, `--show-file-path=false` (cuts ~40–50% of tokens) | | 15 | Context vs full `get` | context: `--context-window 1 --include-text`; full note only on explicit demand: `get ""` | | 16 | Stale-index recovery | a slug that 404s mid-conversation → re-resolve: `search "" --limit 3 --jsonpath="$.results[*].note_slug"` | +| 17 | Reference inventory | `notebrain refs "<slug>" --format json` (all kinds); kind filters: `--images` / `--pdf` / `--other` / `--external-links` | +| 18 | Broken-link audit | `notebrain refs "<slug>" --include-missing --format tsv` → rows with `missing` = `true` are broken; omit `--include-missing` to see only existing files | ## Semantics (verified) @@ -32,6 +34,7 @@ Quick reference: the major scenarios with the proven command sequence. Pair with - **`--section` is exact-match**: it compares against the stored `heading_path` string verbatim. Partial or parent paths return 0 results silently — copy the full `heading_path` from a search result. - **`--jsonpath`**: dotted paths, `[*]`, and `[0]` only — no jq-style pipe expressions, filters, or object construction. Multi-field extraction → `--format tsv` or two `--jsonpath` calls. - **Config overrides defaults**: `~/.notebrain/config/config.toml` can enable `include-text`/`context-window` (and set `min-score`/`limit`/`top-k`) — output then carries `text`/`context` even without flags. Pass `--include-text=false`/`--context-window=0` explicitly for lean output. +- **`refs` reads the file, not the index**: results come from a fresh parse of the note on disk — never stale, but only reflect what the current file contains. Order is first occurrence in the note; rows dedupe by resolved path (or exact URL). ## Pitfalls (verified) @@ -41,6 +44,7 @@ Quick reference: the major scenarios with the proven command sequence. Pair with - **Weak matches**: add `--min-score 0.3` (or `0.5` for precision); results below ~0.30 are noise. Note: config may already set a `min-score` floor, so low-score results can be absent by design. - **`get`**: `--meta` (header only: title, path, tags, chunk count) or `--head N` (first N chunks, `Chunks` still shows the total) cover most needs for cheap reads — reach for the full note only on demand. For metadata see also scenarios 3/14. - **Stale index**: scheduled re-ingest can invalidate cached slugs mid-conversation; re-verify via `search` before `--deep`/`backlinks` after any 404. +- **`refs` scope**: attachments (image/pdf/other) and external http(s) links only — links to other notes never appear (use `backlinks`/`connections`). So `refs --include-missing` catches broken attachments, not broken `[[wikilinks]]` to notes. `refs` is markdown-notes only; PDF extractions error out. ## Phrase → Scenario Map @@ -56,4 +60,6 @@ Quick reference: the major scenarios with the proven command sequence. Pair with | "unlinked / hidden concepts near Y" | 12 | | "concepts about X around note Y" | 13 | | "everything on topic X" | 5 or 6 | +| "what images / attachments does this note use" | 17 | +| "are any links / attachments broken" | 18 | | "why did that search return nothing" | 9 | diff --git a/.agents/skills/notebrain/references/flags.md b/.agents/skills/notebrain/references/flags.md index 02ae9a7..166f5c6 100644 --- a/.agents/skills/notebrain/references/flags.md +++ b/.agents/skills/notebrain/references/flags.md @@ -76,9 +76,21 @@ These flags are available only on the commands listed. Takes a single positional argument: `<slug>` (note slug, title, or file path — auto-resolved). Without flags, `get` returns the full reconstructed note; in text format, the header block prints a `Tags:` line (rendered as `#`-chips) — a lightweight way to read a note's tags without JSON. `--meta`/`--head` are mutually independent modes; `--head 0` means full note. +### `refs` + +| Flag | Purpose | Default | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------- | ------- | +| `--images` | Include image attachments (png, jpg, gif, svg, webp, …). | `false` | +| `--pdf` | Include PDF attachments. | `false` | +| `--other` | Include other attachments (video, audio, archives, office docs). | `false` | +| `--external-links` | Include external http(s) website links. | `false` | +| `--include-missing` | Include references whose file is missing from the vault (broken links). Hidden by default. | `false` | + +Takes a single positional argument: `<note>` (note slug, title, or file path — auto-resolved, markdown notes only; PDF extractions error out). No kind flags = every kind. `refs` reads the note file fresh from disk, so results never go stale — but they cover only what the current file contains. External links are never `missing` and are never contacted over the network. Note: `refs` lists attachments and external links only — links to other notes are not included (use `backlinks`/`connections`). + ## Global Flags (Available on Subcommands) -These flags work on `search`, `backlinks`, `connections`, `hidden`, `tags`, `boosted`, `get`, and `stats`. +These flags work on `search`, `backlinks`, `connections`, `hidden`, `tags`, `boosted`, `get`, `refs`, and `stats`. ### Output Format & Extraction diff --git a/.agents/skills/notebrain/references/schema.md b/.agents/skills/notebrain/references/schema.md index 239193d..0c5d392 100644 --- a/.agents/skills/notebrain/references/schema.md +++ b/.agents/skills/notebrain/references/schema.md @@ -149,6 +149,59 @@ Tag matching and normalization semantics: see `tags` in [flags.md](flags.md). The `note` object shape is the same for the `get` modes: default returns the full text, `--meta` returns the header with `text` empty (and `chunks` still the total), and `--head N` returns the first N chunks while `chunks` still reports the full total. For metadata-only lookups prefer `get "<slug>" --meta` (or `get --format text` for the compact `Tags:` header line) over a full fetch. +### `refs` + +`refs` uses its own envelope — a `refs` array (not `results`): + +`notebrain refs "architecture/event-driven-systems" --format=json` + +```json +{ + "command": "refs", + "note_slug": "architecture/event-driven-systems", + "title": "Event Driven Systems", + "total": 3, + "refs": [ + { + "path": "/home/user/vault/Attachments/eda-diagram.png", + "relative_path": "Attachments/eda-diagram.png", + "kind": "image", + "missing": false + }, + { + "path": "https://martinfowler.com/articles/eda.html", + "kind": "external-links", + "missing": false + } + ] +} +``` + +| Field | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `note_slug` | URL-safe unique identifier of the note queried. | +| `title` | Note title. | +| `total` | Number of rows in `refs`. | +| `refs[].path` | Absolute vault path for attachments; the full URL for external links. | +| `refs[].relative_path` | Vault-relative path (`/`-separated). Omitted for external links. | +| `refs[].kind` | `image`, `pdf`, `other`, or `external-links`. | +| `refs[].missing`| `true` when the file is not on disk (only visible with `--include-missing`). External links always carry `false` — they are never missing and never contacted over the network. | + +Rows are deduped by resolved path (or exact URL) in first-occurrence order. No kind flags = every kind. + +TSV shape — header `path<TAB>kind<TAB>missing<TAB>relative_path`; the `missing` cell is empty for external links: + +<!-- markdownlint-disable MD010 --> +```text +path kind missing relative_path +/home/user/vault/Attachments/eda-diagram.png image false Attachments/eda-diagram.png +/home/user/vault/broken.png image true broken.png +https://martinfowler.com/articles/eda.html external-links +``` +<!-- markdownlint-enable MD010 --> + +Text format prints one row per reference: `[kind] path (missing)` — e.g. `[image] /home/user/vault/broken.png (missing)`. Empty result prints `No references found`. + ### TSV Format `notebrain backlinks "architecture/event-driven-systems" --format=tsv` From 75f451eab83d492ef0078b7cbc287aa797befae2 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 03:59:19 +0530 Subject: [PATCH 08/18] chore: ignore the plans directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b74c5dc..26db918 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ graphify-out .graphifyignore .agents/skills/notebrain-workspace +.agents/plans/ From 4b8823e8268990efac9d0baafc75025b7e1788ba Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 04:01:00 +0530 Subject: [PATCH 09/18] docs(plan): track Plan.md and mark refs tasks complete - un-ignore the plans directory so Plan.md is tracked - tick tasks 0-7 for the refs feature --- .agents/plans/Plan.md | 157 ++++++++++++++++++++++++++++++++++++++++++ .gitignore | 1 - 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 .agents/plans/Plan.md diff --git a/.agents/plans/Plan.md b/.agents/plans/Plan.md new file mode 100644 index 0000000..e9a33f1 --- /dev/null +++ b/.agents/plans/Plan.md @@ -0,0 +1,157 @@ +# Plan: `notebrain refs` command + +## Goal + +Add a new `notebrain refs <note-slug/note-name>` command that lists the **direct references** of a markdown note: local attachment file paths (images, PDFs, archives, …) **and external website links** (URLs), so AI agents can fetch the actual files and sources instead of only note text. The command resolves the note exactly like sibling commands (slug, title, filename, or partial path) and prints absolute file paths / URLs in text/JSON/TSV, filterable with `--images`, `--pdf`, `--other`, `--external-links`. + +Outcome: an agent can run `notebrain refs kubernetes-notes --images --format=json` and get `["/vault/assets/arch.png", ...]`, or `notebrain refs kubernetes-notes --external-links --format=json` and get the note's website references. + +## Current State + +**Note resolution already exists and is the right seam.** + +- `store.ResolveNoteSlug` resolves exact slug, title, filename, or partial path in one metadata scan (`internal/store/query.go:1477`). +- `GetNoteMeta` returns `NoteContent{NoteSlug, Title, FilePath, Tags, Chunks}` (`internal/store/query.go:1624`, struct at `query.go:118`) — `FilePath` is vault-relative. +- `storeAPI` in `cmd/helpers.go:16` already exposes `GetNoteMeta`; the fake store used by cmd tests already implements it (`cmd/testhelpers_test.go:102`). **No store changes needed.** + +**The index does NOT track attachments — the command must parse the note file fresh from the vault.** + +- `metadataTransformer` deliberately skips attachment embeds (`![[img.png]]`) from the link set (`internal/parser/ast.go:105-110`), and `upsertLinks` drops attachment links when `SkipAttachments` is set (`internal/store/upsert.go:200-201`). `nb_links` therefore cannot answer "what does this note reference". + +**Classification primitives already exist in `internal/parser/parser.go`.** + +- `attachmentExts` allowlist (parser.go:64) — images, video, audio, canvas, archives, office docs. PDFs are deliberately excluded there because they are ingested as notes (parser.go:62). +- `imageExts` (parser.go:98), `IsAttachmentLink` (parser.go:76, strips `|alias` and `#anchor`), private `attachmentKind` (parser.go:108). +- The goldmark AST walk pattern to copy is `metadataTransformer` (`ast.go:89-119`); `mdParser` is a shared package-level goldmark instance with the wikilink extension (`ast.go:52-70`). + +**External-link nodes are already reachable in the same AST walk.** + +- `mdParser` enables `extension.GFM` (ast.go:60), which includes the Linkify extension (`vendor/github.com/yuin/goldmark/extension/gfm.go:14`) producing `*ast.AutoLink` nodes for bare `http://`/`https://`/`ftp://` and `www.` URLs (`vendor/github.com/yuin/goldmark/extension/linkify.go:14-16`); the node stores `AutoLinkType` (URL vs Email) and `Protocol` separately from the URL text (`vendor/github.com/yuin/goldmark/ast/inline.go:557-567`), which the renderer assembles as `protocol + "://" + text` (`vendor/github.com/yuin/goldmark/renderer/html/html.go:506-515`). +- Markdown links and image embeds are `*ast.Link` / `*ast.Image` nodes with a raw `Destination` (`vendor/github.com/yuin/goldmark/ast/inline.go:447-527`); `handleInlineNode` already renders these kinds (`internal/parser/ast.go:606`). +- Obsidian renders `[[https://…]]` wikilinks as external links, so URL-prefixed wikilink targets belong in the same collection. + +**Vault evidence (user's vault, `/home/nimendra/Documents/Second Brain 2.0`, 824 notes)** — the feature is scoped to observed reality: + +- 1,683 wiki-link image embeds vs 9 markdown image embeds and 4 markdown links to local files → wiki syntax dominates, but markdown links exist and are in scope (user decision). +- Markdown destinations are percent-encoded in practice (`Router%20Modes.webp`, `1101-Practical-Router-Configuration.md`) → resolution must percent-decode. +- 108 wiki-links to PDFs → `--pdf` filter earns its keep. +- 2,269 lines with http(s) URLs → external-link listing is a real need. +- `attachmentFolderPath = "99.Storage-Shed/Attachments"` in `.obsidian/app.json` → centralized attachment folder confirms the resolution order (note folder → vault root → attachment folder). + +**Path resolution rules (Obsidian semantics).** + +- Wiki-link targets containing a folder path start at the vault root: `[[Projects/Three laws of motion]]` (<https://obsidian.md/help/links>). Links to non-markdown files require the extension: `[[Figure 1.png]]` (same source). +- Bare wiki names (no `/`) resolve by searching the note's own folder, then the vault root, then the configured attachment folder; we verify each candidate with `os.Stat`, so the search order only decides which existing file wins. `./` prefix resolves relative to the note's folder. +- **Markdown links** (`[doc](file.pdf)`, `![alt](img.png)`) resolve **relative to the note's folder** (Obsidian/CommonMark semantics — unlike wiki links), with percent-decoding; fragment (`#page=3`) stripped. +- The attachment folder is `attachmentFolderPath` in `.obsidian/app.json`, already parsed by `ingest.LoadExcludedPaths` (`internal/ingest/ignore.go:15-36`) — but only the merged ignore-filters list is returned, not the folder itself. A small refactor/exposure is needed. + +**Command conventions to follow.** + +- Registration: `CLI` struct in `cmd/cli.go:89-122`; flag groups, `completion-predictor:"note-slug"` for the positional arg (as in `TagsCmd`, `cmd/tags.go:23-34`). No name collision: `refs` is distinct from all existing commands (`ingest`, `search`, `backlinks`, `connections`, `hidden`, `tags`, `boosted`, `stats`, `get`, `reset`, `doctor`, `init`, `version`, `completion`). +- Missing vault path → `UsageError` with the exact message pattern from `cmd/ingest.go:47-49`. +- Output follows `printTagsFormattedToWriter` (`cmd/print.go:444-476`): JSON envelope object + `--jsonpath` support via `printJSONPathResultToWriter` (print.go:392), TSV with header row, simple text lines. Formats: `formatText`/`formatJSON`/`formatTSV`. + +**Git workflow (branch-per-feature convention).** + +- The repo develops features on `feat/*` branches off `master` (evidence: `feat/deep-hidden-connections`, `feat/shell-completion`, `feat/pdf-ocr-support`, `feat/goldmark-extensions`, `feat/cli-ux-improvements` all exist locally/remotely), with Conventional Commits (AGENTS.md) and Lefthook pre-commit hooks. +- This feature gets its own branch: `feat/refs-command`. `master` stays untouched until the user reviews and approves the merge. + +**Docs that must stay in sync** (last 4 commits were skill rewrites — the skill is the agent-facing contract): + +- `README.md` (Features bullet list at line 26-41; Quick Start chaining example at step 6) +- `wiki/Commands.md` (command sections; insert after `get`, ~line 256) +- `.agents/skills/notebrain/SKILL.md` (retrieval ladder, section "Retrieval ladder" at line 32) +- `.agents/skills/notebrain/references/flags.md` (per-command flag tables) +- `.agents/skills/notebrain/references/example.md` (worked scenarios) +- `.agents/skills/notebrain/references/schema.md` (JSON envelope + example outputs; the `refs` envelope is a new shape) +- `.agents/AGENTS.md` (project structure tree + CLI testing standards) +- `CHANGELOG.md` (v2.12.0 is latest; no `Unreleased` section yet) + +**Skill development environment (for the skill update task).** + +- The `skill-creator` skill (`/home/nimendra/.agents/skills/skill-creator/`) defines the create → evaluate → iterate loop: snapshot baseline, with-skill vs baseline runs, grading via `agents/grader.md`, `scripts/aggregate_benchmark.py`, and `eval-viewer/generate_review.py`. The `writing-for-agents` skill (`/home/nimendra/.agents/skills/writing-for-agents/`, incl. `SKILL-MECHANICS.md`) defines the craft rules for skill text (progressive disclosure, leading words, positive phrasing, single source of truth, pruning). +- A workspace already exists: `.agents/skills/notebrain-workspace/` with `evals/evals.json` (3 evals: kubernetes-reconciliation, kubernetes-architecture-graph, message-broker-backpressure), a `skill-snapshot/`, and `iteration-1/2/3` — iteration-3 is the latest (with `benchmark.json`/`benchmark.md` and `review.html`). The refs skill update runs as **iteration-4**. + +## Decisions + +1. **Parse the note file fresh; resolve the note via the index.** The index skips attachments by design (ast.go:105-110), so file parsing is required; the store is still the best slug/title resolver. Consequence: the command requires `--vault-path` and requires the note to be indexed (consistent with every sibling command). Vault-only resolution without an index is out of scope. +2. **Extraction lives in `internal/parser` as a new exported AST walker**, not regex in `cmd/`: reuses `mdParser`, correctly ignores links inside code fences, and handles aliases/anchors with existing helpers. Parser owns markdown semantics. +3. **Local attachments: wiki links AND markdown links.** Wiki syntax (`[[…]]` / `![[…]]`) per decision 7's resolution rules; markdown links and image embeds (`*ast.Link` / `*ast.Image`) whose destination is **not** an http(s) URL are also local attachment candidates, resolved note-folder-relative with percent-decoding and a vault-traversal guard. +4. **External links: three syntaxes, http/https only.** (a) markdown links and image embeds (`*ast.Link` / `*ast.Image`) whose `Destination` starts with `http://` or `https://`; (b) bare/angle autolinks (`*ast.AutoLink` with `AutoLinkType == AutoLinkURL`, `Protocol` `http`/`https`); (c) wikilink targets with `http://`/`https://` prefix. Excluded: email autolinks (`AutoLinkEmail`), `ftp://`, `mailto:`, `obsidian://` and other schemes. URL kept exactly as written; dedupe by exact string (no normalization). +5. **Kind taxonomy:** `image` (imageExts incl. `.heic`), `pdf` (`.pdf` — included here even though ingestion treats PDFs as notes), `other` (everything else in `attachmentExts`), `external-links` (URLs). Both embeds and plain links to attachments count; unknown extensions are notes, not attachments (allowlist already enforces this). +6. **Filters:** `--images`, `--pdf`, `--other`, `--external-links` — boolean, OR-combined; no flag = all kinds (files + URLs mixed). Audio/video stay under `--other` (no split in v1). +7. **Path resolution order:** wiki target contains `/` → vault-root candidate only (plus `./` → note-folder candidate); bare wiki target → note's folder, vault root, then `attachmentFolderPath` (first `os.Stat` hit wins; no hit → missing). Markdown destination → note's folder only, percent-decoded (`url.PathUnescape`), fragment stripped, and the resolved path must stay inside the vault (reject `..` escapes via `filepath.Rel`). External URLs: no filesystem resolution. +8. **Output:** absolute paths (agents must open them), with vault-relative `relative_path` alongside; external rows carry the URL in `path` with `relative_path` omitted and `missing` always `false` (see decision 11). JSON envelope `{"command":"refs","note_slug":…,"title":…,"total":N,"refs":[{"path":…,"relative_path":…,"kind":…,"missing":bool}]}`; TSV header `path\tkind\tmissing\trelative_path` (missing column empty for external); text one line per entry with `[kind]` label (`[external-links]` for URLs) and `(missing)` marker; empty result prints "No references found" in text mode only. `--jsonpath` works on the envelope. **Broken links are hidden by default**; `--include-missing` lists them marked `missing: true` — a quiet list beats noise, the opt-in flag keeps the information available. +9. **Deterministic ordering:** first-occurrence document order; dedupe in two layers — parser dedupes by cleaned target string, cmd dedupes by resolved absolute path (catches cross-syntax dupes like `[[a.png]]` + `[x](a.png)` pointing at the same file). URLs dedupe by exact string. +10. **PDF note input** (`file_path` ends in `.pdf`) → clear error ("is a PDF; refs are listed for markdown notes"). Indexed note whose file is gone from disk → error hinting `--vault-path` mismatch, not silent empty output. +11. **Offline by design: no network checks.** External links are never verified (no HTTP HEAD/GET); `missing` is `false` for them and `--include-missing` does not apply to them. Reachability checking would violate the tool's offline-first contract. +12. **Naming (settled by grilling):** command `refs`, JSON array `refs`, struct `RefsCmd` in `cmd/refs.go`, flag group `refs`, kind `external-links` (flag `--external-links`). "Refs" properly covers both local attachments and external URLs; it collides with nothing in the existing command tree. + +## Scope + +In scope: parser extractor (attachments via wiki + markdown syntax, external links) + tests, attachment-folder exposure + tests, `refs` command + tests, CLI registration, output formats, docs (README, wiki, skill ×4 files via the skill-creator loop, AGENTS.md, CHANGELOG). + +Out of scope: frontmatter references (wiki/markdown syntax only), audio/video sub-filters, `obsidian://` and other non-http(s) URI schemes, email addresses, URL reachability checks (network), transitive (nested-note) reference listing, vault-only resolution without an index, references inside indexed PDFs. + +## Tasks + +- [x] **Task 0: branch — create `feat/refs-command`.** + From clean master: `git switch master && git pull && git switch -c feat/refs-command`. Commit after each task with a Conventional Commit scoped to its package (`feat(parser): …`, `feat(ingest): …`, `feat(cmd): …`, `docs(wiki): …`, `docs(skill): …`). Push the branch (`git push -u origin feat/refs-command`) once Task 1 lands so work is never local-only. Do NOT merge to master — merging is the user's call after review. + (**Seam:** n/a; **Files:** none (git only); **Verify:** `git branch --show-current` = `feat/refs-command`; `git status` clean before switching; `git log --oneline` shows the feature commits on the branch only.) + +- [x] **Task 1: parser — extract refs (local attachments + external links) from a note body.** + New file `internal/parser/attachments.go`: `type AttachmentKind string` (`KindImage`, `KindPDF`, `KindOther`, `KindExternalLinks`); `type AttachmentRef struct { Target string; Kind AttachmentKind }` (Target = cleaned — alias/anchor stripped for wiki refs, raw destination for markdown refs, full URL for external); `type ExtractedRefs struct { Attachments []AttachmentRef; External []string }`; `func ExtractReferences(body string) ExtractedRefs` — walks `mdParser`'s AST like `metadataTransformer` (ast.go:89-119), collecting: + - attachments from wiki nodes: every `*wikilink.Node` whose target classifies as attachment (image/other via new classification that includes `.pdf`); + - attachments from markdown nodes: `*ast.Link` and `*ast.Image` whose `Destination` is **not** http(s) and classifies as attachment (percent-encoding and fragments left raw here — resolution is the cmd layer's job); + - external: `*ast.Link` and `*ast.Image` nodes with `Destination` starting `http://`/`https://`; `*ast.AutoLink` nodes with `AutoLinkType == ast.AutoLinkURL` and `Protocol` `http`/`https` (assemble `string(n.Protocol)+"://"+string(n.Text(src))` — protocol and text are stored separately, cf. renderer `html.go:506-515`); and `*wikilink.Node` targets with `http://`/`https://` prefix. Email autolinks, `ftp://`, `mailto:` and other schemes are skipped. + Dedupe by Target (attachments) / exact URL (external), first-occurrence order. Reuses `imageExts`; adds `.pdf` handling alongside `attachmentExts`. `IsAttachmentLink`/`attachmentKind` untouched (ingestion semantics unchanged). + (**Seam:** `internal/parser/parser_test.go` table tests; **Files:** `internal/parser/attachments.go`, `internal/parser/attachments_test.go`; **Verify:** `go test -count=1 ./internal/parser/` — wiki cases: `![[img.png|200]]`, `[[doc.pdf]]`, `[[img.png|alt]]`, `[[img.png#anchor]]`, `![[sub/img.png]]`, `[[./local.png]]`, code fence containing `![[x.png]]` (must not match), duplicate embeds (dedupe), unknown ext `[[archive.xyz]]` (not an attachment), `[[Note 1.2.3]]` (not an attachment), `.PNG` case-insensitivity; markdown cases: `![alt](img.png)`, `[doc](sub/file.pdf)`, `![alt](Router%20Modes.webp)` (raw destination kept), `[x](../up.pdf)`, `[x](STP.pdf#page=5)`, `[text](https://example.com)` (NOT an attachment — external), `[rel](../other-note)` (no ext, not an attachment), `#anchor`-only links (not an attachment), code fence containing `![x](img.png)` (must not match); external cases: `[text](https://example.com/a)`, `![alt](https://example.com/i.png)`, bare `https://example.com`, `<https://example.com>`, `www.example.com` (protocol `http`), `[[https://example.com]]`; excluded: `[mail](mailto:a@b.c)`, `[ftp](ftp://x.y/z)`, email autolink `a@b.c`, code fence containing `https://example.com` (must not match).) + +- [x] **Task 2: ingest — expose the Obsidian attachment folder.** + In `internal/ingest/ignore.go`, add `func LoadAttachmentFolderPath(vaultPath string) string` reading `.obsidian/app.json` (reuse the `ObsidianAppConfig` struct); refactor `LoadExcludedPaths` to share the read (keep its return value identical — non-regression guardrail per AGENTS.md). Returns `""` when absent/unreadable. + (**Seam:** `internal/ingest/ignore_test.go`; **Files:** `internal/ingest/ignore.go`, `internal/ingest/ignore_test.go`; **Verify:** `go test -count=1 ./internal/ingest/` — folder read, absent file → `""`, existing `LoadExcludedPaths` tests still pass.) + +- [x] **Task 3: cmd — `refs` command.** + New file `cmd/refs.go`: `RefsCmd{ Note string (arg,`completion-predictor:"note-slug"`); Images, PDF, Other, ExternalLinks, IncludeMissing bool (group "refs") }`, help text "List a note's attachments and external links". `Run`: empty vault path → `UsageError` (message pattern of ingest.go:47-49); empty note → `UsageError`; `st.GetNoteMeta(ctx, c.Note)`; PDF note → error; read `filepath.Join(globals.VaultPath, meta.FilePath)` (missing file → error hinting `--vault-path`); `parser.ExtractReferences`; resolve each attachment ref (decision 7): wiki refs via candidate search + `os.Stat`; markdown refs via `url.PathUnescape(destination)`, fragment strip, `filepath.Join(noteDir, decoded)`, traversal guard (`filepath.Rel` must not escape vault); external URLs become `kind=external-links` rows directly (no stat, `missing` always false); dedupe by resolved absolute path; drop missing rows unless `--include-missing` (external rows unaffected); filter by kind flags (OR — `--external-links` selects only URL rows); sort stays first-occurrence; print via Task 4. Register `Refs RefsCmd` in `CLI` struct (cli.go:89-122). + (**Seam:** `cmd` tests with the existing `fakeStore` (testhelpers_test.go:102 sets `f.noteMeta`) + real temp vault dirs via `t.TempDir()`; **Files:** `cmd/refs.go`, `cmd/refs_test.go`, `cmd/cli.go`; **Verify:** `go test -count=1 ./cmd/` — resolution from fake `noteMeta.FilePath`, bare wiki target found in note folder vs vault root vs attachment folder, wiki path target from vault root, `./` from note folder, markdown target from note folder (percent-decoded, e.g. `Router%20Modes.webp` → `Router Modes.webp`), markdown `../` traversal escaping vault rejected, cross-syntax dedupe (`[[a.png]]` + `[x](a.png)` → one row), missing file hidden by default, `--include-missing` shows it marked, filters (`--images` only, `--pdf` only, `--external-links` only, combined OR), external rows skip stat and always report `missing: false`, PDF-note error, empty-vault-path UsageError, note-not-found error passthrough.) + +- [x] **Task 4: cmd — output formatting.** + In `cmd/refs.go` (or `cmd/print.go`): `printRefsFormattedToWriter(w io.Writer, env, globals)` following `printTagsFormattedToWriter` (print.go:444-476) — text lines with `[kind]`/`[external-links]`/`(missing)` markers (empty → "No references found" line in text mode only), TSV header `path\tkind\tmissing\trelative_path` (external rows: empty missing and relative_path columns), JSON envelope (decision 8; external rows omit `relative_path`), `--jsonpath` via `printJSONPathResultToWriter`. + (**Seam:** writer-based tests like `printTagsFormattedToWriter`; **Files:** `cmd/refs.go`, `cmd/refs_test.go`; **Verify:** golden-ish assertions per format + `--jsonpath='$.refs[*].path'`.) + +- [x] **Task 5: docs — user-facing docs (README, wiki, AGENTS.md, CHANGELOG).** + - `README.md`: new Features bullet after "Full Note Retrieval" (e.g. "**Reference Listing**: List a note's local attachments and external website links with `notebrain refs`, filterable by kind — images, PDFs, or URLs."); extend the Quick Start chaining example (step 6) with a `refs` snippet (e.g. `notebrain refs "$SLUG" --images --jsonpath='$.refs[*].path'`). + - `wiki/Commands.md`: new `### refs` section after `get` (~line 256): usage, argument, flags, examples (`--images --format=json`, `--pdf --format=tsv`, `--external-links --format=json`), JSON shape. + - `.agents/AGENTS.md`: add `cmd/refs.go` to the structure tree; add one line to CLI Testing & Flag Standards (positional `<note>`, `--images`/`--pdf`/`--external-links` filters). + - `CHANGELOG.md`: add `## [Unreleased]` → `### Added` entry for the `refs` command. + (**Seam:** none (docs); **Files:** the four listed; **Verify:** `grep -n refs` across the four files; no stale `attachments` command references and no stale flag names.) + +- [x] **Task 6: skill — update `notebrain-assistant` via the skill-creator loop.** + Update `.agents/skills/notebrain/` (SKILL.md + references/flags.md, example.md, schema.md) for the `refs` command, following the `skill-creator` skill's "Improving an existing skill" workflow (`/home/nimendra/.agents/skills/skill-creator/SKILL.md`) and the `writing-for-agents` craft rules (`/home/nimendra/.agents/skills/writing-for-agents/SKILL.md`; also read `SKILL-MECHANICS.md` before editing the frontmatter/description). Steps: + 1. Snapshot the current skill first: `cp -r .agents/skills/notebrain .agents/skills/notebrain-workspace/iteration-4/skill-snapshot/`. + 2. Edit the four files: SKILL.md **description** gains attachment/ref phrasing so queries like "list/fetch the images, files, attachments, or external links of a note" trigger it; the retrieval ladder gains a `refs` step (absolute paths, broken links hidden unless `--include-missing`, `--external-links` for URLs); flags.md gains the `### refs` flag table; example.md gains the two worked scenarios; schema.md gains the `refs` envelope + TSV columns. Craft rules: imperative voice, progressive disclosure (keep SKILL.md lean, references on demand), positive phrasing (no negations), single source of truth (verify every documented command against the real binary before writing it), leading words only where they earn their place. + 3. Extend `evals/evals.json` with refs-focused test prompts (e.g. "what images does my kubernetes note reference?", "fetch the attachments of kubernetes-architecture", "which external websites does the message broker note link to?") with assertions; keep the existing three evals unchanged. + 4. Run the eval loop as iteration-4 in `.agents/skills/notebrain-workspace/iteration-4/`: spawn with-skill vs old-skill-snapshot baseline runs in parallel per eval; capture `timing.json` per run; grade via `agents/grader.md`; aggregate via `python -m scripts.aggregate_benchmark <workspace>/iteration-4 --skill-name notebrain-assistant` (scripts in `/home/nimendra/.agents/skills/skill-creator/scripts/`); generate the review viewer headlessly with `eval-viewer/generate_review.py --static <output_path>` (no display in this environment; iteration-3 produced `eval_review_notebrain-assistant.html` the same way). + 5. Present results to the user, apply feedback, iterate (iteration-5 only if feedback demands). + (**Seam:** eval workspace `.agents/skills/notebrain-workspace/`; **Files:** `.agents/skills/notebrain/{SKILL.md,references/flags.md,references/example.md,references/schema.md}` + `notebrain-workspace/iteration-4/`; **Verify:** `benchmark.md` shows the refs evals passing with-skill ≥ baseline; description triggers on attachment/ref phrasing.) + +- [x] **Task 7: full verification.** + `make test` (all packages), `make lint`, then a manual end-to-end against a real vault: `go build -o notebrain .` + `./notebrain refs "<note>"`, `--images --format=json`, `--jsonpath='$.refs[*].path'`, `--pdf` on a note linking `[[doc.pdf]]`, `--external-links` on a note with `[text](https://…)` links, a markdown-linked image (percent-encoded), and a deliberate broken link to confirm it is hidden by default and marked with `--include-missing`. + (**Seam:** n/a; **Verify:** `make test && make lint`; manual outputs match decision 8.) + +## Verification + +- All work lands on `feat/refs-command`; `master` has zero commits from this feature until the user approves the merge (`git log master..feat/refs-command --oneline` lists exactly the feature commits). +- `go test -count=1 ./...` green (parser, ingest, cmd — store untouched, but run full suite). +- `notebrain refs "Note Title"` lists absolute paths; `--images`/`--pdf`/`--other`/`--external-links` filter; combined flags union. +- `--external-links` returns the note's http/https URLs (markdown links, image embeds, bare autolinks, `[[https://…]]`), deduped, first-occurrence order; email/ftp/relative links never appear; URLs inside code fences never appear. +- Markdown-linked local files resolve note-folder-relative, percent-decoded, traversal-guarded; `[[a.png]]` + `[x](a.png)` dedupe to one row. +- Broken links hidden by default; `--include-missing` lists them with `missing: true`; external rows never appear missing; no network I/O happens for external rows. +- JSON envelope matches decision 8 (`"refs": [...]`); `--jsonpath` works; TSV has the header row; text output marks `(missing)` and `[external-links]`. +- Skill docs (SKILL.md, flags.md, example.md, schema.md), README.md, wiki/Commands.md, AGENTS.md, CHANGELOG.md all mention the `refs` command with matching flag names. +- Task 6's iteration-4 benchmark.md shows the refs evals passing with-skill ≥ baseline, and the user reviewed the viewer output. + +## Open Questions + +- None — the design tree is fully settled (grilling rounds 1-2 complete). +- Non-blocking follow-ups (not part of v1): `--audio`/`--video` sub-filters (trivial enum extension), frontmatter reference scanning (needs per-field mapping rules), PDF page-anchor passthrough (`"anchor": "page=3"` in JSON), vault-only resolution without an index, and skill description trigger optimization via `scripts/run_loop.py` (skill-creator's Description Optimization step — 20 trigger queries; only if the user wants the description tuned beyond the manual wording update in Task 6). diff --git a/.gitignore b/.gitignore index 26db918..b74c5dc 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,3 @@ graphify-out .graphifyignore .agents/skills/notebrain-workspace -.agents/plans/ From d12029c26e22184cd4f53088dfbae81f44ef75cd Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 04:03:48 +0530 Subject: [PATCH 10/18] refactor(cmd): reuse kindPDF constant in print output --- cmd/print.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/print.go b/cmd/print.go index 5c46ee8..35cecef 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -260,7 +260,7 @@ func printTextResults(w io.Writer, commandName, query string, queries []string, rank := rankStyle.Render(fmt.Sprintf("%d.", i+1)) displayTitle := r.Title - if r.FileType == "pdf" { + if r.FileType == kindPDF { displayTitle = pdfTagStyle.Render("[PDF] ") + displayTitle } From 5cf85bf0820a9b3bde949cea0a53ac37241ab342 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:05:21 +0530 Subject: [PATCH 11/18] refactor(store): extract FileType and parser block-kind constants - add FileTypeMD/FileTypePDF to internal/store (single source; ingest imports store, so no cycle) and use them in query.go where file_type filters compare against a bare "md" literal - drop ingest's private fileTypeMD/fileTypePDF mirrors in favor of store.FileTypeMD/FileTypePDF - add blockKindList/blockKindTable/blockKindBlockquote in the parser and replace the remaining string literals in ast.go block handling - comment the renderer's "attachment" label and leave its value unchanged (chunk text change would force a full re-ingest) --- internal/ingest/ingest.go | 11 ++++------- internal/ingest/ingest_pdf_test.go | 4 +++- internal/ingest/ingest_test.go | 2 +- internal/parser/ast.go | 21 ++++++++++++--------- internal/parser/parser.go | 3 +++ internal/store/query.go | 2 +- internal/store/store.go | 8 ++++++++ 7 files changed, 32 insertions(+), 19 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 632bce6..c471456 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -25,9 +25,6 @@ import ( ) const ( - fileTypeMD = "md" - fileTypePDF = "pdf" - // chunkSchemaVersion is bumped whenever chunk content semantics change so // that already-ingested files are re-ingested even if their bytes are // unchanged (e.g. the chunk-overlap duplication fix, has_code metadata). @@ -123,7 +120,7 @@ func (p *Pipeline) Run(ctx context.Context, vaultPath string, glob string) error skipPDF := false if p.EnablePDF { if p.LLMModel == "" { - slog.Warn("PDF ingestion requested via --enable-pdf, but no --llm-model was provided. Skipping PDF ingestion (previously ingested PDFs will be preserved). (hint: use --llm-model)") + slog.Warn("PDF ingestion requested via --with-pdf, but no --llm-model was provided. Skipping PDF ingestion (previously ingested PDFs will be preserved). (hint: use --llm-model)") skipPDF = true } else { slog.Info("initializing PDF extractor backend", "pool_size", p.workers) @@ -189,7 +186,7 @@ func (p *Pipeline) Run(ctx context.Context, vaultPath string, glob string) error staleSlugs := make([]string, 0, len(hashes)) for slug, meta := range hashes { if _, ok := validSlugs[slug]; !ok { - if skipPDF && meta.FileType == fileTypePDF { + if skipPDF && meta.FileType == store.FileTypePDF { continue } staleSlugs = append(staleSlugs, slug) @@ -383,7 +380,7 @@ func (p *Pipeline) processFile(ctx context.Context, vaultPath string, filePath s // Frontmatter title overrides the filename-derived title for markdown // notes only; PDF titles come from the file name (see processPdfFile). - return p.buildIngestData(ctx, relPath, slug, title, hash, fileTypeMD, string(content), true) + return p.buildIngestData(ctx, relPath, slug, title, hash, store.FileTypeMD, string(content), true) } // estimateTokens returns a conservative rough token count for English/mixed text. @@ -488,7 +485,7 @@ func (p *Pipeline) processPdfFile(ctx context.Context, vaultPath string, filePat // Empty LLM output is usually a transient conversion failure. Preserve the // previously indexed PDF instead of deleting it from the index; an empty // batch would remove its chunks permanently (see buildChunkRecords). - return p.buildIngestData(ctx, relPath, slug, title, hash, fileTypePDF, markdown, false) + return p.buildIngestData(ctx, relPath, slug, title, hash, store.FileTypePDF, markdown, false) } // noteIdentity reads, identifies, and hashes a note file. changed=false means diff --git a/internal/ingest/ingest_pdf_test.go b/internal/ingest/ingest_pdf_test.go index e4edd65..bea2606 100644 --- a/internal/ingest/ingest_pdf_test.go +++ b/internal/ingest/ingest_pdf_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/nmdra/notebrain-cli/v2/internal/store" ) type mockPDFBackend struct{} @@ -60,7 +62,7 @@ func TestProcessPdfFile(t *testing.T) { } chunk := res.ChunkRecords[0] - if chunk.FileType != fileTypePDF { + if chunk.FileType != store.FileTypePDF { t.Errorf("Expected FileType 'pdf', got %q", chunk.FileType) } if chunk.HeadingPath != "Dummy PDF Page" { diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 71df821..66ac23b 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -363,7 +363,7 @@ func TestPipeline_PDFFallbackPreservesPDFs(t *testing.T) { FilePath: "PDF Document.pdf", ChunkIndex: 0, ContentHash: "abcdef", - FileType: fileTypePDF, + FileType: store.FileTypePDF, Embedding: []float32{1.0, 0.0, 0.0}, }, }, diff --git a/internal/parser/ast.go b/internal/parser/ast.go index b97e636..ba86a02 100644 --- a/internal/parser/ast.go +++ b/internal/parser/ast.go @@ -79,9 +79,12 @@ var ( ) const ( - blockKindCode = "code" - blockKindParagraph = "paragraph" - blockKindTaskList = "task_list" + blockKindCode = "code" + blockKindParagraph = "paragraph" + blockKindTaskList = "task_list" + blockKindList = "list" + blockKindTable = "table" + blockKindBlockquote = "blockquote" ) type metadataTransformer struct{} @@ -336,7 +339,7 @@ func (e *sectionExtractor) processList(node *ast.List) (ast.WalkStatus, error) { if t == "" { return ast.WalkSkipChildren, nil } - kind := "list" + kind := blockKindList if isTask { kind = blockKindTaskList } @@ -354,7 +357,7 @@ func (e *sectionExtractor) processTable(node *extast.Table) (ast.WalkStatus, err return ast.WalkSkipChildren, nil } e.current.blocks = append(e.current.blocks, block{ - kind: "table", + kind: blockKindTable, text: t, }) return ast.WalkSkipChildren, nil @@ -367,7 +370,7 @@ func (e *sectionExtractor) processBlockquote(node *ast.Blockquote) (ast.WalkStat return ast.WalkSkipChildren, nil } e.current.blocks = append(e.current.blocks, block{ - kind: "blockquote", + kind: blockKindBlockquote, text: t, }) return ast.WalkSkipChildren, nil @@ -424,7 +427,7 @@ func buildChunks(sections []section, noteSlug string, maxRunes, overlapRunes int var codeInfos []codeBlockInfo for idx, b := range sec.blocks { if idx > 0 { - if b.kind == "paragraph" && sec.blocks[idx-1].kind == "paragraph" { + if b.kind == blockKindParagraph && sec.blocks[idx-1].kind == blockKindParagraph { prose.WriteByte(' ') } else { prose.WriteString("\n\n") @@ -436,10 +439,10 @@ func buildChunks(sections []section, noteSlug string, maxRunes, overlapRunes int codeIdx := len(codeInfos) codeInfos = append(codeInfos, codeBlockInfo{lang: b.language, code: b.codeText}) _, _ = fmt.Fprintf(&prose, "\x00CODE:%d:%s\x00", codeIdx, b.language) - case "table": + case blockKindTable: hasTable = true prose.WriteString(b.text) - case blockKindTaskList, "list": + case blockKindTaskList, blockKindList: if b.kind == blockKindTaskList { hasTask = true } diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 5412c04..7a4ff9f 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -113,6 +113,9 @@ func attachmentKind(target string) string { if _, ok := imageExts[ext]; ok { return "image" } + // "attachment" covers every non-image embed (audio, video, generic + // files). The label differs from refs' "other" kind but is baked into + // stored chunk text: changing it would force a full re-ingest. return "attachment" } diff --git a/internal/store/query.go b/internal/store/query.go index 96263e7..713f9ee 100644 --- a/internal/store/query.go +++ b/internal/store/query.go @@ -1079,7 +1079,7 @@ func (f *SearchFilter) Build() chroma.WhereFilter { filters = append(filters, chroma.EqBool("has_code", true)) } if !f.IncludePDF { - filters = append(filters, chroma.EqString("file_type", "md")) + filters = append(filters, chroma.EqString("file_type", FileTypeMD)) } if f.ResolveTags && f.Tag != "" { filters = append(filters, TagWhereClause(f.Tag)) diff --git a/internal/store/store.go b/internal/store/store.go index e947048..1a91f69 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -14,6 +14,14 @@ const ( CollectionLinks = "nb_links" ) +// FileTypeMD and FileTypePDF are the file_type metadata values written on +// ingested chunks. Kept in store (not ingest) because store queries filter on +// them; ingest imports store, so there is no import cycle. +const ( + FileTypeMD = "md" + FileTypePDF = "pdf" +) + var defaultChunksMeta = map[string]any{ "hnsw:space": "cosine", "hnsw:search_ef": 50, // Lower value improves query speed From a13c39228a72017efedd4da6a11dfbee67465de4 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:06:24 +0530 Subject: [PATCH 12/18] fix(cmd): TSV/JSONPath output parity and ignored-format warnings - search TSV header slug -> note_slug; scores at 4dp like JSON - get TSV fields escaped via tsvEscape; get/stats apply --jsonpath to the command envelope so $.command works (get paths shift to $.note.*) - warn once on stderr when --format/--jsonpath are given to text-only commands (ingest, reset, doctor, doctor-probe, init, version, completion, suggest-notes); stdout stays clean - share the vault-path usage error between commands via one constant BREAKING CHANGE: get --jsonpath values like $.note_slug must move to $.note.note_slug. --- cmd/cli.go | 62 ++++++++++++++++++++++++++++++++------- cmd/output_parity_test.go | 50 +++++++++++++++++++++++++++++++ cmd/print.go | 6 ++-- cmd/print_test.go | 23 ++++++++++++++- cmd/stats.go | 17 ++++++----- cmd/testhelpers_test.go | 5 +++- 6 files changed, 139 insertions(+), 24 deletions(-) create mode 100644 cmd/output_parity_test.go diff --git a/cmd/cli.go b/cmd/cli.go index a4f1acb..5556262 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -30,17 +30,18 @@ type ChunkDisplayFlags struct { // Flag group keys. The order here determines the order of the titled // sections in --help output. const ( - groupGlobal = "global" - groupDisplay = "display" - groupIngest = "ingest" - groupSearch = "search" - groupConn = "connections" - groupHidden = "hidden" - groupBoosted = "boosted" - groupTags = "tags" - groupReset = "reset" - groupGet = "get" - groupRefs = "refs" + groupGlobal = "global" + groupDisplay = "display" + groupIngest = "ingest" + groupSearch = "search" + groupConn = "connections" + groupHidden = "hidden" + groupBoosted = "boosted" + groupTags = "tags" + groupReset = "reset" + groupGet = "get" + groupRefs = "refs" + groupCompletion = "completion" ) // helpGroups returns the titled flag groups shown in --help output. Flags in @@ -130,6 +131,10 @@ func ParseAndRun(ctx context.Context, version, commit, date string, defaultConfi // operational failures (1). type UsageError struct{ Err error } +// vaultPathUsageError is shared by every command that requires an explicit +// vault location; keep the wording in one place. +const vaultPathUsageError = "--vault-path flag or config file setting must be specified — run 'notebrain init' to create a config" + func (e *UsageError) Error() string { return e.Err.Error() } func (e *UsageError) Unwrap() error { return e.Err } @@ -186,6 +191,37 @@ func argsWantJSON(args []string) bool { return false } +// warnIgnoredOutputFlags tells the user (once, on stderr) that a text-only +// command does not honor --format/--jsonpath. stdout stays clean for +// machine consumers. +func warnIgnoredOutputFlags(w io.Writer, format, jsonpath, cmdName string) { + switch { + case format != formatText && jsonpath != "": + fmt.Fprintf(w, "warning: --format and --jsonpath are ignored by '%s' (output is textual only)\n", cmdName) + case format != formatText: + fmt.Fprintf(w, "warning: --format is ignored by '%s' (output is textual only)\n", cmdName) + case jsonpath != "": + fmt.Fprintf(w, "warning: --jsonpath is ignored by '%s' (output is textual only)\n", cmdName) + } +} + +// textOnlyCommands are the commands whose output is inherently textual and +// never honors --format/--jsonpath. +var textOnlyCommands = map[string]bool{ + groupIngest: true, + "reset": true, + "doctor": true, + "doctor-probe": true, + "init": true, + "version": true, + groupCompletion: true, + "suggest-notes": true, +} + +func isTextOnlyCommand(cmdName string) bool { + return textOnlyCommands[cmdName] +} + // validateLogLevel rejects log level values that are not one of the supported // severities. An empty value means "not set" and defers to the env var and // default. @@ -286,6 +322,10 @@ Examples: } } + if isTextOnlyCommand(ctxParser.Command()) { + warnIgnoredOutputFlags(os.Stderr, cli.Format, cli.JSONPath, ctxParser.Command()) + } + err = ctxParser.Run(&cli.Globals) if err != nil { if cli.Format == formatJSON { diff --git a/cmd/output_parity_test.go b/cmd/output_parity_test.go new file mode 100644 index 0000000..6f4fe94 --- /dev/null +++ b/cmd/output_parity_test.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "bytes" + "strings" + "testing" +) + +func TestWarnIgnoredOutputFlags(t *testing.T) { + t.Run("silent when text and no jsonpath", func(t *testing.T) { + var buf bytes.Buffer + warnIgnoredOutputFlags(&buf, formatText, "", "reset") + if buf.Len() != 0 { + t.Errorf("expected no warning, got %q", buf.String()) + } + }) + + t.Run("warns once for format", func(t *testing.T) { + var buf bytes.Buffer + warnIgnoredOutputFlags(&buf, formatJSON, "", "reset") + out := buf.String() + if !strings.Contains(out, "--format") || !strings.Contains(out, "reset") { + t.Errorf("expected --format warning for reset, got %q", out) + } + if strings.Contains(out, "--jsonpath") { + t.Errorf("must not mention --jsonpath, got %q", out) + } + }) + + t.Run("warns once for jsonpath", func(t *testing.T) { + var buf bytes.Buffer + warnIgnoredOutputFlags(&buf, formatText, "$.x", "ingest") + out := buf.String() + if !strings.Contains(out, "--jsonpath") || !strings.Contains(out, "ingest") { + t.Errorf("expected --jsonpath warning for ingest, got %q", out) + } + if strings.Contains(out, "--format") { + t.Errorf("must not mention --format, got %q", out) + } + }) + + t.Run("warns for both", func(t *testing.T) { + var buf bytes.Buffer + warnIgnoredOutputFlags(&buf, formatTSV, "$.x", "doctor") + out := buf.String() + if !strings.Contains(out, "--format") || !strings.Contains(out, "--jsonpath") { + t.Errorf("expected both warnings, got %q", out) + } + }) +} diff --git a/cmd/print.go b/cmd/print.go index 35cecef..fccbebd 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -184,10 +184,10 @@ func tsvEscape(s string) string { } func printTSVResults(w io.Writer, filtered []store.Result) { - _, _ = fmt.Fprintln(w, "slug\ttitle\tfile_path\tscore\ttags\textra\theading_path\ttext") + _, _ = fmt.Fprintln(w, "note_slug\ttitle\tfile_path\tscore\ttags\textra\theading_path\ttext") for _, r := range filtered { tagsStr := formatTags(r.Tags) - _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%f\t%s\t%s\t%s\t%s\n", + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%.4f\t%s\t%s\t%s\t%s\n", tsvEscape(r.NoteSlug), tsvEscape(r.Title), tsvEscape(r.FilePath), r.Score, tsvEscape(tagsStr), tsvEscape(r.Extra), tsvEscape(r.HeadingPath), tsvEscape(r.Text)) } @@ -260,7 +260,7 @@ func printTextResults(w io.Writer, commandName, query string, queries []string, rank := rankStyle.Render(fmt.Sprintf("%d.", i+1)) displayTitle := r.Title - if r.FileType == kindPDF { + if r.FileType == store.FileTypePDF { displayTitle = pdfTagStyle.Render("[PDF] ") + displayTitle } diff --git a/cmd/print_test.go b/cmd/print_test.go index a1d8f37..3ef76e6 100644 --- a/cmd/print_test.go +++ b/cmd/print_test.go @@ -336,7 +336,7 @@ func TestPrintResultsFormatted_Formats(t *testing.T) { globals.Format = "tsv" printResultsFormattedToWriter(&buf, "search", "query", "query", nil, results, globals, nil) outTSV := buf.String() - if !strings.Contains(outTSV, "slug\ttitle\tfile_path") || !strings.Contains(outTSV, "json-note\tJSON Note") { + if !strings.Contains(outTSV, "note_slug\ttitle\tfile_path") || !strings.Contains(outTSV, "json-note\tJSON Note") { t.Errorf("Expected tsv header and row, got %q", outTSV) } } @@ -373,6 +373,27 @@ func TestPrintTSVResults_EscapesMultilineText(t *testing.T) { } } +func TestPrintTSVResults_ScoreFourDecimalPlaces(t *testing.T) { + var buf bytes.Buffer + results := []store.Result{ + {NoteSlug: "score-note", Title: "Score Note", Score: 0.123456}, + } + globals := &Globals{Format: "tsv"} + printResultsFormattedToWriter(&buf, "search", "query", "query", nil, results, globals, nil) + lines := strings.Split(buf.String(), "\n") + if len(lines) < 3 { + t.Fatalf("expected header+row, got %q", buf.String()) + } + line := lines[1] + cols := strings.Split(line, "\t") + if len(cols) < 5 { + t.Fatalf("row has %d cols, want >= 5: %q", len(cols), line) + } + if cols[3] != "0.1235" { + t.Errorf("score col = %q, want 0.1235 (4dp, same rounding as JSON)", cols[3]) + } +} + func TestPrintResultsFormatted_MinScore(t *testing.T) { var buf bytes.Buffer results := []store.Result{ diff --git a/cmd/stats.go b/cmd/stats.go index 019bed1..ab479c8 100644 --- a/cmd/stats.go +++ b/cmd/stats.go @@ -45,18 +45,19 @@ func (c *StatsCmd) Run(globals *Globals) error { return err } + env := struct { + Command string `json:"command"` + *store.Stats + }{ + Command: "stats", + Stats: stats, + } + if globals.JSONPath != "" { - return printJSONPathResult(stats, globals.JSONPath) + return printJSONPathResult(env, globals.JSONPath) } if globals.Format == formatJSON { - env := struct { - Command string `json:"command"` - *store.Stats - }{ - Command: "stats", - Stats: stats, - } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") return enc.Encode(env) diff --git a/cmd/testhelpers_test.go b/cmd/testhelpers_test.go index d837b1f..437f8a7 100644 --- a/cmd/testhelpers_test.go +++ b/cmd/testhelpers_test.go @@ -25,6 +25,7 @@ type fakeStore struct { noteMeta *store.NoteContent noteMetaErr error noteHead *store.NoteContent + stats *store.Stats } func (f *fakeStore) Close() error { return nil } @@ -122,7 +123,9 @@ func (f *fakeStore) GetNoteMetadata(context.Context) (map[string]store.NoteMeta, } func (f *fakeStore) Stats(context.Context) (*store.Stats, error) { - return &store.Stats{}, nil + f.mu.Lock() + defer f.mu.Unlock() + return f.stats, nil } func (f *fakeStore) PopulateContext(context.Context, []store.Result, int) error { From b3673363bb8a57b46a13c54b2349d2395d362918 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:06:56 +0530 Subject: [PATCH 13/18] feat(cmd): rename refs kind filters to --only-* with deprecated aliases - --image/--pdf/--other/--external become --only-images, --only-pdfs, --only-other, --only-external; old names stay as hidden aliases merging into the same sets - help text now states: no filter = all kinds; multiple filters are unioned - --only-* combos also apply to --tsv output, where missing is now always false (missing files are not ingested yet, so they are not "missing"; silence the noise when piping) - hoist the vault-path usage error to the shared constant so ingest and refs print the same message --- cmd/refs.go | 71 +++++++++++++++++-------- cmd/refs_test.go | 135 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 178 insertions(+), 28 deletions(-) diff --git a/cmd/refs.go b/cmd/refs.go index 6ec0a59..4956e48 100644 --- a/cmd/refs.go +++ b/cmd/refs.go @@ -23,6 +23,7 @@ package cmd import ( "encoding/json" + "errors" "fmt" "io" "net/url" @@ -46,11 +47,18 @@ const ( type RefsCmd struct { Note string `arg:"" help:"note slug, title, or file path (auto-resolved)" completion-predictor:"note-slug"` - Images bool `group:"refs" help:"include image attachments" default:"false"` - PDF bool `group:"refs" help:"include PDF attachments" default:"false"` - Other bool `group:"refs" help:"include other attachments (video, audio, archives, office docs)" default:"false"` - ExternalLinks bool `group:"refs" name:"external-links" help:"include external website links (URLs)" default:"false"` + OnlyImages bool `group:"refs" name:"only-images" help:"limit to image attachments (no filter = all kinds; combine filters to union)" default:"false"` + OnlyPDF bool `group:"refs" name:"only-pdf" help:"limit to PDF attachments (no filter = all kinds; combine filters to union)" default:"false"` + OnlyOther bool `group:"refs" name:"only-other" help:"limit to other attachments (video, audio, archives, office docs) (no filter = all kinds; combine filters to union)" default:"false"` + OnlyExternal bool `group:"refs" name:"only-external-links" help:"limit to external website links (URLs) (no filter = all kinds; combine filters to union)" default:"false"` IncludeMissing bool `group:"refs" name:"include-missing" help:"include references whose file is missing from the vault" default:"false"` + + // Deprecated aliases for the kind filters, kept parseable but hidden from + // --help. Use the --only-* flags instead. + Images bool `group:"refs" name:"images" hidden:"" help:"deprecated: use --only-images" default:"false"` + PDF bool `group:"refs" name:"pdf" hidden:"" help:"deprecated: use --only-pdf" default:"false"` + Other bool `group:"refs" name:"other" hidden:"" help:"deprecated: use --only-other" default:"false"` + ExternalLinks bool `group:"refs" name:"external-links" hidden:"" help:"deprecated: use --only-external-links" default:"false"` } // refEntry is one resolved reference row. Path is absolute for attachments and @@ -75,7 +83,7 @@ func (c *RefsCmd) Run(globals *Globals) error { ctx := globals.Ctx vaultPath := globals.VaultPath if vaultPath == "" { - return &UsageError{Err: fmt.Errorf("--vault-path flag or config file setting must be specified — run 'notebrain init' to create a config")} + return &UsageError{Err: errors.New(vaultPathUsageError)} } if strings.TrimSpace(c.Note) == "" { return &UsageError{Err: fmt.Errorf("%s requires a note slug, title, or file path", groupRefs)} @@ -239,31 +247,53 @@ func filterExistingRefs(entries []refEntry) []refEntry { } // filterRefKinds keeps rows matching any selected kind flag; no flags select -// every kind. +// every kind. Deprecated aliases (Images/PDF/Other/ExternalLinks) count the +// same as their --only-* replacements. func filterRefKinds(entries []refEntry, c *RefsCmd) []refEntry { - if !c.Images && !c.PDF && !c.Other && !c.ExternalLinks { + if len(entries) == 0 { + return entries + } + filter := refsKindFilterFromCmd(c) + if !filter.images && !filter.pdf && !filter.other && !filter.external { return entries } kept := make([]refEntry, 0, len(entries)) for _, e := range entries { - keep := false - switch e.Kind { - case kindImage: - keep = c.Images - case kindPDF: - keep = c.PDF - case kindOther: - keep = c.Other - case kindExternal: - keep = c.ExternalLinks - } - if keep { + if filter.keep(e.Kind) { kept = append(kept, e) } } return kept } +// refsKindFilter merges the --only-* flags with their deprecated aliases. +type refsKindFilter struct { + images, pdf, other, external bool +} + +func refsKindFilterFromCmd(c *RefsCmd) refsKindFilter { + return refsKindFilter{ + images: c.OnlyImages || c.Images, + pdf: c.OnlyPDF || c.PDF, + other: c.OnlyOther || c.Other, + external: c.OnlyExternal || c.ExternalLinks, + } +} + +func (f refsKindFilter) keep(kind string) bool { + switch kind { + case kindImage: + return f.images + case kindPDF: + return f.pdf + case kindOther: + return f.other + case kindExternal: + return f.external + } + return false +} + // printRefsFormatted renders a refs envelope to stdout based on the requested // format. JSONPath extraction applies to the envelope when requested. func printRefsFormatted(env refsEnvelope, globals *Globals) error { @@ -284,9 +314,6 @@ func printRefsFormattedToWriter(w io.Writer, env refsEnvelope, globals *Globals) _, _ = fmt.Fprintln(w, "path\tkind\tmissing\trelative_path") for _, r := range env.Refs { missing := strconv.FormatBool(r.Missing) - if r.Kind == kindExternal { - missing = "" - } _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", tsvEscape(r.Path), r.Kind, missing, tsvEscape(r.RelativePath)) } return nil diff --git a/cmd/refs_test.go b/cmd/refs_test.go index 1978407..f228155 100644 --- a/cmd/refs_test.go +++ b/cmd/refs_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/alecthomas/kong" + "github.com/nmdra/notebrain-cli/v2/internal/store" ) @@ -106,28 +108,46 @@ func TestRefsFilters(t *testing.T) { }{ { name: "images only", - cmd: RefsCmd{Note: "router", Images: true}, + cmd: RefsCmd{Note: "router", OnlyImages: true}, include: []string{filepath.Join(vaultDir, "Notes", "cover.png")}, exclude: []string{"att.pdf", "https://example.com/docs", "[external-links]"}, }, { name: "pdf only", - cmd: RefsCmd{Note: "router", PDF: true}, + cmd: RefsCmd{Note: "router", OnlyPDF: true}, include: []string{filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, exclude: []string{"cover.png", "https://example.com/docs"}, }, { name: "external links only", - cmd: RefsCmd{Note: "router", ExternalLinks: true}, + cmd: RefsCmd{Note: "router", OnlyExternal: true}, include: []string{"[external-links] https://example.com/docs", "[external-links] https://links.example.com"}, exclude: []string{"cover.png", "att.pdf", "localhost", ".png"}, }, { name: "combined or", - cmd: RefsCmd{Note: "router", Images: true, PDF: true}, + cmd: RefsCmd{Note: "router", OnlyImages: true, OnlyPDF: true}, include: []string{filepath.Join(vaultDir, "Notes", "cover.png"), filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, exclude: []string{"https://example.com"}, }, + { + name: "deprecated images alias", + cmd: RefsCmd{Note: "router", Images: true}, + include: []string{filepath.Join(vaultDir, "Notes", "cover.png")}, + exclude: []string{"att.pdf", "https://example.com/docs", "[external-links]"}, + }, + { + name: "deprecated pdf alias", + cmd: RefsCmd{Note: "router", PDF: true}, + include: []string{filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, + exclude: []string{"cover.png", "https://example.com/docs"}, + }, + { + name: "deprecated mixed aliases", + cmd: RefsCmd{Note: "router", Images: true, OnlyExternal: true}, + include: []string{filepath.Join(vaultDir, "Notes", "cover.png"), "[external-links] https://example.com/docs"}, + exclude: []string{"att.pdf"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -327,7 +347,7 @@ func TestRefsTSV(t *testing.T) { t.Errorf("tsv row = %q, want %q", lines[1], want) } last := lines[len(lines)-1] - if want := "https://links.example.com\texternal-links\t\t"; last != want { + if want := "https://links.example.com\texternal-links\tfalse\t"; last != want { t.Errorf("external tsv row = %q, want %q", last, want) } } @@ -424,6 +444,109 @@ func TestRefsNoteNotFoundPassthrough(t *testing.T) { } } +// refsParser builds a kong parser bound to a RefsCmd so flag-level behavior +// (alias parsing, help visibility) can be asserted without running the command. +func refsParser(t *testing.T) (*kong.Kong, *RefsCmd) { + t.Helper() + var cli struct { + Globals Globals `embed:""` + Refs RefsCmd `cmd:""` + } + parser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + return parser, &cli.Refs +} + +func TestRefsOnlyFlagsParseAndLegacyAliasesHidden(t *testing.T) { + parser, _ := refsParser(t) + + tests := []struct { + name string + args []string + }{ + {name: "only-images", args: []string{"refs", "x", "--only-images"}}, + {name: "only-pdf", args: []string{"refs", "x", "--only-pdf"}}, + {name: "only-other", args: []string{"refs", "x", "--only-other"}}, + {name: "only-external-links", args: []string{"refs", "x", "--only-external-links"}}, + {name: "include-missing", args: []string{"refs", "x", "--include-missing"}}, + {name: "legacy images alias", args: []string{"refs", "x", "--images"}}, + {name: "legacy pdf alias", args: []string{"refs", "x", "--pdf"}}, + {name: "legacy other alias", args: []string{"refs", "x", "--other"}}, + {name: "legacy external-links alias", args: []string{"refs", "x", "--external-links"}}, + {name: "new and legacy mixed", args: []string{"refs", "x", "--only-images", "--pdf"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, err := parser.Parse(tt.args) + if err != nil { + t.Fatalf("parse %v: %v", tt.args, err) + } + if !strings.HasPrefix(ctx.Command(), "refs ") { + t.Errorf("command = %q, want refs", ctx.Command()) + } + }) + } + + var help strings.Builder + parser.Stdout = &help + ctx, err := parser.Parse([]string{"refs", "x"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := ctx.PrintUsage(false); err != nil { + t.Fatalf("PrintUsage: %v", err) + } + for _, visible := range []string{"--only-images", "--only-pdf", "--only-other", "--only-external-links", "--include-missing"} { + if !strings.Contains(help.String(), visible) { + t.Errorf("help must show %s:\n%s", visible, help.String()) + } + } + for _, hidden := range []string{"--images", "--image", "--pdf", "--other", "--external-links", "deprecated"} { + if strings.Contains(help.String(), hidden) { + t.Errorf("help must not show deprecated alias %q:\n%s", hidden, help.String()) + } + } + + var cli struct { + Globals Globals `embed:""` + Refs RefsCmd `cmd:""` + } + flagParser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + if _, err := flagParser.Parse([]string{"refs", "x", "--only-images"}); err != nil { + t.Fatalf("parse: %v", err) + } + if !cli.Refs.OnlyImages { + t.Errorf("--only-images did not set OnlyImages") + } +} + +func TestRefsMultipleOnlyFlagsUnion(t *testing.T) { + parser, refs := refsParser(t) + + t.Run("only-images + only-pdf", func(t *testing.T) { + _, err := parser.Parse([]string{"refs", "x", "--only-images", "--only-pdf"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !refs.OnlyImages || !refs.OnlyPDF { + t.Errorf("expected both flags set, got %+v", *refs) + } + }) + t.Run("only-images + legacy pdf", func(t *testing.T) { + if _, err := parser.Parse([]string{"refs", "x", "--only-images", "--pdf"}); err != nil { + t.Fatalf("parse: %v", err) + } + if !refs.OnlyImages || !refs.PDF { + t.Errorf("expected new + legacy flags set, got %+v", *refs) + } + }) +} + func TestPrintRefsFormattedToWriter(t *testing.T) { env := refsEnvelope{ Command: "refs", @@ -479,7 +602,7 @@ func TestPrintRefsFormattedToWriter(t *testing.T) { if err := printRefsFormattedToWriter(&sb, env, &Globals{Format: formatTSV}); err != nil { t.Fatal(err) } - want := "path\tkind\tmissing\trelative_path\n/vault/Notes/cover.png\timage\tfalse\tNotes/cover.png\nhttps://example.com\texternal-links\t\t\n" + want := "path\tkind\tmissing\trelative_path\n/vault/Notes/cover.png\timage\tfalse\tNotes/cover.png\nhttps://example.com\texternal-links\tfalse\t\n" if sb.String() != want { t.Errorf("tsv = %q, want %q", sb.String(), want) } From c050e2f6fdee704016fe295795c9757533f31d16 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:07:04 +0530 Subject: [PATCH 14/18] feat(cmd): unify cross-command flag naming - ingest --enable-pdf => --with-pdf (hidden --enable-pdf alias kept for compatibility); init wizard writes the with-pdf config key while still honouring a legacy enable-pdf key on read - get takes the note slug positionally: get <NOTE> (legacy --slug flag kept); echoed field is Note - search --exclude-note => --exclude-notes (hidden alias merges) - remove the hidden --top-k in favour of sole --candidate-chunks with default 3; config key remains top_words - config.example.toml documents with-pdf at ingest and drops the deprecated enable-pdf line in favour of the with-pdf key BREAKING CHANGE: the hidden --top-k flag is removed; use --candidate-chunks instead. --- cmd/get.go | 31 ++++++------- cmd/get_flags_test.go | 6 +-- cmd/hidden.go | 10 ++--- cmd/hidden_flags_test.go | 62 ++++++++++++++++++++++++++ cmd/ingest.go | 9 ++-- cmd/ingest_flags_test.go | 95 ++++++++++++++++++++++++++++++++++++++++ cmd/init.go | 9 ++-- cmd/init_test.go | 6 +-- cmd/search.go | 9 +++- cmd/search_flags_test.go | 90 +++++++++++++++++++++++++++++++++++++ config.example.toml | 5 ++- 11 files changed, 293 insertions(+), 39 deletions(-) create mode 100644 cmd/hidden_flags_test.go create mode 100644 cmd/ingest_flags_test.go create mode 100644 cmd/search_flags_test.go diff --git a/cmd/get.go b/cmd/get.go index 0be3a20..2372f83 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -10,7 +10,7 @@ import ( ) type GetCmd struct { - Slug string `arg:"" help:"note slug, title, or file path (auto-resolved)" completion-predictor:"note-slug"` + Note string `arg:"" help:"note slug, title, or file path (auto-resolved)" completion-predictor:"note-slug"` Meta bool `group:"get" help:"show only the note header (title, path, tags, chunk count) without any text" default:"false"` Head int `group:"get" help:"show only the first N chunks of text (0 = full note)" default:"0"` } @@ -26,31 +26,32 @@ func (c *GetCmd) Run(globals *Globals) error { var note *store.NoteContent switch { case c.Meta: - note, err = st.GetNoteMeta(ctx, c.Slug) + note, err = st.GetNoteMeta(ctx, c.Note) case c.Head > 0: - note, err = st.GetNoteHead(ctx, c.Slug, c.Head) + note, err = st.GetNoteHead(ctx, c.Note, c.Head) default: - note, err = st.GetNote(ctx, c.Slug) + note, err = st.GetNote(ctx, c.Note) } if err != nil { return err } + env := struct { + Command string `json:"command"` + Query string `json:"query"` + Note *store.NoteContent `json:"note"` + }{ + Command: groupGet, + Query: c.Note, + Note: note, + } + if globals.JSONPath != "" { - return printJSONPathResult(note, globals.JSONPath) + return printJSONPathResult(env, globals.JSONPath) } switch globals.Format { case formatJSON: - env := struct { - Command string `json:"command"` - Query string `json:"query"` - Note *store.NoteContent `json:"note"` - }{ - Command: groupGet, - Query: c.Slug, - Note: note, - } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") return enc.Encode(env) @@ -59,7 +60,7 @@ func (c *GetCmd) Run(globals *Globals) error { fmt.Println("note_slug\ttitle\tfile_path\ttags\tchunks\ttext") tagsStr := formatTags(note.Tags) fmt.Printf("%s\t%s\t%s\t%s\t%d\t%s\n", - note.NoteSlug, note.Title, note.FilePath, tagsStr, note.Chunks, note.Text) + tsvEscape(note.NoteSlug), tsvEscape(note.Title), tsvEscape(note.FilePath), tsvEscape(tagsStr), note.Chunks, tsvEscape(note.Text)) return nil default: // "text" diff --git a/cmd/get_flags_test.go b/cmd/get_flags_test.go index 64a7655..149a020 100644 --- a/cmd/get_flags_test.go +++ b/cmd/get_flags_test.go @@ -15,7 +15,7 @@ func TestGetMetaMode(t *testing.T) { withFakeStore(t, fs) out := captureStdout(t, func() { - if err := (&GetCmd{Slug: "multi", Meta: true}).Run(&Globals{Ctx: context.Background(), Format: formatText}); err != nil { + if err := (&GetCmd{Note: "multi", Meta: true}).Run(&Globals{Ctx: context.Background(), Format: formatText}); err != nil { t.Errorf("Run: %v", err) } }) @@ -37,7 +37,7 @@ func TestGetHeadMode(t *testing.T) { withFakeStore(t, fs) out := captureStdout(t, func() { - if err := (&GetCmd{Slug: "multi", Head: 2}).Run(&Globals{Ctx: context.Background(), Format: formatText}); err != nil { + if err := (&GetCmd{Note: "multi", Head: 2}).Run(&Globals{Ctx: context.Background(), Format: formatText}); err != nil { t.Errorf("Run: %v", err) } }) @@ -54,7 +54,7 @@ func TestGetFullModeUsesGetNote(t *testing.T) { withFakeStore(t, fs) _ = captureStdout(t, func() { - _ = (&GetCmd{Slug: "multi"}).Run(&Globals{Ctx: context.Background(), Format: formatJSON}) + _ = (&GetCmd{Note: "multi"}).Run(&Globals{Ctx: context.Background(), Format: formatJSON}) }) if fs.metaCalls != 0 || fs.headCalls != 0 { t.Errorf("full get must not call meta/head (metaCalls=%d headCalls=%d)", fs.metaCalls, fs.headCalls) diff --git a/cmd/hidden.go b/cmd/hidden.go index 7981d4f..bba9909 100644 --- a/cmd/hidden.go +++ b/cmd/hidden.go @@ -33,8 +33,7 @@ type HiddenCmd struct { Note string `arg:"" help:"note slug, title, or file path (auto-resolved)" completion-predictor:"note-slug"` Limit int `group:"hidden" help:"maximum number of hidden connections to return" default:"10"` Deep bool `group:"hidden" help:"analyze each chunk individually for granular section-level matches"` - TopK int `group:"hidden" name:"top-k" help:"chunks to evaluate per candidate note in --deep mode (deprecated alias: --candidate-chunks)" default:"3"` - CandidateChunks int `group:"hidden" name:"candidate-chunks" help:"chunks to evaluate per candidate note in --deep mode (replaces --top-k)"` + CandidateChunks int `group:"hidden" name:"candidate-chunks" help:"chunks to evaluate per candidate note in --deep mode" default:"3"` IncludeLinked bool `group:"hidden" name:"include-linked" help:"include notes even if they are already linked directly or indirectly"` ChunkDisplayFlags } @@ -42,10 +41,7 @@ type HiddenCmd struct { func (c *HiddenCmd) Run(globals *Globals) error { targetNote := c.Note limit := c.Limit - topK := c.TopK - if c.CandidateChunks != 0 { - topK = c.CandidateChunks - } + topK := c.CandidateChunks ctx := globals.Ctx st, err := openStore(ctx, globals) @@ -102,7 +98,7 @@ func (c *HiddenCmd) Run(globals *Globals) error { return err } - cmdName := "hidden" + cmdName := groupHidden title := fmt.Sprintf("Hidden connections for: %q (slug: %s)", targetNote, targetSlug) if c.IncludeLinked { cmdName = "hidden --include-linked" diff --git a/cmd/hidden_flags_test.go b/cmd/hidden_flags_test.go new file mode 100644 index 0000000..e5004c4 --- /dev/null +++ b/cmd/hidden_flags_test.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "strings" + "testing" + + "github.com/alecthomas/kong" +) + +// hiddenParser builds a kong parser bound to a HiddenCmd so flag-level +// behavior can be asserted without running the command. +func hiddenParser(t *testing.T) (*kong.Kong, *HiddenCmd) { + t.Helper() + var cli struct { + Globals Globals `embed:""` + Hidden HiddenCmd `cmd:""` + } + parser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + return parser, &cli.Hidden +} + +func TestHiddenTopKRemovedInFavorOfCandidateChunks(t *testing.T) { + parser, hidden := hiddenParser(t) + + t.Run("candidate-chunks accepts value", func(t *testing.T) { + if _, err := parser.Parse([]string{"hidden", "x", "--candidate-chunks", "5"}); err != nil { + t.Fatalf("parse: %v", err) + } + if hidden.CandidateChunks != 5 { + t.Errorf("CandidateChunks = %d, want 5", hidden.CandidateChunks) + } + }) + + t.Run("top-k is rejected", func(t *testing.T) { + _, err := parser.Parse([]string{"hidden", "x", "--top-k", "5"}) + if err == nil { + t.Errorf("expected --top-k to be rejected, got no error") + } + if err != nil && !strings.Contains(err.Error(), "unknown flag") { + t.Errorf("expected unknown-flag error, got: %v", err) + } + }) + + var help strings.Builder + parser.Stdout = &help + ctx, err := parser.Parse([]string{"hidden", "x"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := ctx.PrintUsage(false); err != nil { + t.Fatalf("PrintUsage: %v", err) + } + if !strings.Contains(help.String(), "--candidate-chunks") { + t.Errorf("help must show --candidate-chunks:\n%s", help.String()) + } + if strings.Contains(help.String(), "--top-k") || strings.Contains(help.String(), "deprecated") { + t.Errorf("help must not mention removed --top-k:\n%s", help.String()) + } +} diff --git a/cmd/ingest.go b/cmd/ingest.go index 4609013..97b2afe 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -22,7 +22,7 @@ THE SOFTWARE. package cmd import ( - "fmt" + "errors" "log/slog" "github.com/nmdra/notebrain-cli/v2/internal/embedder" @@ -37,7 +37,8 @@ type IngestCmd struct { ChunkSize int `group:"ingest" name:"chunk-size" help:"max runes per chunk" default:"800"` ChunkOverlap int `group:"ingest" name:"chunk-overlap" help:"overlap runes between sub-chunks" default:"100"` RespectExclude bool `group:"ingest" help:"respect Obsidian userIgnoreFilters and attachmentFolderPath settings during ingest" default:"false"` - EnablePDF bool `group:"ingest" help:"enable indexing of PDF attachments" default:"false"` + WithPDF bool `group:"ingest" name:"with-pdf" help:"include PDF attachments in indexing" default:"false"` + EnablePDF bool `group:"ingest" name:"enable-pdf" hidden:"" help:"deprecated: use --with-pdf" default:"false"` LLMModel string `group:"ingest" name:"llm-model" help:"LLM model to use for PDF parsing (e.g. openrouter/anthropic/claude-sonnet, deepseek-chat). Requires API key in env." default:"" completion-predictor:"llm-model"` LLMContextWindow int `group:"ingest" name:"llm-context-window" help:"total context window size of the LLM in tokens. Set this to match your specific model." default:"128000"` } @@ -46,7 +47,7 @@ func (c *IngestCmd) Run(globals *Globals) error { workers := c.Workers vaultPath := globals.VaultPath if vaultPath == "" { - return &UsageError{Err: fmt.Errorf("--vault-path flag or config file setting must be specified — run 'notebrain init' to create a config")} + return &UsageError{Err: errors.New(vaultPathUsageError)} } glob := c.Glob @@ -72,7 +73,7 @@ func (c *IngestCmd) Run(globals *Globals) error { pipeline := ingest.NewPipeline(st, emb, workers) pipeline.RespectExclude = c.RespectExclude - pipeline.EnablePDF = c.EnablePDF + pipeline.EnablePDF = c.WithPDF || c.EnablePDF pipeline.LLMModel = c.LLMModel pipeline.LLMContextWindow = c.LLMContextWindow pipeline.MinChunkWords = c.MinChunkWords diff --git a/cmd/ingest_flags_test.go b/cmd/ingest_flags_test.go new file mode 100644 index 0000000..e030415 --- /dev/null +++ b/cmd/ingest_flags_test.go @@ -0,0 +1,95 @@ +package cmd + +import ( + "strings" + "testing" + + "github.com/alecthomas/kong" +) + +// ingestParser builds a kong parser bound to an IngestCmd so flag-level +// behavior (alias parsing, help visibility) can be asserted without running +// the command. +func ingestParser(t *testing.T) (*kong.Kong, *IngestCmd) { + t.Helper() + var cli struct { + Globals Globals `embed:""` + Ingest IngestCmd `cmd:""` + } + parser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + return parser, &cli.Ingest +} + +func TestIngestWithPDFFlagAndLegacyAlias(t *testing.T) { + parser, ingest := ingestParser(t) + + for _, tt := range []struct { + name string + args []string + }{ + {name: "with-pdf", args: []string{"ingest", "--with-pdf"}}, + {name: "legacy enable-pdf alias", args: []string{"ingest", "--enable-pdf"}}, + {name: "both", args: []string{"ingest", "--with-pdf", "--enable-pdf"}}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx, err := parser.Parse(tt.args) + if err != nil { + t.Fatalf("parse %v: %v", tt.args, err) + } + if !strings.HasPrefix(ctx.Command(), "ingest") { + t.Errorf("command = %q, want ingest", ctx.Command()) + } + }) + } + + if _, err := parser.Parse([]string{"ingest", "--with-pdf"}); err != nil { + t.Fatalf("parse: %v", err) + } + if !ingest.WithPDF { + t.Errorf("--with-pdf parse did not set the command field") + } + + var help strings.Builder + parser.Stdout = &help + ctx, err := parser.Parse([]string{"ingest"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := ctx.PrintUsage(false); err != nil { + t.Fatalf("PrintUsage: %v", err) + } + if !strings.Contains(help.String(), "--with-pdf") { + t.Errorf("help must show --with-pdf:\n%s", help.String()) + } + if strings.Contains(help.String(), "--enable-pdf") || strings.Contains(help.String(), "deprecated") { + t.Errorf("help must hide deprecated --enable-pdf alias:\n%s", help.String()) + } + + var cli struct { + Globals Globals `embed:""` + Ingest IngestCmd `cmd:""` + } + flagParser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + if _, err := flagParser.Parse([]string{"ingest", "--with-pdf"}); err != nil { + t.Fatalf("parse: %v", err) + } + if !cli.Ingest.WithPDF { + t.Errorf("--with-pdf did not set WithPDF") + } +} + +func TestIngestFlagValuesDefault(t *testing.T) { + _, ingest := ingestParser(t) + if ingest.EnablePDF { + t.Errorf("deprecated EnablePDF must default false") + } + if ingest.WithPDF { + t.Errorf("WithPDF must default false") + } +} diff --git a/cmd/init.go b/cmd/init.go index 3383df1..0b941bd 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -15,7 +15,8 @@ type InitCmd struct{} // existingConfig mirrors the config keys the wizard can prefill. type existingConfig struct { VaultPath string `toml:"vault-path"` - EnablePDF bool `toml:"enable-pdf"` + WithPDF bool `toml:"with-pdf"` + EnablePDF bool `toml:"enable-pdf"` // deprecated config key, still honored } func (c *InitCmd) Run(globals *Globals) error { @@ -72,11 +73,11 @@ func (c *InitCmd) Run(globals *Globals) error { } // Ask for PDF support - enablePDF := askYesNo(reader, "Enable text extraction for PDF attachments?", existing.EnablePDF) + enablePDF := askYesNo(reader, "Enable text extraction for PDF attachments?", existing.WithPDF || existing.EnablePDF) // Preview the changes before writing anything. fmt.Println() - printWarning("Ready to write", fmt.Sprintf("vault-path = %q\nenable-pdf = %t", vaultPath, enablePDF)) + printWarning("Ready to write", fmt.Sprintf("vault-path = %q\nwith-pdf = %t", vaultPath, enablePDF)) if !askYesNo(reader, fmt.Sprintf("Write configuration to %s?", configPath), true) { fmt.Println("Initialization aborted.") return nil @@ -92,7 +93,7 @@ func (c *InitCmd) Run(globals *Globals) error { // Replace PDF flag if enablePDF { - configStr = strings.Replace(configStr, "# enable-pdf = false", "enable-pdf = true", 1) + configStr = strings.Replace(configStr, "# with-pdf = false", "with-pdf = true", 1) } // Write the config file diff --git a/cmd/init_test.go b/cmd/init_test.go index 0c77397..6782205 100644 --- a/cmd/init_test.go +++ b/cmd/init_test.go @@ -12,7 +12,7 @@ import ( // initTestConfig is a minimal stand-in for config.example.toml containing the // marker lines the init wizard rewrites. const initTestConfig = `# vault-path = "/home/user/Documents/Obsidian Vault" -# enable-pdf = false +# with-pdf = false ` func TestInitCmdWritesConfig(t *testing.T) { @@ -38,8 +38,8 @@ func TestInitCmdWritesConfig(t *testing.T) { if !strings.Contains(s, wantVault) { t.Errorf("config missing %q, got:\n%s", wantVault, s) } - if !strings.Contains(s, "enable-pdf = true") { - t.Errorf("config missing enable-pdf = true, got:\n%s", s) + if !strings.Contains(s, "with-pdf = true") { + t.Errorf("config missing with-pdf = true, got:\n%s", s) } } diff --git a/cmd/search.go b/cmd/search.go index 5cddc83..4b51256 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -42,7 +42,8 @@ type SearchCmd struct { HasTasks bool `group:"search" help:"only return chunks containing task lists (checkboxes)"` HasCode bool `group:"search" help:"only return chunks containing fenced code blocks"` WithPDF bool `group:"search" help:"include PDF results in search"` - ExcludeNotes []string `group:"search" name:"exclude-note" help:"exclude notes from results (slug, title, or path; repeatable or comma-separated)" completion-predictor:"note-slug"` + ExcludeNotes []string `group:"search" name:"exclude-notes" help:"exclude notes from results (slug, title, or path; repeatable or comma-separated)" completion-predictor:"note-slug"` + ExcludeNote []string `group:"search" name:"exclude-note" hidden:"" help:"deprecated: use --exclude-notes"` ChunkDisplayFlags } @@ -111,6 +112,9 @@ func (c *SearchCmd) Run(globals *Globals) error { } defer func() { _ = st.Close() }() + if len(c.ExcludeNote) > 0 { + c.ExcludeNotes = append(append([]string(nil), c.ExcludeNotes...), c.ExcludeNote...) + } excluded, err := c.resolveExcludes(ctx, st) if err != nil { return err @@ -126,7 +130,8 @@ func (c *SearchCmd) Run(globals *Globals) error { return c.runStatic(ctx, globals, st, emb, resolved, displayQueries, excluded) } -// resolveExcludes normalizes, resolves, and validates --exclude-note values. +// resolveExcludes normalizes, resolves, and validates --exclude-notes +// (and the deprecated --exclude-note alias) values. // Each value may be a slug, title, filename, or partial path (the same // resolution `get` and `hidden` use). Values that resolve to nothing are // reported as a warning so typos do not silently no-op. Returns the resolved diff --git a/cmd/search_flags_test.go b/cmd/search_flags_test.go new file mode 100644 index 0000000..40d0f95 --- /dev/null +++ b/cmd/search_flags_test.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "strings" + "testing" + + "github.com/alecthomas/kong" +) + +// searchParser builds a kong parser bound to a SearchCmd so flag-level +// behavior (alias parsing, help visibility) can be asserted without running +// the command. +func searchParser(t *testing.T) (*kong.Kong, *SearchCmd) { + t.Helper() + var cli struct { + Globals Globals `embed:""` + Search SearchCmd `cmd:""` + } + parser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + return parser, &cli.Search +} + +func TestSearchExcludeNotesFlagRename(t *testing.T) { + parser, search := searchParser(t) + + t.Run("exclude-notes accepts one value", func(t *testing.T) { + if _, err := parser.Parse([]string{"search", "q", "--exclude-notes", "alpha.md"}); err != nil { + t.Fatalf("parse: %v", err) + } + if len(search.ExcludeNotes) != 1 || search.ExcludeNotes[0] != "alpha.md" { + t.Errorf("ExcludeNotes = %v, want [alpha.md]", search.ExcludeNotes) + } + }) + + t.Run("exclude-notes accepts repeats", func(t *testing.T) { + if _, err := parser.Parse([]string{"search", "q", "--exclude-notes", "alpha.md", "--exclude-notes", "beta.md"}); err != nil { + t.Fatalf("parse: %v", err) + } + if len(search.ExcludeNotes) != 2 { + t.Errorf("ExcludeNotes = %v, want 2 entries", search.ExcludeNotes) + } + }) + + t.Run("legacy exclude-note alias still parses", func(t *testing.T) { + if _, err := parser.Parse([]string{"search", "q", "--exclude-note", "alpha.md"}); err != nil { + t.Fatalf("parse legacy alias: %v", err) + } + if len(search.ExcludeNote) != 1 || search.ExcludeNote[0] != "alpha.md" { + t.Errorf("ExcludeNote = %v, want [alpha.md]", search.ExcludeNote) + } + }) + + var help strings.Builder + parser.Stdout = &help + ctx, err := parser.Parse([]string{"search", "q"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := ctx.PrintUsage(false); err != nil { + t.Fatalf("PrintUsage: %v", err) + } + if !strings.Contains(help.String(), "--exclude-notes") { + t.Errorf("help must show --exclude-notes:\n%s", help.String()) + } + if strings.Contains(strings.ReplaceAll(help.String(), "--exclude-notes", ""), "--exclude-note") || strings.Contains(help.String(), "deprecated") { + t.Errorf("help must hide deprecated --exclude-note alias:\n%s", help.String()) + } +} + +func TestSearchMergesDeprecatedExcludeNoteAlias(t *testing.T) { + var cli struct { + Globals Globals `embed:""` + Search SearchCmd `cmd:""` + } + parser, err := kong.New(&cli, kong.ExplicitGroups(helpGroups())) + if err != nil { + t.Fatalf("kong.New: %v", err) + } + if _, err := parser.Parse([]string{"search", "q", "--exclude-notes", "a.md", "--exclude-note", "b.md"}); err != nil { + t.Fatalf("parse: %v", err) + } + merged := append([]string(nil), cli.Search.ExcludeNotes...) + merged = append(merged, cli.Search.ExcludeNote...) + if len(merged) != 2 { + t.Errorf("merged excludes = %v, want [a.md b.md]", merged) + } +} diff --git a/config.example.toml b/config.example.toml index 5dec522..417424c 100644 --- a/config.example.toml +++ b/config.example.toml @@ -64,7 +64,10 @@ # respect-exclude = false # default: false # Enable text extraction from PDF attachments during ingestion (requires --llm-model). -# enable-pdf = false # default: false +# with-pdf = false # default: false + +# Deprecated config key for the same setting; use with-pdf instead. +# enable-pdf = false # LLM model for converting PDF attachments into clean Markdown. # Backend is auto-detected based on which API key environment variable is present (DEEPSEEK_API_KEY, OPENROUTER_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY, OLLAMA_HOST). From 11b52ca4e8d728fc98afd1db9b5a1e0d37e96148 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:07:16 +0530 Subject: [PATCH 15/18] docs: sync flag renames across README, wiki, skill and changelog - commands docs: refs --only-* filters, ingest --with-pdf, get <NOTE>, search --exclude-notes, --candidate-chunks; hidden aliases listed for each command - PDF ingestion doc: with-pdf key + legacy enable-pdf read note - architecture doc: get --jsonpath paths shift to $.note.* - AGENTS.md notes the flag-naming standards (verb flags, --only-* filter pattern) and updates the file tree - skill references (flags, example, schema) match the new flags and config keys - changelog: Unreleased entries under Added/Changed/Deprecated/Removed --- .agents/AGENTS.md | 4 +-- .agents/skills/notebrain/SKILL.md | 2 +- .../skills/notebrain/references/example.md | 4 +-- .agents/skills/notebrain/references/flags.md | 14 ++++----- .agents/skills/notebrain/references/schema.md | 4 +-- CHANGELOG.md | 20 +++++++++++- README.md | 6 ++-- wiki/Architecture.md | 2 +- wiki/Commands.md | 31 +++++++++---------- wiki/PDF_Ingestion.md | 6 ++-- 10 files changed, 55 insertions(+), 38 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 166d6d6..9f9ce8b 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -54,7 +54,7 @@ notebrain-cli/ │ ├── client.go │ └── *_test.go └── cmd/ - ├── root.go + ├── cli.go ├── ingest.go ├── search.go ├── backlinks.go @@ -76,7 +76,7 @@ notebrain-cli/ - Name test files `*_test.go` alongside the source file. - **Go Vendoring:** This repository uses Go vendoring (`vendor/`). Whenever dependencies in `go.mod` or `go.sum` are added, removed, or updated, you MUST run `go mod vendor` before running tests or builds. - **Strict Non-Regression Guardrails:** When refactoring or removing features, always add explicit assertion tests across `internal/configfile/` and `internal/store/` to verify that existing core functions, default settings, TOML key resolution, and database initialization do not regress or depend on removed parameters. -- **CLI Testing & Flag Standards:** When executing CLI commands or writing automated tests/scripts for NoteBrain, strictly use the exact flag names `--vault-path` and `--chroma-path` (never `--vault` or `--db`). For graph and note commands (`backlinks`, `connections`, `hidden`, `tags`, `get`, `refs`), pass exactly one positional argument: the note slug (`<note>`). For `refs`, use `--images`/`--pdf`/`--other`/`--external-links` to filter by kind and `--include-missing` to surface broken attachment links. For `boosted` search, always provide the required `--seed=<slug>` flag. When testing `hidden` connection discovery where already-linked notes should be included, pass `--include-linked`. Note that `backlinks` and `connections` canonicalize link targets by stripping `#heading` anchors and matching exact vault subfolders. Use `--show-tags` to show tags in CLI output. Debug logging is enabled via `--debug`. When testing `reset` in automated scripts, pipe confirmation via stdin (`echo yes | ./notebrain reset`). To avoid contextual empty-result hints in automated scripts, always request machine formats (`--format=json`, `tsv`, or `--jsonpath`). When testing LLM-based PDF ingestion, use `--llm-model` and provide the required API key via environment variables (`DEEPSEEK_API_KEY`, or `OPENROUTER_API_KEY`). +- **CLI Testing & Flag Standards:** When executing CLI commands or writing automated tests/scripts for NoteBrain, strictly use the exact flag names `--vault-path` and `--chroma-path` (never `--vault` or `--db`). For graph and note commands (`backlinks`, `connections`, `hidden`, `tags`, `get`, `refs`), pass exactly one positional argument: the note slug (`<note>`). For `refs`, use `--only-images`/`--only-pdf`/`--only-other`/`--only-external-links` to limit by kind (no flags = all kinds; old `--images`/`--pdf`/`--other`/`--external-links` still parse but are deprecated) and `--include-missing` to surface broken attachment links. For `boosted` search, always provide the required `--seed=<slug>` flag. When testing `hidden` connection discovery where already-linked notes should be included, pass `--include-linked`. Use `--candidate-chunks` (not the removed `--top-k`) to control deep hidden-analysis depth. Note that `backlinks` and `connections` canonicalize link targets by stripping `#heading` anchors and matching exact vault subfolders. Use `--show-tags` to show tags in CLI output. Debug logging is enabled via `--debug`. When testing `reset` in automated scripts, pipe confirmation via stdin (`echo yes | ./notebrain reset`). To avoid contextual empty-result hints in automated scripts, always request machine formats (`--format=json`, `tsv`, or `--jsonpath`). When testing LLM-based PDF ingestion, use `--with-pdf` (deprecated alias: `--enable-pdf`) and `--llm-model`, and provide the required API key via environment variables (`DEEPSEEK_API_KEY`, or `OPENROUTER_API_KEY`). Use `--exclude-notes` (not the deprecated `--exclude-note`) to exclude notes from search. ## Coding Conventions diff --git a/.agents/skills/notebrain/SKILL.md b/.agents/skills/notebrain/SKILL.md index 54ed1ec..d2dc86d 100644 --- a/.agents/skills/notebrain/SKILL.md +++ b/.agents/skills/notebrain/SKILL.md @@ -46,7 +46,7 @@ notebrain search "<topic>" --format json --include-text --context-window 1 --lim **Lean shapes:** - Top candidates/slugs only: drop `--context-window`, use `--jsonpath="$.results[*].note_slug"`. - Note-level (not chunk-level) list: `--group-by-note` to collapse to the best chunk per note; `dedupe` via `--jsonpath="$.results[*].note_slug" | sort -u`. -- Weak matches above the `--min-score 0.5` floor, or `--tag`, `--section`, `--has-tasks`, `--has-code`, `--exclude-note`. +- Weak matches above the `--min-score 0.5` floor, or `--tag`, `--section`, `--has-tasks`, `--has-code`, `--exclude-notes`. - Multi-topic at once — boost by adding positional queries: `search "redis pubsub" "kafka brokers"`. - A show-tags + `--jsonpath="$.results[0].tags"` reveals real note tags in one call. diff --git a/.agents/skills/notebrain/references/example.md b/.agents/skills/notebrain/references/example.md index 2ce80cf..a956a1f 100644 --- a/.agents/skills/notebrain/references/example.md +++ b/.agents/skills/notebrain/references/example.md @@ -15,7 +15,7 @@ Quick reference: the major scenarios with the proven command sequence. Pair with | 5 | List all notes tagged X | `notebrain tags "X" --children --limit 50 --format tsv` | | 6 | Semantic search | `search "<q>" --format=json --include-text --limit 3`; escalate: `--top-k 2 --context-window 1`; stop when top score ≥ 0.75 | | 7 | Multi-topic comparison | `notebrain search "redis pubsub" "kafka brokers" --limit 5 --top-k 2 --format json` | -| 8 | Filtered search | add `--tag "kubernetes"`, `--section "Architecture > Components"`, `--has-tasks`, `--has-code`, `--exclude-note "<slug>"`, `--min-score 0.3` | +| 8 | Filtered search | add `--tag "kubernetes"`, `--section "Architecture > Components"`, `--has-tasks`, `--has-code`, `--exclude-notes "<slug>"`, `--min-score 0.3` | | 9 | Zero-result handling | short common words now fall back to a lexical token scan (`"lexical": true`, `score: 0`); if still nothing → longer descriptive phrase or `tags` query; never grep the vault | | 10 | Backlinks | `notebrain backlinks "<slug>" --format json --limit 10` | | 11 | Connections | `notebrain connections "<slug>" --hops 2 --format tsv` | @@ -24,7 +24,7 @@ Quick reference: the major scenarios with the proven command sequence. Pair with | 14 | Metadata-only extraction | `--jsonpath`, `--format tsv`, `--show-file-path=false` (cuts ~40–50% of tokens) | | 15 | Context vs full `get` | context: `--context-window 1 --include-text`; full note only on explicit demand: `get "<slug>"` | | 16 | Stale-index recovery | a slug that 404s mid-conversation → re-resolve: `search "<title>" --limit 3 --jsonpath="$.results[*].note_slug"` | -| 17 | Reference inventory | `notebrain refs "<slug>" --format json` (all kinds); kind filters: `--images` / `--pdf` / `--other` / `--external-links` | +| 17 | Reference inventory | `notebrain refs "<slug>" --format json` (all kinds); kind filters: `--only-images` / `--only-pdf` / `--only-other` / `--only-external-links` | | 18 | Broken-link audit | `notebrain refs "<slug>" --include-missing --format tsv` → rows with `missing` = `true` are broken; omit `--include-missing` to see only existing files | ## Semantics (verified) diff --git a/.agents/skills/notebrain/references/flags.md b/.agents/skills/notebrain/references/flags.md index 166f5c6..f3f8817 100644 --- a/.agents/skills/notebrain/references/flags.md +++ b/.agents/skills/notebrain/references/flags.md @@ -21,7 +21,7 @@ These flags are available only on the commands listed. | `--with-pdf` | Include PDF text extraction results in the search. Defaults to false (Markdown-only). | `false` | | `--min-score F` | Suppress results below this similarity score (0.0–1.0). Use to filter weak matches (e.g. `0.3` for meaningful hits, `0.5` for precision). Also available on `hidden` and `boosted`. | `0` | | `--group-by-note` | Collapse results to one row per note: keeps the best-scoring chunk, drops the rest. When a note has multiple matching chunks, the surviving row gains `extra: "N matching chunks"`. Text/TSV/JSON all flow through this — handy for note-level result lists. | `false` | -| `--exclude-note "SLUG"` | Exclude notes from results. Accepts a note slug, title, or path, resolved automatically; repeat the flag or use comma-separated values. Unknown notes are skipped with a warning. | — | +| `--exclude-notes "SLUG"` | Exclude notes from results. Accepts a note slug, title, or path, resolved automatically; repeat the flag or use comma-separated values. Unknown notes are skipped with a warning. Deprecated alias: `--exclude-note`. | — | > **Lexical fallback:** When semantic retrieval returns zero results — or every result is below `--min-score` — `search` automatically falls back to a token-based lexical scan over note titles, paths, tags, and text (case-insensitive, substring-style token matching, min token length 2). The header reads `Lexical Search (no semantic matches)` and rows are marked with `"lexical": true` in JSON (`score: 0`). This is why short queries like `Lecture` now return hits even when no semantic match clears the score bar. There is no lexical fallback for `boosted` or `hidden`. @@ -33,7 +33,7 @@ These flags are available only on the commands listed. | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | | `--limit N` | Maximum number of hidden connections to return. | `10` | | `--deep` | Analyze each chunk individually for granular section-level matches using stored vectors (no re-embedding required). **Requires the target note to have indexed chunks.** If resolution fails, the error names the resolved slug: `note "<slug>" has no indexed chunks ... run 'notebrain ingest' ...` — re-resolve the slug via `search` first if the note demonstrably has chunks. | `false` | -| `--top-k N` | Chunks to evaluate per candidate note in `--deep` mode. Deprecated alias: `--candidate-chunks` (replaces `--top-k`). | `3` | +| `--candidate-chunks N` | Chunks to evaluate per candidate note in `--deep` mode. The old `--top-k` alias was removed — this is the only flag. | `3` | | `--include-linked` | Include notes that are already linked directly/indirectly, while still excluding self-references. | `false` | ### `connections` @@ -80,13 +80,13 @@ Takes a single positional argument: `<slug>` (note slug, title, or file path — | Flag | Purpose | Default | | ------------------- | --------------------------------------------------------------------------------------------------------------------------- | ------- | -| `--images` | Include image attachments (png, jpg, gif, svg, webp, …). | `false` | -| `--pdf` | Include PDF attachments. | `false` | -| `--other` | Include other attachments (video, audio, archives, office docs). | `false` | -| `--external-links` | Include external http(s) website links. | `false` | +| `--only-images` | Limit to image attachments (png, jpg, gif, svg, webp, …). | `false` | +| `--only-pdf` | Limit to PDF attachments. | `false` | +| `--only-other` | Limit to other attachments (video, audio, archives, office docs). | `false` | +| `--only-external-links` | Limit to external http(s) website links. | `false` | | `--include-missing` | Include references whose file is missing from the vault (broken links). Hidden by default. | `false` | -Takes a single positional argument: `<note>` (note slug, title, or file path — auto-resolved, markdown notes only; PDF extractions error out). No kind flags = every kind. `refs` reads the note file fresh from disk, so results never go stale — but they cover only what the current file contains. External links are never `missing` and are never contacted over the network. Note: `refs` lists attachments and external links only — links to other notes are not included (use `backlinks`/`connections`). +Takes a single positional argument: `<note>` (note slug, title, or file path — auto-resolved, markdown notes only; PDF extractions error out). No kind flags = every kind; combine `--only-*` flags to union kinds. The old names `--images`/`--pdf`/`--other`/`--external-links` still parse but are deprecated. `refs` reads the note file fresh from disk, so results never go stale — but they cover only what the current file contains. External links are never `missing` and are never contacted over the network. Note: `refs` lists attachments and external links only — links to other notes are not included (use `backlinks`/`connections`). ## Global Flags (Available on Subcommands) diff --git a/.agents/skills/notebrain/references/schema.md b/.agents/skills/notebrain/references/schema.md index 0c5d392..54bfb42 100644 --- a/.agents/skills/notebrain/references/schema.md +++ b/.agents/skills/notebrain/references/schema.md @@ -189,14 +189,14 @@ The `note` object shape is the same for the `get` modes: default returns the ful Rows are deduped by resolved path (or exact URL) in first-occurrence order. No kind flags = every kind. -TSV shape — header `path<TAB>kind<TAB>missing<TAB>relative_path`; the `missing` cell is empty for external links: +TSV shape — header `path<TAB>kind<TAB>missing<TAB>relative_path`; the `missing` cell shows `false` for external links: <!-- markdownlint-disable MD010 --> ```text path kind missing relative_path /home/user/vault/Attachments/eda-diagram.png image false Attachments/eda-diagram.png /home/user/vault/broken.png image true broken.png -https://martinfowler.com/articles/eda.html external-links +https://martinfowler.com/articles/eda.html external-links false ``` <!-- markdownlint-enable MD010 --> diff --git a/CHANGELOG.md b/CHANGELOG.md index 002cb86..756ec2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- **Reference Listing**: `notebrain refs <note>` lists a note's local attachments (images, PDFs, archives, …) and external website links, filterable with `--images`, `--pdf`, `--other`, and `--external-links`. Broken links are hidden by default and surfaced with `--include-missing` (`feat(cmd)`). +- **Reference Listing**: `notebrain refs <note>` lists a note's local attachments (images, PDFs, archives, …) and external website links, filterable with `--only-images`, `--only-pdf`, `--only-other`, and `--only-external-links`. Broken links are hidden by default and surfaced with `--include-missing` (`feat(cmd)`). + +### Changed +- **Refs kind filters renamed**: `refs` filters are now `--only-images`/`--only-pdf`/`--only-other`/`--only-external-links` with "limit to" semantics; no flags = all kinds, combine to union (`refactor(cmd)`). +- **PDF flag unified**: `ingest` now uses `--with-pdf` (matching `search`/`boosted`); the config key is `with-pdf` (`refactor(cmd)`). +- **Hidden depth flag**: `--candidate-chunks` is now the single flag for `hidden --deep` depth (`refactor(cmd)`). +- **Search exclude flag**: `--exclude-note` is now `--exclude-notes` (`refactor(cmd)`). +- **Get positional**: `get` positional argument renamed `<SLUG>` → `<NOTE>` in documentation and conventions (`refactor(cmd)`). +- **TSV parity**: `search` TSV header `slug` → `note_slug`; `get` TSV fields escaped; scores at 4 decimal places in TSV, matching JSON (`fix(output)`). +- **Refs TSV**: external-link rows now emit `false` in the `missing` column instead of a blank cell (`fix(output)`). +- **JSONPath envelope parity**: `get` and `stats` apply `--jsonpath` to their full command envelope; `$.command` now works, and `get` paths shift from `$.note_slug` to `$.note.note_slug` (`fix(output)`). + +### Deprecated +- `refs --images`/`--pdf`/`--other`/`--external-links` — use `--only-*` (aliases still parse, hidden from `--help`). +- `ingest --enable-pdf` (and config key `enable-pdf`) — use `--with-pdf` (`with-pdf`). +- `search --exclude-note` — use `--exclude-notes`. + +### Removed +- `hidden --top-k` (alias of `--candidate-chunks`) — breaking; use `--candidate-chunks`, which now defaults to `3`. ## [v2.12.0] - 2026-08-02 diff --git a/README.md b/README.md index fb5313b..6b4a997 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ NoteBrain includes an [AI agent skill](wiki/Skill_Usage.md) and an [OpenCode Age - **Graph-Boosted Ranking**: Combine semantic similarity with graph relationships for better search results. - **Advanced Filtering**: Filter results by sections, tags, code blocks, tasks, and other metadata. - **Full Note Retrieval**: Get the complete note from the indexed content. -- **Reference Listing**: List a note's local attachments (images, PDFs, archives) and external website links with `notebrain refs`, filterable by kind — `--images`, `--pdf`, or `--external-links`. +- **Reference Listing**: List a note's local attachments (images, PDFs, archives) and external website links with `notebrain refs`, filterable by kind — `--only-images`, `--only-pdf`, or `--only-external-links`. - **Structured Output**: Export results as JSON or TSV. Use built-in JSONPath queries for automation. - **AI Agent Integration**: NoteBrain has a built-in AI agent skill for autonomous knowledge retrieval. - **Terminal Hyperlinks**: Use OSC 8 hyperlinks to open notes from supported terminals. @@ -85,7 +85,7 @@ This command starts an interactive wizard. The wizard configures your vault path notebrain ingest # To index PDFs, you must provide an LLM model and an API key (DEEPSEEK_API_KEY, OPENROUTER_API_KEY, etc.): -# OPENROUTER_API_KEY="sk-or-..." notebrain ingest --enable-pdf --llm-model "tencent/hy3" +# OPENROUTER_API_KEY="sk-or-..." notebrain ingest --with-pdf --llm-model "tencent/hy3" ``` > Note: The first indexing operation takes several minutes. The time depends on your vault size and the quantity of PDFs. @@ -139,7 +139,7 @@ SLUG=$(notebrain search "message broker" --limit 1 --jsonpath="$.results[0].note notebrain get "$SLUG" --jsonpath="$.text" # Fetch absolute paths of the note's image attachments -notebrain refs "$SLUG" --images --jsonpath='$.refs[*].path' +notebrain refs "$SLUG" --only-images --jsonpath='$.refs[*].path' ``` **7. Automate indexing:** Set a cron job or systemd timer to keep your index current. Read [Scheduled Ingestion](wiki/Scheduled_Ingestion.md). diff --git a/wiki/Architecture.md b/wiki/Architecture.md index 262a43d..8b0332a 100644 --- a/wiki/Architecture.md +++ b/wiki/Architecture.md @@ -180,7 +180,7 @@ This collection stores directed edges. These edges represent wikilinks between n 4. **Flat tag encoding**: The tool flattens array metadata into `tag_0` to `tag_19`. This keeps ChromaDB compatible with the Go client binding. The tag limit is 20 per note. 5. **Dummy 16-dimensional vectors for edges**: ChromaDB requires non-empty vectors. The `nb_links` collection uses 16-dimensional vectors in L2 space. The tool seeds each vector from the slug pair with FNV-32a. This reproduces identical vectors on re-ingest and prevents HNSW churn. 6. **Cached link resolver**: The tool builds a slug-to-slug resolver once per store lifetime. Graph commands reuse it instead of scanning the vault per command. `BatchIngest` rebuilds the resolver after each batch. `Reset` invalidates it. -7. **Single-scan exclude resolution**: The `--exclude-note` filter resolves all exclusions in one metadata scan. It also verifies the existence of each excluded note for the typo warning. +7. **Single-scan exclude resolution**: The `--exclude-notes` filter resolves all exclusions in one metadata scan. It also verifies the existence of each excluded note for the typo warning. 8. **Batch shared-tag search**: The `tags --shared` command runs one union scan over the seed tags. It accumulates per-note counts in Go. A per-tag scan costs one full vault walk per tag. 9. **FFI-safe pagination**: The embedded ChromaDB FFI caps responses at 1 MiB. The tool pages metadata fetches at 200 records. It caps semantic fetch limits at 100 results. A limit above 100 triggers a warning and truncation. 10. **TTY-aware output styling**: Text output uses ANSI colors only on an interactive terminal. The tool disables colors when `NO_COLOR` is set, when `TERM=dumb`, or when stdout is piped. Piped output stays machine-clean. The `[PDF]` tag renders in blue. diff --git a/wiki/Commands.md b/wiki/Commands.md index 447b3c6..fc33450 100644 --- a/wiki/Commands.md +++ b/wiki/Commands.md @@ -160,7 +160,7 @@ notebrain ingest [<glob>] [flags] | `--min-chunk-words` | `integer` | `10` | Does not include chunks that have fewer words than this value. | | `--chunk-size` | `integer` | `800` | The maximum number of runes per chunk for the parser. | | `--chunk-overlap` | `integer` | `100` | The number of overlap runes between sub-chunks when the parser splits a section. | -| `--enable-pdf` | `boolean` | `false` | Enables the extraction of PDF text. This requires `--llm-model`. | +| `--with-pdf` | `boolean` | `false` | Enables the extraction of PDF text. This requires `--llm-model`. Deprecated alias: `--enable-pdf`. | | `--llm-model` | `string` | `""` | The LLM model to parse PDFs (for example, `openrouter/anthropic/claude-3.5-haiku`). | | `--llm-context-window` | `integer` | `128000` | The total context window size of the LLM in tokens. | | `--respect-exclude` | `boolean` | `false` | Obeys the Obsidian user filters and attachment exclusions during ingestion. | @@ -216,7 +216,7 @@ notebrain search [<query>] [flags] | `--has-tasks` | `boolean` | `false` | Shows only the chunks that contain markdown task lists (`- [ ]`).| | `--has-code` | `boolean` | `false` | Shows only the chunks that contain code blocks. | | `--with-pdf` | `boolean` | `false` | Includes the PDF results in the search (the default is markdown only). | -| `--exclude-note` | `string` | _(None)_ | Excludes notes from the results. Accepts a note slug, title, or path; repeat the flag or use comma-separated values to exclude multiple notes. | +| `--exclude-notes` | `string` | _(None)_ | Excludes notes from the results. Accepts a note slug, title, or path; repeat the flag or use comma-separated values to exclude multiple notes. Deprecated alias: `--exclude-note`. | #### Examples @@ -234,11 +234,11 @@ notebrain search "message brokers" "redis queue" notebrain search "redis streams" --show-tags # Exclude private and archive notes from results (slug, title, or path) -notebrain search "reconciliation loop in kubernetes" --exclude-note "private/daily-journal" --exclude-note "archive" -notebrain search "redis queues" --exclude-note "zeta-note.md,beta.md" +notebrain search "reconciliation loop in kubernetes" --exclude-notes "private/daily-journal" --exclude-notes "archive" +notebrain search "redis queues" --exclude-notes "zeta-note.md,beta.md" # Text output notes that some notes were excluded -notebrain search "kubernetes" --exclude-note "archive" +notebrain search "kubernetes" --exclude-notes "archive" ``` #### How Multi-Query Matching and Ranking Works @@ -306,13 +306,13 @@ notebrain refs <note> [flags] | Flag | Description | | --- | --- | -| `--images` | Include image attachments only | -| `--pdf` | Include PDF attachments only | -| `--other` | Include other attachments (video, audio, archives, office docs) | -| `--external-links` | Include external website links (URLs) only | +| `--only-images` | Limit to image attachments only | +| `--only-pdf` | Limit to PDF attachments only | +| `--only-other` | Limit to other attachments (video, audio, archives, office docs) | +| `--only-external-links` | Limit to external website links (URLs) only | | `--include-missing` | Include references whose file is missing from the vault (marked `missing: true`) | -Filters combine with OR semantics; with no filter flags every kind is listed. +Flags combine with OR semantics; with no filter flags every kind is listed. The old names `--images`/`--pdf`/`--other`/`--external-links` still parse but are deprecated. #### Examples @@ -321,16 +321,16 @@ Filters combine with OR semantics; with no filter flags every kind is listed. notebrain refs "kubernetes-notes" # List image attachments as machine-readable JSON -notebrain refs "kubernetes-notes" --images --format=json +notebrain refs "kubernetes-notes" --only-images --format=json # List PDF attachments as TSV -notebrain refs "kubernetes-notes" --pdf --format=tsv +notebrain refs "kubernetes-notes" --only-pdf --format=tsv # List external website links -notebrain refs "kubernetes-notes" --external-links --format=json +notebrain refs "kubernetes-notes" --only-external-links --format=json # Feed attachment paths straight into a script -notebrain refs "$SLUG" --images --jsonpath='$.refs[*].path' +notebrain refs "$SLUG" --only-images --jsonpath='$.refs[*].path' ``` #### JSON shape @@ -425,8 +425,7 @@ notebrain hidden <note> [flags] | :----------------- | :-------- | :------ | :----------------------------------------------------------------------------------------------------------------------------------- | | `--deep` | `boolean` | `false` | Does a chunk-by-chunk analysis across individual note sections with the stored vectors. | | `--include-linked` | `boolean` | `false` | Includes notes that already have direct or indirect links in the hidden connections output. This strictly excludes self-references. | -| `--candidate-chunks` | `integer` | _(None)_ | The maximum number of matching target sections to evaluate and show for each candidate note (in `--deep` mode). Replaces `--top-k`. | -| `--top-k` | `integer` | `3` | A deprecated alias for `--candidate-chunks`. | +| `--candidate-chunks` | `integer` | `3` | The maximum number of matching target sections to evaluate and show for each candidate note (in `--deep` mode). | | `--limit` | `integer` | `10` | The maximum number of hidden connections to show. | #### How `--deep` Ranking Works diff --git a/wiki/PDF_Ingestion.md b/wiki/PDF_Ingestion.md index f17baad..63d27b5 100644 --- a/wiki/PDF_Ingestion.md +++ b/wiki/PDF_Ingestion.md @@ -22,12 +22,12 @@ NoteBrain detects the provider automatically. It uses the model prefix or the av ## How to Enable -To index PDFs during an ingestion run, provide the `--enable-pdf` and `--llm-model` flags: +To index PDFs during an ingestion run, provide the `--with-pdf` and `--llm-model` flags: ```bash export OPENROUTER_API_KEY="your-key-here" -notebrain ingest --enable-pdf --llm-model="tencent/hy3" +notebrain ingest --with-pdf --llm-model="tencent/hy3" ``` You can set these values permanently with the CLI wizard: @@ -45,7 +45,7 @@ llm_model = "tencent/hy3" ## Fallbacks and Cost Control -If you run `notebrain ingest` with `--enable-pdf` but your API key is missing, NoteBrain has a fallback process: +If you run `notebrain ingest` with `--with-pdf` but your API key is missing, NoteBrain has a fallback process: - NoteBrain prints a warning that PDF ingestion is disabled. - NoteBrain skips new or updated PDFs. From 1b37efa99cf779aa774420ea9ada1af2708e5438 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:07:19 +0530 Subject: [PATCH 16/18] chore(plan): mark consistency-pass tasks complete --- .agents/plans/Plan.md | 118 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/.agents/plans/Plan.md b/.agents/plans/Plan.md index e9a33f1..6a7120c 100644 --- a/.agents/plans/Plan.md +++ b/.agents/plans/Plan.md @@ -155,3 +155,121 @@ Out of scope: frontmatter references (wiki/markdown syntax only), audio/video su - None — the design tree is fully settled (grilling rounds 1-2 complete). - Non-blocking follow-ups (not part of v1): `--audio`/`--video` sub-filters (trivial enum extension), frontmatter reference scanning (needs per-field mapping rules), PDF page-anchor passthrough (`"anchor": "page=3"` in JSON), vault-only resolution without an index, and skill description trigger optimization via `scripts/run_loop.py` (skill-creator's Description Optimization step — 20 trigger queries; only if the user wants the description tuned beyond the manual wording update in Task 6). + +--- + +# Plan: Cross-command flag & output consistency pass + +All changes land on `feat/refs-command` (extending PR #37), after Task 7. Scope decision: **A + B + C all in this PR** (user decision — one review pass over a larger, but cohesive, diff). + +## Goal + +Eliminate flag/help-text confusion and output-format drift found by a full-codebase audit (3 exploration passes over `cmd/`, `internal/`, `internal/parser`, `internal/ingest`, `internal/store`). The `refs` feature introduced the worst offender — kind-filter flags advertised as additive ("include …") that actually restrict — and surfaced pre-existing inconsistencies across commands. + +## Current State (audit findings, with file refs) + +**Flag semantics — `refs` kind filters are restrictive but help says "include":** +- `cmd/refs.go:49-52` help: "include image attachments" (additive reading); `filterRefKinds` (`refs.go:243-265`): all-false → show everything; any-true → drop unselected kinds. Same command's `--include-missing` (`refs.go:53`) is truly additive → two semantic models in one command. +- `--images` (plural) vs kind value `"image"` (singular) and vs `--pdf`/`--other`/`--external-links` — plurality mismatch (`refs.go:49` vs kind table `refs.go:40-45`). + +**Same concept, multiple flag names (PDF):** +- `ingest --enable-pdf` (`cmd/ingest.go:40`), `search --with-pdf` (`cmd/search.go:44`), `boosted --with-pdf` (`cmd/boosted.go:37`), `refs --pdf` (`cmd/refs.go:50`). + +**Duplicate/alias flags:** +- `hidden --top-k` (deprecated) + `--candidate-chunks`; `--candidate-chunks 0` inexpressible (`hidden.go:45-48`). `--top-k` also means "chunks per note" in `search` (collision). +- `tags --for-note` alias of `--shared` (`tags.go:37`); global `--debug` alias of `--log-level=debug`; `--version` flag + `version` subcommand duplicate. +- `search --exclude-note` singular name for plural field `ExcludeNotes` (`search.go:45`). +- `get` positional `<SLUG>` vs `<NOTE>` everywhere else (`get.go:13` vs backlinks/connections/hidden/refs). + +**Output drift:** +- `refs` TSV: external rows emit blank `missing` column while JSON emits `false` (`refs.go:287-289`). +- Search TSV header `slug` vs `get` TSV `note_slug` (`print.go:187` vs `get.go:59`). +- `get` TSV unescaped (raw tabs/newlines) vs escaped everywhere else (`get.go:59-62`). +- Score: JSON 4dp rounded; TSV `%f` 6dp (`print.go:119` vs `:190`). +- JSONPath: `get`/`stats` extract from bare structs — `$.command` fails; search/refs/tags extract from envelopes (`get.go:40`, `stats.go:49`). +- `--jsonpath`/`--format json` silently ignored by ingest/reset/doctor/init/version/completion. +- `command` envelope values decorated (`"connections --hops 2"`, `"hidden --deep --include-linked"`) — consumers must whitelist; `emptyResultHint` relies on `HasPrefix`. +- `tags --list` text mode prints `#tag\t(2 notes)` — tab in human output (`print.go:469`). + +**Stringly-typed enum coupling (breakage risk):** +- `"pdf"` × 4 sites, 2 semantics: refs kind (`parser/attachments.go:21`, `cmd/refs.go:42`) vs `file_type` (`ingest/ingest.go:29`); kept apart only because refs refuses PDF notes (`refs.go:94-96`). +- `cmd/print.go:263` compares `r.FileType == kindPDF` (refs constant!) — works by coincidence. +- `"md"` literal at `store/query.go:1082` bypasses `fileTypeMD` (`ingest/ingest.go:28`). +- Vault-path UsageError duplicated verbatim (`ingest.go:49` == `refs.go:78`). +- Parser block-kind literals partially unconstantine (`ast.go:427` compares `"paragraph"` literally). +- Parser renderer labels non-image attachments `"attachment"` in chunk text vs `"other"` in refs — BUT stored chunk text change forces re-ingest; defer value change. + +**Authored decision (user):** flag renames happen with working deprecated aliases (hidden from `--help`, still parse); TSV `missing` emits `false`. + +## Decisions + +1. **`refs` kind filters rename to `--only-*`** — `--only-images`, `--only-pdf`, `--only-other`, `--only-external-links`; help text states "limit to … (no filter = all kinds; combine to union)". Old flags (`--images`/`--pdf`/`--other`/`--external-links`) become hidden deprecated aliases, still functional. Satisfies both the semantics lie and the plurality mismatch. +2. **PDF family unifies on `--with-pdf`** for ingest (search/boosted already use it) via rename + hidden deprecated alias `--enable-pdf`; config key `enable-pdf` keeps working (alias field), `with-pdf` added to config.example.toml. +3. **`hidden --top-k` removed; `--candidate-chunks` becomes the single flag** (gets `default:"3"`). Eliminates the cross-command `--top-k` collision with search. Breaking — flagged in CHANGELOG. +4. **`get` positional → `<NOTE>`** (field rename `Slug` → `Note`); help text already matches siblings. +5. **`search --exclude-note` → `--exclude-notes`** with hidden deprecated alias field merged in Run. +6. **TSV `missing` = `false` for external rows** — delete the blanking special case (`refs.go:287-289`); matches JSON. +7. **TSV column parity:** search TSV header `slug` → `note_slug`; `get` TSV fields routed through `tsvEscape`; score printed at 4dp in TSV (same value as JSON). +8. **JSONPath envelope parity:** `get` and `stats` apply JSONPath to their full envelope (so `$.command` works). Breaking for existing paths like `$.note_slug` on get → `$.note.note_slug`; documented in CHANGELOG. +9. **Ignored `--jsonpath`/`--format` on text-only commands:** stderr warning, no output change. +10. **Enum hygiene:** `FileTypeMD`/`FileTypePDF` constants live in `internal/store` (or ingest-exported; decided at implementation — whichever avoids import cycles; store imports nothing from ingest, so constants in store and referenced from ingest + query.go + print.go); `print.go:263` drops the accidental `kindPDF` reuse; parser block-kind consts applied; vault-path UsageError shared constant. Parser renderer `"attachment"` label: **comment + const only, value unchanged** (would force re-ingest of every vault). +11. **`--has-tasks`/`--has-code` AND semantics:** documented in help text + wiki, no behavior change. +12. **`command` envelope decorated values + `tags --list` tab:** documented in wiki/Commands.md JSON schema section; no code change (human-facing text; stable-enough contract). + +## Scope + +In scope: refs flag renames + aliases + help text, TSV `missing` fix, ingest `--with-pdf` rename + config key, hidden `--top-k` removal, get `<NOTE>`, search `--exclude-notes`, TSV/JSONPath parity (`get`, `stats`, search family), ignored-flag warnings, enum constant hygiene, tests for every change, docs (README, wiki, AGENTS.md, CHANGELOG, skill ×4, config.example.toml). + +Out of scope: `--debug`/`--for-note`/`--version` alias removals (harmless, users rely on them), parser renderer `"attachment"` value change, `command` envelope value normalization, tags text-mode tab, `--min-score` lexical-bypass bug (pre-existing behavior, separate fix), `--limit` default unification (tags 0 is intentional), `--skip-phantom` inverted default. + +## Tasks + +- [x] **Task A: `refs` kind filters → `--only-*` + TSV `missing` fix.** + `cmd/refs.go`: rename fields to `OnlyImages/OnlyPDF/OnlyOther/OnlyExternal` with `name:"only-…"`; add hidden deprecated alias fields (`Images/PDF/Other/ExternalLinks`, `hidden:""`, help "deprecated: use --only-…"); `filterRefKinds` reads new|old; reword main help ("limit to …; no filter = all kinds; combine filters to union"). Delete the external-blank special case in TSV (`refs.go:287-289`). Verify kong `hidden:""` on flags still parses (fallback: keep visible with "(deprecated)"). + (**Files:** `cmd/refs.go`, `cmd/refs_test.go`; **Verify:** legacy flags still filter; hidden from `--help`; external TSV row shows `false`; `go test -count=1 ./cmd/`.) + +- [x] **Task B: cross-command renames (PDF, hidden, get, search, ingest).** + - `cmd/ingest.go`: `EnablePDF` → `WithPDF` (`name:"with-pdf"`); alias field `EnablePDF` hidden/deprecated still wiring `pipeline.EnablePDF`. `config.example.toml`: add `with-pdf`, keep `enable-pdf` with deprecation comment. `init.go` wizard wording check. + - `cmd/hidden.go`: delete `TopK` + override merge; `CandidateChunks` gets `default:"3"`. + - `cmd/get.go`: field `Slug` → `Note`. + - `cmd/search.go`: `ExcludeNotes` gets `name:"exclude-notes"`; alias field `ExcludeNote` (name `exclude-note`) merged in Run. + (**Files:** the four cmd files + their tests, `config.example.toml`; **Verify:** flags parse under new names; aliases work; hidden `--top-k` rejected with clear kong error; `go test -count=1 ./cmd/ ./internal/configfile/`.) + +- [x] **Task C: output parity (TSV, JSONPath, ignored-flag warnings).** + - `cmd/print.go:187`: `slug` → `note_slug`; score `%f` → 4dp rounded value (match JSON). + - `cmd/get.go:59-62`: `tsvEscape` on text/tags/title columns. + - `cmd/get.go:39-41` + `cmd/stats.go:48-50`: JSONPath against the envelope struct, not the bare NoteContent/Stats. + - Also check refs TSV `kind` column: unescaped today (`refs.go:291` prints raw kind) — escape for parity or leave (enum-driven, safe); decide at implementation. + - New warning: in `runMain` or per text-only command, if `globals.Format != formatText || globals.JSONPath != ""` → stderr warning that the command ignores them (ingest/reset/doctor/init/version/completion). + (**Files:** `cmd/print.go`, `cmd/get.go`, `cmd/stats.go`, `cmd/cli.go` (+tests); **Verify:** TSV headers/escaping/score assertions updated; `$.command` works on get/stats JSONPath; warning appears on reset/doctor/ingest only once, stderr only.) + +- [x] **Task D: enum hygiene + docs sync.** + - Constants: `FileTypeMD`/`FileTypePDF` (home decided by import graph — likely `internal/store`); replace `"md"` at `store/query.go:1082`, `kindPDF` misuse at `cmd/print.go:263`, `.pdf`/`.md` suffix literals where a constant improves safety without churn. + - Parser block-kind consts: `ast.go:339,357,370,427,439,442`. + - Vault-path UsageError: shared constant (`ingest.go:49` + `refs.go:78`). + - Parser renderer label `"attachment"`: const + comment only (no value change — re-ingest risk). + - Docs: README.md:35,142, wiki/Commands.md:309-333, AGENTS.md flag-standards paragraph, CHANGELOG (`### Changed` renames table + `### Deprecated` aliases + breaking notes for `--top-k`, `--exclude-notes`, get JSONPath), skill files (SKILL.md:63,79, flags.md:83-87, example.md:27-28, schema.md:187-199 + TSV example row), `config.example.toml`. + (**Files:** listed above; **Verify:** `grep` shows no stale flag names in docs; `golangci-lint run ./...` clean (goconst); `go test -count=1 ./...`.) + +- [x] **Task E: full verification + review.** + `make test`, `make lint`, rebuild binary (`make build` + `cp notebrain ~/.local/bin/`), manual probes: new flags, alias flags, hidden `--top-k` rejection, `--exclude-notes`, get `<NOTE>`, `--with-pdf` flag+config key, get/stats `$.command` JSONPath, TSV headers (search `note_slug`, refs external `false`, get escaped). Then `/code-review` over `master...HEAD` (standards + spec axes), apply fixes, route the final commit batch through the `git-commiter` skill in groups (feat/fix × code files, docs, config). + (**Verify:** `git log master..HEAD` shows only feature+consistency commits; review findings resolved before merge; merge to master remains the user's call.) + +## Verification + +- `go test -count=1 ./...` green; `golangci-lint run ./...` 0 issues. +- `notebrain refs --help` shows `--only-*` with "limit to" wording, no `--images`/`--pdf`/`--other`/`--external-links`; legacy flags still parse and filter correctly (checked via CLI probe, not just unit test). +- `notebrain refs … --format tsv` external row: `false` in the `missing` column. +- `notebrain ingest --with-pdf` and config key `with-pdf` both work; `enable-pdf` config key still honored (deprecated). +- `notebrain hidden … --top-k 5` → kong error (flag removed); `--candidate-chunks` works. +- `notebrain search … --exclude-notes x` works; `--exclude-note` still accepted (deprecated). +- `notebrain get <note> --jsonpath='$.command'` → `get`; `--jsonpath='$.note.note_slug'` works. +- TSV: search header `note_slug`; get TSV survives multiline/tab text (single parseable row); scores at 4dp. +- `notebrain reset --format json` warns on stderr; stdout unchanged. +- Docs: zero stale references to old flag names (`rg -- '--images|--enable-pdf|--top-k|--exclude-note|SLUG'` across README/wiki/AGENTS.md/skill/CHANGELOG/config.example.toml, allowing explicit "deprecated:" notes). +- Skill evals 17/18 (refs scenarios) use `--only-*` names only if the fixture asserts flag names — regenerate fixture-based assertions if needed, rerun affected gradings; iteration-4 benchmark stays green. + +## Open Questions (non-blocking) + +- Long flag `--only-external-links` — accepted as-is (matches the kind value `external-links`; no shortcut alias in v1). +- Parser renderer `"attachment"` label — value change deferred forever unless a future `chunkSchemaVersion` bump justifies re-ingest; documented via const + comment. (This decision matches the earlier draft; no change unless user insists.) From 50dec477ce3413bf97d4444413ba095461ce7167 Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:16:32 +0530 Subject: [PATCH 17/18] fix(config): single global with-pdf key, wizard targets it - config keys are flat (not section-scoped), so the template advertised two independent with-pdf keys in different sections; uncommenting the search one collided with the ingest one at parse time (duplicate TOML key) and implied search PDFs could be toggled separately - search section now documents that with-pdf is the same global key as ingest's; one value governs both ingest and search - init wizard comment clarifies its replace targets that single key --- cmd/init.go | 4 +++- config.example.toml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/init.go b/cmd/init.go index 0b941bd..12bb190 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -91,7 +91,9 @@ func (c *InitCmd) Run(globals *Globals) error { newVaultLine := fmt.Sprintf(`vault-path = %q`, vaultPath) configStr = strings.Replace(configStr, targetVaultLine, newVaultLine, 1) - // Replace PDF flag + // Replace PDF flag. The target line is the single with-pdf key in the + // Ingestion Pipeline Settings section; config keys are flat, so this one + // key governs both ingest and search. if enablePDF { configStr = strings.Replace(configStr, "# with-pdf = false", "with-pdf = true", 1) } diff --git a/config.example.toml b/config.example.toml index 417424c..bec2f6f 100644 --- a/config.example.toml +++ b/config.example.toml @@ -90,6 +90,8 @@ # ------------------------------------------------------------------------------ # Include PDF text extraction results in semantic search (notebrain search / boosted). +# Note: this is the same global with-pdf key as in Ingestion Pipeline Settings above; +# one value governs both ingest and search (config keys are flat, not section-scoped). # with-pdf = false # default: false # Include matched markdown text snippets in search results. From e94e07901abde2946a50287b6b2c94b674ba3cef Mon Sep 17 00:00:00 2001 From: nmdra <nimendradilshan11@gmail.com> Date: Thu, 13 Aug 2026 13:45:40 +0530 Subject: [PATCH 18/18] feat(cmd): style refs text output with house rendering - note title header banner, per-kind colored [kind] chips (images accent, PDFs blue, other muted, external links bold), amber (missing) markers via warnBoldStyle, and a hintStyle empty state - OSC 8 hyperlinks when the terminal supports them: attachments open obsidian:// URIs, external links open their URL; Ctrl+click footer - attachment rows display vault-relative paths instead of absolute ones to reduce clutter (external links keep their URL; JSON/TSV keep absolute paths unchanged); hyperlink targets unaffected - rows truncated to terminal width with ansi.Truncate - tests: new plain/colored/empty render tests mirroring the search PDF-tag pattern; text assertions updated to relative paths --- CHANGELOG.md | 1 + cmd/refs.go | 66 ++++++++++++++++++++++++++++-- cmd/refs_render_test.go | 90 +++++++++++++++++++++++++++++++++++++++++ cmd/refs_test.go | 37 +++++++++-------- 4 files changed, 173 insertions(+), 21 deletions(-) create mode 100644 cmd/refs_render_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 756ec2a..f655811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **TSV parity**: `search` TSV header `slug` → `note_slug`; `get` TSV fields escaped; scores at 4 decimal places in TSV, matching JSON (`fix(output)`). - **Refs TSV**: external-link rows now emit `false` in the `missing` column instead of a blank cell (`fix(output)`). - **JSONPath envelope parity**: `get` and `stats` apply `--jsonpath` to their full command envelope; `$.command` now works, and `get` paths shift from `$.note_slug` to `$.note.note_slug` (`fix(output)`). +- **Refs text styling**: `refs` text output now matches the house style — note title header, per-kind colored `[kind]` chips (images, PDFs, other, external links), amber `(missing)` markers, clickable links (obsidian:// for attachments, the URL for external links), vault-relative paths instead of absolute ones, and terminal-width truncation (`feat(cmd)`). ### Deprecated - `refs --images`/`--pdf`/`--other`/`--external-links` — use `--only-*` (aliases still parse, hidden from `--help`). diff --git a/cmd/refs.go b/cmd/refs.go index 4956e48..9ee0894 100644 --- a/cmd/refs.go +++ b/cmd/refs.go @@ -32,6 +32,9 @@ import ( "strconv" "strings" + "charm.land/lipgloss/v2" + "github.com/charmbracelet/x/ansi" + "github.com/nmdra/notebrain-cli/v2/internal/ingest" "github.com/nmdra/notebrain-cli/v2/internal/parser" ) @@ -318,17 +321,72 @@ func printRefsFormattedToWriter(w io.Writer, env refsEnvelope, globals *Globals) } return nil default: // "text" + initStyles() + title := env.Title + if title == "" { + title = env.NoteSlug + } + _, _ = fmt.Fprintln(w, headerStyle.Render(title)) + if len(env.Refs) == 0 { - _, _ = fmt.Fprintln(w, "No references found") + _, _ = fmt.Fprintln(w, hintStyle.Render(" No references found")) return nil } + + termWidth := getTerminalWidth() + useLinks := hyperlinkSupported() && globals.ShowFilePath + for _, r := range env.Refs { - marker := "" + chip := refKindChipStyle(r.Kind).Render("[" + r.Kind + "]") + + // Text mode shows vault-relative paths (no base-vault clutter); + // external links keep their URL, which is the path itself. + display := r.Path + if r.Kind != kindExternal && r.RelativePath != "" { + display = r.RelativePath + } + + path := display + if useLinks { + switch r.Kind { + case kindExternal: + path = hyperlink(true, r.Path, display) + default: + if r.RelativePath != "" { + path = hyperlink(true, ObsidianURI(globals.VaultName, r.RelativePath), display) + } + } + } + line := fmt.Sprintf("%s %s", chip, path) if r.Missing { - marker = " (missing)" + line += " " + warnBoldStyle.Render("(missing)") + } + if termWidth > 0 && ansi.StringWidth(line) > termWidth { + line = ansi.Truncate(line, termWidth, "…") } - _, _ = fmt.Fprintf(w, "[%s] %s%s\n", r.Kind, r.Path, marker) + _, _ = fmt.Fprintln(w, line) + } + + if useLinks { + _, _ = fmt.Fprintln(w, "\n "+extraStyle.Render("(Ctrl+click / Cmd+click a reference to open it)")) } return nil } } + +// refKindChipStyle returns the style for a ref kind chip in text output: +// images use the accent, PDFs the blue tag used by search, other +// attachments the muted gray, and external links the bold label. +func refKindChipStyle(kind string) lipgloss.Style { + initStyles() + switch kind { + case kindImage: + return titleStyle + case kindPDF: + return pdfTagStyle + case kindOther: + return metaStyle + default: // kindExternal + return labelStyle + } +} diff --git a/cmd/refs_render_test.go b/cmd/refs_render_test.go new file mode 100644 index 0000000..f8b8e02 --- /dev/null +++ b/cmd/refs_render_test.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "bytes" + "strings" + "sync" + "testing" +) + +func TestPrintRefsFormatted_Rendering(t *testing.T) { + env := refsEnvelope{ + Command: groupRefs, + NoteSlug: "router", + Title: "Router", + Total: 3, + Refs: []refEntry{ + {Path: "/vault/Notes/cover.png", RelativePath: "Notes/cover.png", Kind: kindImage}, + {Path: "/vault/Attachments/att.pdf", RelativePath: "Attachments/att.pdf", Kind: kindPDF, Missing: true}, + {Path: "https://example.com/docs", Kind: kindExternal}, + }, + } + globals := &Globals{Format: formatText} + + t.Run("plain", func(t *testing.T) { + stdoutColorEnabled = stdoutAllowsColor + stylesOnce = sync.Once{} + + var buf bytes.Buffer + if err := printRefsFormattedToWriter(&buf, env, globals); err != nil { + t.Fatalf("printRefsFormattedToWriter: %v", err) + } + out := buf.String() + if strings.Contains(out, "\x1b[") { + t.Errorf("unexpected ANSI codes in plain output: %q", out) + } + if !strings.Contains(out, "[image] Notes/cover.png") { + t.Errorf("expected image row, got: %q", out) + } + if !strings.Contains(out, "[pdf] Attachments/att.pdf (missing)") { + t.Errorf("expected missing pdf row, got: %q", out) + } + if !strings.Contains(out, "[external-links] https://example.com/docs") { + t.Errorf("expected external link row, got: %q", out) + } + if !strings.Contains(out, "Router") { + t.Errorf("expected note title header, got: %q", out) + } + }) + + t.Run("colored", func(t *testing.T) { + old := stdoutColorEnabled + stdoutColorEnabled = func() bool { return true } + stylesOnce = sync.Once{} + defer func() { + stdoutColorEnabled = old + stylesOnce = sync.Once{} + }() + + var buf bytes.Buffer + if err := printRefsFormattedToWriter(&buf, env, globals); err != nil { + t.Fatalf("printRefsFormattedToWriter: %v", err) + } + out := buf.String() + if !strings.Contains(out, "\x1b[") { + t.Errorf("expected ANSI codes in colored output: %q", out) + } + for _, want := range []string{"[image]", "[pdf]", "[external-links]", "(missing)", "Router"} { + if !strings.Contains(out, want) { + t.Errorf("colored output missing %q: %q", want, out) + } + } + }) + + t.Run("empty", func(t *testing.T) { + stdoutColorEnabled = stdoutAllowsColor + stylesOnce = sync.Once{} + + var buf bytes.Buffer + empty := env + empty.Refs = nil + empty.Total = 0 + if err := printRefsFormattedToWriter(&buf, empty, globals); err != nil { + t.Fatalf("printRefsFormattedToWriter: %v", err) + } + out := buf.String() + if !strings.Contains(out, "No references found") { + t.Errorf("expected empty hint, got: %q", out) + } + }) +} diff --git a/cmd/refs_test.go b/cmd/refs_test.go index f228155..dd05def 100644 --- a/cmd/refs_test.go +++ b/cmd/refs_test.go @@ -65,12 +65,12 @@ func TestRefsText(t *testing.T) { }) wantLines := []string{ - "[image] " + filepath.Join(vaultDir, "Notes", "cover.png"), - "[image] " + filepath.Join(vaultDir, "assets", "arch.png"), - "[image] " + filepath.Join(vaultDir, "Notes", "local.png"), - "[image] " + filepath.Join(vaultDir, "Notes", "Router Modes.webp"), - "[pdf] " + filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf"), - "[image] " + filepath.Join(vaultDir, "Notes", "a.png"), + "[image] Notes/cover.png", + "[image] assets/arch.png", + "[image] Notes/local.png", + "[image] Notes/Router Modes.webp", + "[pdf] 99.Storage-Shed/Attachments/att.pdf", + "[image] Notes/a.png", "[external-links] https://example.com/docs", "[external-links] https://links.example.com", } @@ -79,6 +79,9 @@ func TestRefsText(t *testing.T) { t.Errorf("text output missing %q:\n%s", want, out) } } + if !strings.Contains(out, "Router") { + t.Errorf("text output missing the note title header:\n%s", out) + } if strings.Contains(out, "broken.png") { t.Errorf("missing reference shown without --include-missing:\n%s", out) } @@ -109,13 +112,13 @@ func TestRefsFilters(t *testing.T) { { name: "images only", cmd: RefsCmd{Note: "router", OnlyImages: true}, - include: []string{filepath.Join(vaultDir, "Notes", "cover.png")}, + include: []string{"Notes/cover.png"}, exclude: []string{"att.pdf", "https://example.com/docs", "[external-links]"}, }, { name: "pdf only", cmd: RefsCmd{Note: "router", OnlyPDF: true}, - include: []string{filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, + include: []string{"99.Storage-Shed/Attachments/att.pdf"}, exclude: []string{"cover.png", "https://example.com/docs"}, }, { @@ -127,25 +130,25 @@ func TestRefsFilters(t *testing.T) { { name: "combined or", cmd: RefsCmd{Note: "router", OnlyImages: true, OnlyPDF: true}, - include: []string{filepath.Join(vaultDir, "Notes", "cover.png"), filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, + include: []string{"Notes/cover.png", "99.Storage-Shed/Attachments/att.pdf"}, exclude: []string{"https://example.com"}, }, { name: "deprecated images alias", cmd: RefsCmd{Note: "router", Images: true}, - include: []string{filepath.Join(vaultDir, "Notes", "cover.png")}, + include: []string{"Notes/cover.png"}, exclude: []string{"att.pdf", "https://example.com/docs", "[external-links]"}, }, { name: "deprecated pdf alias", cmd: RefsCmd{Note: "router", PDF: true}, - include: []string{filepath.Join(vaultDir, "99.Storage-Shed", "Attachments", "att.pdf")}, + include: []string{"99.Storage-Shed/Attachments/att.pdf"}, exclude: []string{"cover.png", "https://example.com/docs"}, }, { name: "deprecated mixed aliases", cmd: RefsCmd{Note: "router", Images: true, OnlyExternal: true}, - include: []string{filepath.Join(vaultDir, "Notes", "cover.png"), "[external-links] https://example.com/docs"}, + include: []string{"Notes/cover.png", "[external-links] https://example.com/docs"}, exclude: []string{"att.pdf"}, }, } @@ -175,7 +178,7 @@ func TestRefsMissingHiddenUnlessFlagged(t *testing.T) { fs := &fakeStore{noteMeta: &store.NoteContent{NoteSlug: "router", Title: "Router", FilePath: "Notes/router.md"}} withFakeStore(t, fs) - missingPath := filepath.Join(vaultDir, "Notes", "broken.png") + missingPath := "Notes/broken.png" out := captureStdout(t, func() { if err := (&RefsCmd{Note: "router"}).Run(refsTestGlobals(vaultDir)); err != nil { @@ -249,9 +252,9 @@ func TestRefsCrossKindOrder(t *testing.T) { }) wantLines := []string{ "[external-links] https://example.com/docs", - "[image] " + filepath.Join(vaultDir, "Notes", "cover.png"), + "[image] " + "Notes/cover.png", "[external-links] https://links.example.com", - "[image] " + filepath.Join(vaultDir, "Notes", "second.png"), + "[image] Notes/second.png", } idxs := make([]int, len(wantLines)) for i, want := range wantLines { @@ -362,7 +365,7 @@ func TestRefsJSONPath(t *testing.T) { t.Errorf("Run: %v", err) } }) - want := filepath.Join(vaultDir, "Notes", "cover.png") + want := "Notes/cover.png" if !strings.Contains(out, want) { t.Errorf("jsonpath output missing %q:\n%s", want, out) } @@ -565,7 +568,7 @@ func TestPrintRefsFormattedToWriter(t *testing.T) { if err := printRefsFormattedToWriter(&sb, env, globals); err != nil { t.Fatal(err) } - want := "[image] /vault/Notes/cover.png\n[external-links] https://example.com\n" + want := "Router\n \n──────\n[image] Notes/cover.png\n[external-links] https://example.com\n" if sb.String() != want { t.Errorf("text = %q, want %q", sb.String(), want) }