feat: Phase 4+5 UI merge + ops/retrieval quality

Ships the user-visible unification (one SPA for docs + notes) plus
backend observability and retrieval improvements. Delivered in
parallel by two sub-agents with no cross-conflicts.

Phase 4 — UI merge (no new npm deps):
- ui/src/components/notes/: FolderTree, NoteView (inline md+wikilinks
  renderer), NoteEditor, LinkPanel, NotesGraphView
- ui/src/components/shared/UnifiedSearchPanel — single query hits
  both /api/search (docs) and /api/projects/{p}/search (notes);
  results tagged [doc] / [entity] / [note]
- ui/src/hooks/useNotes.ts, useProjects.ts with ?project= URL sync
- TopNav: Notes tab + project selector dropdown
- App: 3-col notes view (tree | view/editor+graph | links)
- GraphView: optional notes-graph overlay toggle
- Design tokens: --accent-notes hue for distinguishing note nodes
- Rebuilt ui/dist/: 312 kB JS (94 kB gzip) + 10 kB CSS
- Backend hook: 20-line GET /api/projects handler

Phase 5 — Ops + retrieval (pure-Go, no CGO, no new deps):
- internal/api/claims_handlers.go + MCP get_entity_claims tool +
  store.ClaimsForEntity / ListClaims — claims finally surfaced
- internal/api/metrics.go — Prometheus text format, stdlib-only:
  docsiq_requests_total, request_duration_seconds, projects_total,
  notes_total, build_info. Public /metrics endpoint.
- internal/api/request_id.go — middleware generates crypto/rand ID
  (or passes through X-Request-ID), sets response header, exposes
  RequestIDFromContext
- cmd --log-format=text|json (DOCSIQ_LOG_FORMAT) for structured
  logging in production
- internal/pipeline incremental re-index: SHA-256 + mtime short-
  circuit (skips unchanged files), new indexed_mtime column,
  Prune() for deleted files, docsiq index --prune flag
- Per-project LLM override: cfg.LLMOverrides + LLMConfigForProject
  + ProviderForProject helper (data layer; incremental rollout
  into search handlers deferred)

Test baseline: 362 → 405 subtests (+43). All packages green.
No sqlite-vec / CGO introduced — stays pure-Go as planned for
Phases 0-5.

Known scope limits:
- Per-project LLM override plumbing into search is the helper
  only; every LocalSearch/GlobalSearch callsite still uses the
  root provider (intentional — broader refactor)
- Pure-Go HNSW vector index deferred alongside sqlite-vec to
  whenever CGO becomes acceptable
- FolderTree lacks new-note UX; notes must be created via API/MCP

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: aksOps

cmd/index.go

@@ -20,6 +20,7 @@ var (
 	indexMaxPages    int
 	indexMaxDepth    int
 	indexSkipSitemap bool
+	indexPrune       bool
 )
 
 var indexCmd = &cobra.Command{
@@ -48,6 +49,21 @@ var indexCmd = &cobra.Command{
 			return nil
 		}
 
+		if indexPrune {
+			// Prune does not need an LLM provider. Opening one here would
+			// force the user to have valid LLM credentials just to remove
+			// dangling rows — explicitly skip NewProvider.
+			pl := pipeline.New(st, nil, cfg)
+			slog.Info("🗑️ pruning documents whose source files are missing")
+			n, err := pl.Prune(cmd.Context())
+			if err != nil {
+				slog.Error("❌ prune failed", "err", err)
+				return err
+			}
+			slog.Info("✅ prune complete", "removed", n)
+			return nil
+		}
+
 		prov, err := llm.NewProvider(&cfg.LLM)
 		if err != nil {
 			return fmt.Errorf("llm provider: %w", err)
@@ -98,5 +114,6 @@ func init() {
 	indexCmd.Flags().IntVar(&indexMaxPages, "max-pages", 500, "Maximum pages to crawl (0 = unlimited)")
 	indexCmd.Flags().IntVar(&indexMaxDepth, "max-depth", 0, "Maximum BFS link depth (0 = unlimited)")
 	indexCmd.Flags().BoolVar(&indexSkipSitemap, "skip-sitemap", false, "Force BFS crawl even if sitemap.xml exists")
+	indexCmd.Flags().BoolVar(&indexPrune, "prune", false, "Remove documents whose source files no longer exist on disk")
 }
 

cmd/logformat_test.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"log/slog"
+	"testing"
+)
+
+// TestLogFormatJSON is a compact smoke-test for the JSON handler — the
+// real --log-format wiring lives in initConfig which hits os.Stderr +
+// viper, neither of which is worth faking here. Exercising the handler
+// directly is enough to catch regressions where we accidentally write a
+// text handler for format=json.
+func TestLogFormatJSON(t *testing.T) {
+	var buf bytes.Buffer
+	logger := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
+	logger.Info("hello", "k", "v")
+
+	var decoded map[string]any
+	if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil {
+		t.Fatalf("log output is not valid JSON: %v\n%s", err, buf.String())
+	}
+	if decoded["msg"] != "hello" {
+		t.Errorf("msg = %v, want hello", decoded["msg"])
+	}
+	if decoded["k"] != "v" {
+		t.Errorf("k = %v, want v", decoded["k"])
+	}
+}

cmd/root.go

@@ -4,15 +4,17 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"strings"
 
 	"github.com/RandomCodeSpace/docscontext/internal/config"
 	"github.com/spf13/cobra"
 )
 
 var (
-	cfgFile  string
-	cfg      *config.Config
-	logLevel string
+	cfgFile   string
+	cfg       *config.Config
+	logLevel  string
+	logFormat string
 )
 
 var rootCmd = &cobra.Command{
@@ -35,6 +37,7 @@ func init() {
 	cobra.OnInitialize(initConfig)
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default ~/.docscontext/config.yaml or ~/.DocsContext/config.yaml)")
 	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "Log level: debug, info, warn, error")
+	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "", "Log format: text|json (env DOCSIQ_LOG_FORMAT; default text)")
 }
 
 func initConfig() {
@@ -50,7 +53,20 @@ func initConfig() {
 	default:
 		level = slog.LevelInfo
 	}
-	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})))
+	// Log format: --log-format wins, else DOCSIQ_LOG_FORMAT, else "text".
+	format := strings.ToLower(strings.TrimSpace(logFormat))
+	if format == "" {
+		format = strings.ToLower(strings.TrimSpace(os.Getenv("DOCSIQ_LOG_FORMAT")))
+	}
+	handlerOpts := &slog.HandlerOptions{Level: level}
+	var handler slog.Handler
+	switch format {
+	case "json":
+		handler = slog.NewJSONHandler(os.Stderr, handlerOpts)
+	default:
+		handler = slog.NewTextHandler(os.Stderr, handlerOpts)
+	}
+	slog.SetDefault(slog.New(handler))
 
 	var err error
 	cfg, err = config.Load(cfgFile)

internal/api/claims_handlers.go

@@ -0,0 +1,35 @@
+package api
+
+import (
+	"net/http"
+)
+
+// claimsForEntity returns all claims attached to the requested entity ID.
+// Unknown entity → 200 with empty array; the client can treat that the
+// same way whether the entity truly has no claims or was never indexed.
+func (h *handlers) claimsForEntity(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if id == "" {
+		writeError(w, r, 400, "id required", nil)
+		return
+	}
+	claims, err := h.store.ClaimsForEntity(r.Context(), id)
+	if err != nil {
+		writeError(w, r, 500, err.Error(), err)
+		return
+	}
+	writeJSON(w, 200, claims)
+}
+
+// listClaims returns claims with optional ?status= and ?limit= filters.
+func (h *handlers) listClaims(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	status := q.Get("status")
+	limit := intQuery(q.Get("limit"), 100)
+	claims, err := h.store.ListClaims(r.Context(), status, limit)
+	if err != nil {
+		writeError(w, r, 500, err.Error(), err)
+		return
+	}
+	writeJSON(w, 200, claims)
+}

internal/api/claims_handlers_test.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+
+	"github.com/RandomCodeSpace/docscontext/internal/config"
+	"github.com/RandomCodeSpace/docscontext/internal/store"
+)
+
+// newClaimsRouter builds a router with a seeded store that has entities
+// and claims, so the REST layer can be exercised end-to-end.
+func newClaimsRouter(t *testing.T) http.Handler {
+	t.Helper()
+	dir := t.TempDir()
+	st, err := store.Open(filepath.Join(dir, "claims_router.db"))
+	if err != nil {
+		t.Fatalf("store.Open: %v", err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+
+	ctx := context.Background()
+	if err := st.UpsertDocument(ctx, &store.Document{
+		ID: "d1", Path: "/tmp/d1.md", DocType: "md", FileHash: "d1h", IsLatest: true,
+	}); err != nil {
+		t.Fatalf("UpsertDocument: %v", err)
+	}
+	if err := st.UpsertEntity(ctx, &store.Entity{ID: "e1", Name: "Alpha"}); err != nil {
+		t.Fatalf("UpsertEntity: %v", err)
+	}
+	claims := []*store.Claim{
+		{ID: "c1", EntityID: "e1", Claim: "fact-1", Status: "verified", DocID: "d1"},
+		{ID: "c2", EntityID: "e1", Claim: "fact-2", Status: "pending", DocID: "d1"},
+	}
+	if err := st.BatchInsertClaims(ctx, claims); err != nil {
+		t.Fatalf("BatchInsertClaims: %v", err)
+	}
+
+	cfg := &config.Config{}
+	cfg.DataDir = dir
+	return NewRouter(st, nil, nil, cfg, nil)
+}
+
+func TestClaimsHandlers(t *testing.T) {
+	t.Run("claims_for_entity_happy_path", func(t *testing.T) {
+		h := newClaimsRouter(t)
+		req := httptest.NewRequest(http.MethodGet, "/api/entities/e1/claims", nil)
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("status = %d, want 200", rec.Code)
+		}
+		var claims []store.Claim
+		if err := json.NewDecoder(rec.Body).Decode(&claims); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		if len(claims) != 2 {
+			t.Errorf("len = %d, want 2", len(claims))
+		}
+	})
+
+	t.Run("claims_for_unknown_entity_returns_empty_array", func(t *testing.T) {
+		h := newClaimsRouter(t)
+		req := httptest.NewRequest(http.MethodGet, "/api/entities/nope/claims", nil)
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("status = %d, want 200", rec.Code)
+		}
+		body := rec.Body.String()
+		if body != "[]\n" {
+			t.Errorf("body = %q, want %q", body, "[]\n")
+		}
+	})
+
+	t.Run("list_claims_status_filter", func(t *testing.T) {
+		h := newClaimsRouter(t)
+		req := httptest.NewRequest(http.MethodGet, "/api/claims?status=verified", nil)
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("status = %d, want 200", rec.Code)
+		}
+		var claims []store.Claim
+		if err := json.NewDecoder(rec.Body).Decode(&claims); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		if len(claims) != 1 {
+			t.Errorf("len = %d, want 1", len(claims))
+		}
+		if len(claims) > 0 && claims[0].Status != "verified" {
+			t.Errorf("status = %q, want verified", claims[0].Status)
+		}
+	})
+
+	t.Run("list_claims_limit_bound", func(t *testing.T) {
+		h := newClaimsRouter(t)
+		req := httptest.NewRequest(http.MethodGet, "/api/claims?limit=1", nil)
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("status = %d, want 200", rec.Code)
+		}
+		var claims []store.Claim
+		if err := json.NewDecoder(rec.Body).Decode(&claims); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		if len(claims) != 1 {
+			t.Errorf("len = %d, want 1", len(claims))
+		}
+	})
+}

internal/api/handlers.go

@@ -19,6 +19,7 @@ import (
 	"github.com/RandomCodeSpace/docscontext/internal/embedder"
 	"github.com/RandomCodeSpace/docscontext/internal/llm"
 	"github.com/RandomCodeSpace/docscontext/internal/pipeline"
+	"github.com/RandomCodeSpace/docscontext/internal/project"
 	"github.com/RandomCodeSpace/docscontext/internal/search"
 	"github.com/RandomCodeSpace/docscontext/internal/store"
 )
@@ -55,6 +56,35 @@ func (h *handlers) health(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, 200, map[string]string{"status": "ok"})
 }
 
+// projectsHandler is a thin read-only JSON shim around registry.List()
+// so the Phase-4 UI can populate its project-selector dropdown.
+type projectsHandler struct {
+	registry *project.Registry
+}
+
+// listProjects returns registered projects as a JSON array. Falls back
+// to [{"slug":"_default","name":"_default"}] when the registry is nil
+// or empty so the UI always has a usable default selection.
+func (p *projectsHandler) listProjects(w http.ResponseWriter, r *http.Request) {
+	type projInfo struct {
+		Slug string `json:"slug"`
+		Name string `json:"name"`
+	}
+	out := []projInfo{}
+	if p.registry != nil {
+		projs, err := p.registry.List()
+		if err == nil {
+			for _, pr := range projs {
+				out = append(out, projInfo{Slug: pr.Slug, Name: pr.Name})
+			}
+		}
+	}
+	if len(out) == 0 {
+		out = append(out, projInfo{Slug: "_default", Name: "_default"})
+	}
+	writeJSON(w, 200, out)
+}
+
 func (h *handlers) getStats(w http.ResponseWriter, r *http.Request) {
 	stats, err := h.store.GetStats(r.Context())
 	if err != nil {