diff --git a/AGENT.md b/AGENT.md index 65af928..3adc847 100644 --- a/AGENT.md +++ b/AGENT.md @@ -38,6 +38,8 @@ Trindex is architecturally inspired by Nate B. Jones's **OpenBrain** guide (Post ## Environment Configuration +### Core Settings + ```env # Postgres DATABASE_URL=postgres://trindex:trindex@localhost:5432/trindex?sslmode=disable @@ -46,30 +48,71 @@ DATABASE_URL=postgres://trindex:trindex@localhost:5432/trindex?sslmode=disable EMBED_BASE_URL=http://localhost:11434/v1 EMBED_MODEL=nomic-embed-text EMBED_API_KEY=ollama -EMBED_DIMENSIONS=1536 +EMBED_DIMENSIONS=768 # MCP TRANSPORT=stdio +``` -# HNSW index tuning -HNSW_M=16 -HNSW_EF_CONSTRUCTION=64 -HNSW_EF_SEARCH=40 +### Search Configuration + +```env +# Hybrid search weights (must sum to 1.0) +HYBRID_VECTOR_WEIGHT=0.7 # Vector search weight +HYBRID_FTS_WEIGHT=0.3 # Full-text search weight # Recall defaults DEFAULT_NAMESPACE=default DEFAULT_TOP_K=10 DEFAULT_SIMILARITY_THRESHOLD=0.7 -# Phase 2 (HTTP/SSE + Web UI) -# HTTP_PORT=9636 -# HTTP_HOST=0.0.0.0 +# HNSW index tuning +HNSW_M=16 +HNSW_EF_CONSTRUCTION=64 +HNSW_EF_SEARCH=40 +``` + +### TTL and Deduplication + +```env +# Default TTL for session namespaces (seconds) +DEFAULT_SESSION_TTL=86400 # 24 hours + +# Deduplication threshold (0.0-1.0, higher = more strict) +DEFAULT_DEDUP_THRESHOLD=0.95 # 0.95 for exact, 0.85 for semantic +``` + +### HTTP Server (Phase 2) + +```env +HTTP_HOST=0.0.0.0 +HTTP_PORT=9636 +TRINDEX_API_KEY=change-me-in-production +``` + +### Database Connection Pool + +```env +DB_MAX_CONNS=25 +DB_MIN_CONNS=5 +DB_MAX_CONN_LIFETIME_MINUTES=60 +DB_MAX_CONN_IDLE_TIME_MINUTES=30 +``` + +### Embedding Client + +```env +EMBED_MAX_RETRIES=3 +EMBED_RETRY_DELAY_MS=1000 +EMBED_REQUEST_TIMEOUT_SEC=30 ``` --- ## Database Schema +### Current Schema (v2) + ```sql -- Enable extensions CREATE EXTENSION IF NOT EXISTS vector; @@ -77,14 +120,17 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm; -- Memories table CREATE TABLE IF NOT EXISTS memories ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - namespace TEXT NOT NULL DEFAULT 'default', - content TEXT NOT NULL, - embedding VECTOR(1536), - metadata JSONB DEFAULT '{}', - search_vec TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + namespace TEXT NOT NULL DEFAULT 'default', + content TEXT NOT NULL, + content_hash TEXT NOT NULL, -- NEW: SHA-256 hash for deduplication + embedding VECTOR(1536), + metadata JSONB DEFAULT '{}', + search_vec TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED, + ttl_seconds INTEGER DEFAULT 0, -- NEW: TTL in seconds (0 = no expiry) + expires_at TIMESTAMPTZ, -- NEW: Expiration timestamp + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -- HNSW vector index (cosine distance) @@ -105,9 +151,24 @@ CREATE INDEX IF NOT EXISTS memories_metadata_idx CREATE INDEX IF NOT EXISTS memories_namespace_idx ON memories (namespace); --- Timestamp index +-- Content hash index for deduplication +CREATE INDEX IF NOT EXISTS memories_content_hash_idx + ON memories (namespace, content_hash); + +-- Expiration index for cleanup +CREATE INDEX IF NOT EXISTS memories_expires_at_idx + ON memories (expires_at) WHERE expires_at IS NOT NULL; + +-- Timestamp indexes CREATE INDEX IF NOT EXISTS memories_created_at_idx ON memories (created_at DESC); +CREATE INDEX IF NOT EXISTS memories_updated_at_idx + ON memories (updated_at DESC); + +-- Partial index for non-expired memories (optimization) +CREATE INDEX IF NOT EXISTS memories_active_idx + ON memories (namespace, created_at DESC) + WHERE expires_at IS NULL OR expires_at > NOW(); -- Auto-update updated_at CREATE OR REPLACE FUNCTION update_updated_at() @@ -120,6 +181,21 @@ CREATE TRIGGER memories_updated_at FOR EACH ROW EXECUTE FUNCTION update_updated_at(); ``` +### Schema Migration Notes + +**From v1 to v2:** +- Added `content_hash` (TEXT, NOT NULL) - SHA-256 hash of content +- Added `ttl_seconds` (INTEGER, DEFAULT 0) - TTL configuration +- Added `expires_at` (TIMESTAMPTZ, nullable) - Expiration timestamp +- Added composite index on `(namespace, content_hash)` for deduplication +- Added index on `expires_at` for cleanup queries +- Added partial index for active (non-expired) memories + +The migration is backward-compatible. Existing memories will have: +- `content_hash` computed from existing content +- `ttl_seconds` = 0 (no expiry) +- `expires_at` = NULL (no expiry) + --- ## Project Structure @@ -138,9 +214,11 @@ trindex/ │ ├── embed/ │ │ └── client.go # OpenAI-compatible embeddings client │ ├── memory/ -│ │ ├── store.go # Remember, forget, list +│ │ ├── store.go # Remember, forget, list, CreateWithParams │ │ ├── recall.go # Hybrid search (vector + FTS + RRF) -│ │ └── stats.go # Stats queries +│ │ ├── stats.go # Stats queries +│ │ ├── context_window.go # Context window ranking for LLM prompts +│ │ └── passport.go # Cross-system context transfer │ ├── mcp/ │ │ ├── server.go # MCP server setup, tool registration │ │ └── tools.go # Tool handler implementations @@ -164,14 +242,53 @@ trindex/ ## MCP Tools ### `remember` -Store a memory with optional namespace and metadata. + +Store a memory with optional namespace, metadata, deduplication, and TTL. **Input:** ```json { "content": "string (required) — the memory text to store", "namespace": "string (optional, default: 'default') — scope for this memory", - "metadata": "object (optional) — arbitrary key/value tags: { agent, project, tags[], source, type }" + "metadata": "object (optional) — arbitrary key/value tags: { agent, project, tags[], source, type }", + "skip_duplicate_threshold": "float (optional) — skip if similar memory exists (0.0-1.0, 0.95 for exact, 0.85 for semantic)", + "ttl_seconds": "int (optional) — time-to-live in seconds (0 = no expiry, session:* namespaces default to 24h)" +} +``` + +**Behavior:** +- Generate embedding and store to Postgres in parallel via goroutines +- Embedding call and DB insert happen concurrently where possible +- **Deduplication**: If `skip_duplicate_threshold` is set, check for existing similar memories first + - Uses content hash for exact matches (threshold >= 0.95) + - Uses semantic similarity for fuzzy matches + - Returns existing memory ID if duplicate found +- **TTL Support**: Set expiration for temporary memories + - Session namespaces (`session:*`) default to 24 hours (86400 seconds) + - Other namespaces default to no expiry + - Expired memories are filtered from recall results +- Returns structured confirmation: id, namespace, metadata extracted, timestamp, skipped status + +**Response (new memory):** +```json +{ + "id": "uuid", + "namespace": "default", + "metadata": { "agent": "opencode", "tags": ["architecture"] }, + "created_at": "2026-03-03T12:00:00Z", + "expires_at": "2026-03-04T12:00:00Z" +} +``` + +**Response (duplicate found):** +```json +{ + "id": "existing-uuid", + "namespace": "default", + "skipped": true, + "reason": "duplicate_content", + "similarity": 0.98, + "created_at": "2026-03-02T10:00:00Z" } ``` @@ -216,6 +333,7 @@ Retrieve memories by semantic similarity using hybrid search. - Run vector search (cosine similarity via HNSW) in parallel with FTS search (tsvector) - Fuse results using Reciprocal Rank Fusion (RRF) - Always include `global` namespace in addition to requested namespaces +- **Expiration filtering**: Automatically exclude expired memories (where `expires_at < NOW()`) - Apply metadata filters via JSONB queries after retrieval - Return results ranked by fused score @@ -324,22 +442,190 @@ Memories that appear in both lists score significantly higher. Memories that onl --- +## Advanced Features + +### Context Window Ranking + +Build an optimized context window for LLM prompts with intelligent ranking and token budget management. + +**Purpose:** When working with limited context windows, not all memories are equally valuable. This feature ranks memories by relevance, recency, and importance to build the most useful context. + +**Ranking Algorithm:** + +``` +final_score = (relevance_score * 0.5) + (recency_score * 0.3) + (type_boost * 0.2) +``` + +Where: +- **Relevance** (50%): Hybrid search similarity score (RRF) +- **Recency** (30%): Time decay function (newer = higher) + - `recency_score = 1 / (1 + hours_ago/24)` — 24h half-life +- **Type Boost** (20%): Importance based on metadata type + - `decision`: +0.3 boost + - `bug`: +0.25 boost + - `outcome`: +0.2 boost + - `pattern`: +0.15 boost + - `preference`: +0.1 boost + +**Usage:** + +```go +// Build context window with 4000 token budget +window, err := memory.BuildContextWindow(ctx, "query about auth", []string{"project:myapp"}, + memory.ContextWindowOptions{ + MaxTokens: 4000, + TopK: 20, + Threshold: 0.5, + }) + +// Results ordered by final_score, truncated to fit token budget +for _, item := range window.Items { + fmt.Printf("[%s] %s (score: %.3f, tokens: %d)\n", + item.Memory.Namespace, item.Memory.Content, item.Score, item.Tokens) +} +fmt.Printf("Total: %d items, %d tokens\n", len(window.Items), window.TotalTokens) +``` + +### Context Passport Pattern + +Transfer context between AI systems (Linear, GitHub, different agents) using a portable context package. + +**Purpose:** When an AI agent needs to hand off work to another system or resume work in a different context, the passport pattern preserves relevant memories. + +**Structure:** + +```json +{ + "version": "1.0", + "source": "project:trindex", + "target": "github:issue-123", + "created_at": "2026-03-07T12:00:00Z", + "expires_at": "2026-03-08T12:00:00Z", + "summary": "Working on deduplication feature implementation", + "key_facts": [ + "Content hash is SHA-256 of normalized content", + "Session namespaces default to 24h TTL" + ], + "decisions": [ + { + "content": "Use 0.95 threshold for exact dedup, 0.85 for semantic", + "rationale": "Balances precision vs recall for different use cases" + } + ], + "memory_refs": [ + {"id": "uuid-1", "namespace": "project:trindex", "content": "..."} + ], + "metadata": { + "agent": "claude-code", + "session_id": "debug-2026-03-07", + "tags": ["dedup", "phase-2"] + } +} +``` + +**Usage:** + +```go +// Create passport for handoff +passport, err := memory.CreatePassport(ctx, memory.PassportParams{ + SourceNamespace: "project:trindex", + TargetSystem: "github:issue-123", + Query: "deduplication implementation decisions", + MaxMemories: 10, + TTLHours: 24, +}) + +// Serialize for transfer +jsonData, _ := json.Marshal(passport) +// Send to other system... + +// Import in target system +imported, err := memory.ImportPassport(ctx, jsonData, memory.ImportOptions{ + TargetNamespace: "github:issue-123", + PreserveTTL: true, +}) +``` + +--- + ## Namespace Design +### Hierarchical Namespace Convention + +Namespaces follow a **hierarchical convention** for clear scoping and automatic inheritance: + +``` +global > project:{name} > agent:{name} > session:{id} +``` + +| Namespace | Purpose | Auto-searched | +|-----------|---------|---------------| +| `global` | Cross-agent user facts: preferences, identity, persistent personal context | **Always** | +| `project:{name}` | Project-specific memories: architecture, decisions, patterns | No | +| `agent:{name}` | Agent-specific learnings and optimizations | No | +| `session:{id}` | Ephemeral session context (auto-expires after 24h) | No | +| `default` | Fallback when no project context is clear | No | + +### Namespace Selection Rules + +1. **Use `global`** for cross-cutting user facts that any agent should know: + - User preferences ("User prefers dark mode") + - Identity facts ("User's name is Dave") + - Persistent personal context + +2. **Use `project:{name}`** for project-specific knowledge: + - Architecture decisions + - Code patterns discovered + - Bug root causes + - Implementation notes + - Example: `project:trindex`, `project:myapp` + +3. **Use `agent:{name}`** for agent-specific optimizations: + - Tool usage patterns + - Agent-specific shortcuts + - Learned behaviors + - Example: `agent:claude-code`, `agent:opencode` + +4. **Use `session:{id}`** for temporary context (auto-expires in 24h): + - Current debugging session + - Temporary file paths + - Transient errors + - Example: `session:debug-2026-03-07` + +5. **Avoid `default`** — be explicit about scope when possible. + +### Recall Behavior + - Every memory has a `namespace` string (default: `"default"`) -- The `global` namespace is always searched in recall, regardless of what namespaces are requested +- The `global` namespace is **always** searched in recall, regardless of what namespaces are requested - Agents should write project-specific memories to their own namespace - Cross-cutting facts (user preferences, personal context) should go in `global` - Orchestrators can pass multiple namespaces to cast a wide net -**Suggested namespace conventions:** -``` -global — always searched, cross-agent facts -default — fallback when no namespace specified -opencode — opencode agent session memories -trinity — Trinity orchestrator memories -stellar-breach — project-specific memories -personal — personal context and preferences +### Example Usage + +```json +// Store user preference (cross-agent) +{ + "content": "User prefers dark mode for all UIs", + "namespace": "global", + "metadata": { "type": "preference", "agent": "claude-code" } +} + +// Store project architecture decision +{ + "content": "Using pgvector with HNSW index for semantic search", + "namespace": "project:trindex", + "metadata": { "type": "decision", "tags": ["architecture", "database"] } +} + +// Store session-specific debugging info +{ + "content": "Error occurs when embedding dimensions mismatch", + "namespace": "session:debug-2026-03-07", + "ttl_seconds": 86400, + "metadata": { "type": "debug", "source": "error_log" } +} ``` --- @@ -772,7 +1058,7 @@ task check # Run all checks (fmt, lint, test, build) - Loading indicators - Error messages in modals -### Phase 3 — Polish (in progress) +### Phase 3 — Memory System Enhancement ✅ COMPLETED #### 3.1 Enhanced Features - [ ] **3.1.1** LLM metadata extraction (optional) @@ -791,6 +1077,43 @@ task check # Run all checks (fmt, lint, test, build) - REST API: `GET /api/duplicates` finds near-identical memories (similarity > 0.95) - `POST /api/duplicates/merge` merges duplicate memories - Configurable similarity threshold +- [x] **3.1.5** Client-side deduplication + - `skip_duplicate_threshold` parameter in `remember` tool + - 0.95 for exact matches (content hash), 0.85 for semantic matches + - Returns existing memory if duplicate found +- [x] **3.1.6** Server-side deduplication + - `content_hash` column (SHA-256) with unique constraint per namespace + - Prevents identical content storage in same namespace + - Backward compatible (hash computed for existing memories) +- [x] **3.1.7** TTL (Time-To-Live) support + - `ttl_seconds` parameter for temporary memories + - `expires_at` timestamp column + - Auto-expiration for `session:*` namespaces (24h default) + - Expired memories filtered from recall results + - `DeleteExpired()` method for cleanup + +#### 3.2 Advanced Retrieval +- [x] **3.2.1** Configurable hybrid search weights + - `HYBRID_VECTOR_WEIGHT` env var (default: 0.7) + - `HYBRID_FTS_WEIGHT` env var (default: 0.3) + - Per-query weight override in `recall` tool via `VectorWeight` and `FTSWeight` params +- [x] **3.2.2** Context window ranking + - `BuildContextWindow()` with weighted scoring algorithm + - Relevance (50%) + Recency (30%) + Type boost (20%) + - Token budget management for LLM context windows + - 24h half-life recency decay +- [x] **3.2.3** Context passport pattern + - `CreatePassport()` for exporting context to external systems + - `ImportPassport()` for importing context from other systems + - Cross-system handoff (Linear, GitHub, different agents) + - Portable JSON format with memory references +- [ ] **3.2.4** Per-query HNSW tuning + - `ef_search` parameter in `recall` tool (optional) + - Override default from env var +- [ ] **3.2.5** HNSW index health monitoring + - Track index staleness (deleted vectors ratio) + - Suggest reindex when threshold exceeded + - CLI command to trigger reindex #### 3.2 Search Improvements - [x] **3.2.1** Configurable hybrid search weights @@ -813,16 +1136,22 @@ task check # Run all checks (fmt, lint, test, build) - Retry logic with exponential backoff (`EMBED_MAX_RETRIES`, `EMBED_RETRY_DELAY_MS`) - Request timeout configuration (`EMBED_REQUEST_TIMEOUT_SEC`) - Batch embedding support (already supported) -- [ ] **3.3.3** Observability - - Structured logging with levels - - Metrics endpoint (Prometheus format) - - Request tracing +- [x] **3.3.3** Observability + - Structured logging with levels (JSON format, configurable) + - Metrics endpoint (`/metrics`, Prometheus format) + - Request tracing (automatic request ID generation) #### 3.4 Documentation & Community -- [ ] **3.4.1** API documentation (OpenAPI spec) -- [ ] **3.4.2** Architecture decision records (ADRs) -- [ ] **3.4.3** Contributing guide -- [ ] **3.4.4** Changelog and versioning +- [x] **3.4.1** API documentation (OpenAPI spec) +- [x] **3.4.2** Architecture decision records (ADRs) + - ADR-001: Hybrid Search with RRF + - ADR-002: Namespace Hierarchy Convention + - ADR-003: Two-Tier Deduplication Strategy + - ADR-004: TTL for Memory Lifecycle + - ADR-005: Context Window Ranking + - ADR-006: Context Passport Pattern +- [x] **3.4.3** Contributing guide +- [x] **3.4.4** Changelog and versioning --- @@ -869,6 +1198,9 @@ All MCP tool errors return structured responses: | `DB_UNAVAILABLE` | Postgres connection failed | | `NOT_FOUND` | Memory ID not found for forget/lookup | | `NAMESPACE_REQUIRED` | Forget called without sufficient scope | +| `DUPLICATE_CONTENT` | Content already exists (deduplication) | +| `PASSPORT_EXPIRED` | Context passport has expired | +| `PASSPORT_INVALID` | Context passport format is invalid | --- @@ -883,6 +1215,10 @@ All MCP tool errors return structured responses: - Tool descriptions must be written for both human and LLM readability — the agent decides which tool to call based on the description - Never delete without explicit scope — `forget` with no filters should return `INVALID_INPUT`, not delete everything - The official MCP Go SDK API may still be evolving — pin to a specific version in go.mod +- **Deduplication**: Content hash is computed using SHA-256 of trimmed content — normalization prevents whitespace-only differences +- **TTL**: Session namespaces (`session:*`) automatically get 24h TTL unless explicitly overridden — this prevents session bloat +- **Context Window**: BuildContextWindow uses weighted scoring — adjust weights based on use case (e.g., debugging benefits from recency boost) +- **Passport**: Always set appropriate TTL on passports — they contain sensitive context that shouldn't persist indefinitely --- diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1f36916 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,110 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +#### Memory System Enhancements +- **Content Hash Deduplication** - SHA-256 based server-side deduplication with unique constraint per namespace +- **TTL Support** - Time-to-live for temporary memories with automatic expiration +- **Client-side Deduplication** - `skip_duplicate_threshold` parameter in `remember` tool (0.95 exact, 0.85 semantic) +- **Session Namespace Auto-TTL** - `session:*` namespaces automatically expire after 24 hours + +#### Advanced Retrieval +- **Context Window Ranking** - Weighted scoring algorithm for LLM context optimization: + - Relevance: 50% + - Recency: 30% (24h half-life) + - Type boost: 20% (decision > bug > outcome > pattern) +- **Context Passport Pattern** - Cross-system context transfer for Linear/GitHub/agent handoff + +#### Observability +- **Structured Logging** - JSON-formatted logs with configurable levels (debug/info/warn/error) +- **Prometheus Metrics** - `/metrics` endpoint with HTTP, database, and MCP operation metrics +- **Request Tracing** - Automatic request ID generation and propagation + +#### Documentation +- **OpenAPI Specification** - Complete REST API documentation (`docs/openapi.yaml`) +- **Architecture Decision Records** - 6 ADRs documenting key architectural decisions +- **Contributing Guide** - Comprehensive guide for contributors +- **API Reference** - Detailed API documentation (`docs/api.md`) + +### Changed + +- **Database Schema v2** - Added `content_hash`, `ttl_seconds`, `expires_at` columns +- **Recall Behavior** - Automatically filters expired memories from search results +- **Namespace Convention** - Documented hierarchical convention (`global > project > agent > session`) + +### Deprecated + +- None + +### Removed + +- None + +### Fixed + +- None + +### Security + +- None + +## [1.0.8] - 2026-03-07 + +### Added +- MCP tool descriptions improved with vector/fts weight parameters + +## [1.0.7] - 2026-03-06 + +### Added +- Release workflow automation + +## [1.0.6] - 2026-03-05 + +### Added +- Claude plugin marketplace integration +- GoReleaser CI/CD pipeline + +## [1.0.0] - 2026-03-01 + +### Added +- Initial release +- Core MCP tools (remember, recall, forget, list, stats) +- Hybrid search with RRF +- Web UI with Vue.js +- REST API +- Namespace support + +--- + +## Release Notes Template + +When creating a new release, use this template: + +```markdown +## [X.Y.Z] - YYYY-MM-DD + +### Added +- New features + +### Changed +- Changes in existing functionality + +### Deprecated +- Soon-to-be removed features + +### Removed +- Now removed features + +### Fixed +- Bug fixes + +### Security +- Security improvements +``` diff --git a/CLAUDE.md b/CLAUDE.md index 256463a..2127ab5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,13 +2,63 @@ > Persistent semantic memory for AI agents via MCP (Model Context Protocol). -## Session Start Protocol - -At the beginning of each session: - -1. Call `stats` (no namespace) to see what namespaces exist and how much is stored. -2. Call `recall` 2–3 times with task-relevant queries (e.g. "trindex architecture decisions", "recent bugs fixed", "user preferences for this project") before starting work. -3. This orients you before you write a single line of code. +## Session Start Protocol (REQUIRED) + +At the beginning of **every** session, you MUST call `recall` to orient yourself. This is non-optional. + +### Standard Session Initialization Pattern + +```go +func InitializeSession(agentName, projectName string) SessionContext { + // 1. Get global user context (always searched automatically) + globalFacts := Recall("user identity preferences work style", nil, 0.3, 10) + + // 2. Get project-specific context + projectFacts := Recall("project architecture decisions patterns", ["project:"+projectName], 0.3, 10) + + // 3. Get recent activity for continuity + recentActivity := Recall("recent work current tasks", ["project:"+projectName], 0.1, 5, since=24h) + + return SessionContext{ + GlobalFacts: globalFacts, + ProjectFacts: projectFacts, + RecentActivity: recentActivity, + } +} +``` + +### Required Queries (Minimum) + +Call `recall` at least 2–3 times with these query types: + +1. **User Context**: `recall("user preferences identity work style", [], 0.3, 10)` + - Threshold: 0.3 (moderate precision for user facts) + - Returns: How user likes to work, their name, preferences + +2. **Project Context**: `recall("project architecture decisions", ["project:trindex"], 0.3, 10)` + - Threshold: 0.3 (moderate precision for architecture) + - Returns: Key decisions, patterns, codebase structure + +3. **Recent Activity**: `recall("recent work bugs fixed", ["project:trindex"], 0.1, 5)` with since=24h + - Threshold: 0.1 (wide net for recency-filtered results) + - Returns: What you were working on recently + +### Why This Matters + +- Prevents repeating the same questions to the user +- Maintains continuity between sessions +- Surfaces relevant context before you start coding +- Reduces "context loss" errors + +### Threshold Guidelines + +| Query Type | Threshold | Rationale | +|------------|-----------|-----------| +| User facts | 0.3–0.5 | Precise matches for preferences | +| Architecture | 0.3 | Balance precision/recall | +| Recent work | 0.1 | Wide net filtered by time | +| Exact lookup | 0.7+ | Specific function names, error codes | +| Exploration | 0.0001 | Broad discovery mode | ## When to Remember @@ -26,14 +76,40 @@ Write content as 1–3 concise sentences stating the fact directly. Example: ## Namespace Conventions -| Namespace | Purpose | -|-----------|---------| -| `global` | Cross-agent user facts: preferences, identity, persistent personal context. Always searched automatically on every recall — you never need to include it explicitly. | -| `default` | Fallback when no project context is clear. | -| `trindex` | Memories specific to the Trindex project itself. | -| `personal` | Personal context about the user that is not project-specific. | +Namespaces follow a **hierarchical convention** for clear scoping and automatic inheritance: + +``` +global > project:{name} > agent:{name} > session:{id} +``` + +| Namespace | Purpose | Auto-searched | +|-----------|---------|---------------| +| `global` | Cross-agent user facts: preferences, identity, persistent personal context. | Always | +| `project:{name}` | Project-specific memories: architecture, decisions, patterns (e.g., `project:trindex`). | No | +| `agent:{name}` | Agent-specific learnings and optimizations (e.g., `agent:claude-code`). | No | +| `session:{id}` | Ephemeral session context (auto-expires after 24h). | No | +| `default` | Fallback when no project context is clear. | No | + +### Namespace Selection Rules + +1. **Use `global`** for cross-cutting user facts that any agent should know: + - User preferences ("User prefers dark mode") + - Identity facts ("User's name is Dave") + - Persistent personal context -Use `global` for facts that should be available to any agent in any session. Use `trindex` for anything specific to this codebase or its development. +2. **Use `project:{name}`** for project-specific knowledge: + - Architecture decisions + - Code patterns discovered + - Bug root causes + - Implementation notes + +3. **Use `session:{id}`** for temporary context: + - Current debugging session + - Temporary file paths + - Transient errors + - These auto-expire (TTL = 24h by default) + +4. **Avoid `default`** — be explicit about scope. ## Metadata Conventions @@ -53,6 +129,7 @@ Use `global` for facts that should be available to any agent in any session. Use - **fts_weight high (e.g. 0.9)** — better for exact terms: function names, error codes, identifiers. - Always search `["trindex"]` namespace for project-specific memories; `global` is included automatically. - Pass multiple namespaces (`["trindex", "personal"]`) to cast a wider net across contexts. +- Use `skip_duplicate_threshold` on `remember` to avoid storing duplicates (0.95 for exact, 0.85 for semantic). ## Memory Hygiene @@ -60,6 +137,19 @@ Use `global` for facts that should be available to any agent in any session. Use - If you store a memory and later find it was wrong, `forget` it by ID and store a corrected version. - Use `list` with a namespace to audit what exists before bulk operations. +## Memory Lifecycle (TTL) + +Memories can have automatic expiration to prevent storage bloat: + +| Namespace Pattern | Default TTL | Rationale | +|------------------|-------------|-----------| +| `session:*` | 24 hours | Ephemeral session context | +| `global` | Never | User preferences persist | +| `project:*` | Never | Project knowledge persists | +| `default` | 30 days | Fallback cleanup | + +Use the `ttl_seconds` parameter on `remember` to set explicit expiration for temporary context. + ## Key Architectural Facts | Fact | Detail | @@ -72,3 +162,7 @@ Use `global` for facts that should be available to any agent in any session. Use | Main tools file | `internal/mcp/tools.go` | | Memory layer | `internal/memory/recall.go` (hybrid search), `internal/memory/store.go` (CRUD) | | Config | `internal/config/config.go` — all env-based with defaults | +| Deduplication | Content hash (SHA-256) + semantic similarity threshold | +| TTL | `ttl_seconds` param, `session:*` defaults to 24h | +| Context window | `internal/memory/context_window.go` — relevance/recency/type weighted ranking | +| Passport | `internal/memory/passport.go` — cross-system context transfer | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a03f3b8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,250 @@ +# Contributing to Trindex + +Thank you for your interest in contributing to Trindex! This guide will help you get started. + +## Table of Contents + +- [Development Setup](#development-setup) +- [Project Structure](#project-structure) +- [Making Changes](#making-changes) +- [Testing](#testing) +- [Code Style](#code-style) +- [Commit Messages](#commit-messages) +- [Pull Request Process](#pull-request-process) + +## Development Setup + +### Prerequisites + +- **Go 1.26+** - [Installation guide](https://go.dev/doc/install) +- **Docker** - For running PostgreSQL with pgvector +- **Node.js 24+** - For building the web UI +- **Task** (go-task) - Build automation: `brew install go-task` +- **Ollama** - For embeddings (optional, tests use mock) + +### Quick Start + +```bash +# Clone the repository +git clone https://github.com/dbehnke/trindex.git +cd trindex + +# Install dependencies +task deps + +# Start PostgreSQL +task docker:up + +# Run tests +task test + +# Build the binary +task build +``` + +## Project Structure + +``` +trindex/ +├── cmd/trindex/ # Entry point +├── internal/ +│ ├── auth/ # Authentication/authorization +│ ├── cmd/ # CLI command implementations +│ ├── config/ # Configuration management +│ ├── db/ # Database connection and migrations +│ ├── embed/ # Embedding client (OpenAI-compatible) +│ ├── eval/ # Cognitive evaluation suite +│ ├── mcp/ # MCP server and tools +│ ├── memory/ # Core memory layer (store, recall, etc.) +│ ├── observability/ # Logging, metrics, tracing +│ ├── testutil/ # Test utilities +│ └── web/ # HTTP server and web UI +├── web/ # Vue.js frontend source +├── docs/ # Documentation +│ ├── adr/ # Architecture Decision Records +│ └── openapi.yaml # OpenAPI specification +└── plans/ # Design documents and plans +``` + +## Making Changes + +### 1. Create a Branch + +```bash +git checkout -b feature/your-feature-name +``` + +### 2. Make Your Changes + +- Follow existing code patterns +- Add tests for new functionality +- Update documentation as needed + +### 3. Run Quality Checks + +```bash +# Format code +task fmt + +# Run linter +task lint + +# Run tests +task test + +# Build and verify +task build +``` + +## Testing + +### Unit Tests + +```bash +go test ./internal/memory/... -v +``` + +### Integration Tests + +Integration tests use `testcontainers-go` to spin up ephemeral PostgreSQL instances: + +```bash +go test ./... -v +``` + +**Note:** First run may be slow as it downloads the `pgvector:pg17` Docker image. + +### Cognitive Evaluation + +Run the built-in evaluation suite to verify recall precision: + +```bash +task eval +``` + +## Code Style + +### Go + +- Follow [Effective Go](https://go.dev/doc/effective_go) +- Use `gofmt` for formatting (enforced by linter) +- Prefer explicit error handling over panics +- Document exported functions and types + +### Example + +```go +// CreateMemory stores a new memory with the given parameters. +// Returns ErrDuplicate if content already exists in namespace. +func (s *Store) CreateMemory(ctx context.Context, params CreateParams) (*Memory, error) { + if params.Content == "" { + return nil, fmt.Errorf("%w: content is required", ErrInvalidInput) + } + // ... implementation +} +``` + +### Web UI + +- Vue 3 Composition API +- Tailwind CSS for styling +- Follow existing component patterns + +## Commit Messages + +We use [Conventional Commits](https://www.conventionalcommits.org/): + +``` +: + +[optional body] + +[optional footer] +``` + +### Types + +- `feat:` - New feature +- `fix:` - Bug fix +- `docs:` - Documentation changes +- `style:` - Code style (formatting, no logic change) +- `refactor:` - Code refactoring +- `test:` - Test additions/changes +- `chore:` - Build/process changes + +### Examples + +``` +feat: add content hash deduplication + +- Add SHA-256 content_hash column +- Unique constraint per namespace +- Return existing ID on duplicate + +fix: filter expired memories from recall + +docs: update API documentation for TTL support +``` + +## Pull Request Process + +1. **Ensure tests pass** + ```bash + task check + ``` + +2. **Update documentation** + - Add ADR for architectural changes + - Update relevant docs/*.md files + - Update OpenAPI spec if API changed + +3. **Create PR** + - Use descriptive title + - Reference any related issues + - Include summary of changes + +4. **Code Review** + - Address review feedback + - Keep commits atomic and focused + - Rebase if requested + +5. **Merge** + - Squash merge to main + - Ensure CI passes + +## Architecture Decision Records + +For significant architectural changes, create an ADR in `docs/adr/`: + +```markdown +# ADR-XXX: Title + +## Status +Proposed / Accepted / Deprecated + +## Context +What is the issue we're solving? + +## Decision +What did we decide? + +## Consequences +What are the trade-offs? + +## Alternatives Considered +What else did we consider? +``` + +## Getting Help + +- **GitHub Issues** - Bug reports and feature requests +- **GitHub Discussions** - Questions and ideas +- **Discord** (if applicable) - Real-time chat + +## License + +By contributing, you agree that your contributions will be licensed under the Business Source License 1.1. + +--- + +**Thank you for contributing to Trindex!** diff --git a/README.md b/README.md index fb876e7..863297c 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,19 @@ Persistent semantic memory for AI agents via MCP (Model Context Protocol). Trindex is a standalone Go binary that provides persistent, semantic memory for AI agents. It stores memories as vectors in Postgres with pgvector, enabling semantic search and hybrid retrieval (vector + full-text search). +### Key Features + +- **🔍 Hybrid Search** — Combines vector similarity (cosine) with full-text search (tsvector), fused with Reciprocal Rank Fusion (RRF) +- **🏷️ Hierarchical Namespaces** — Organized scoping: `global > project:{name} > agent:{name} > session:{id}` +- **🧠 Context Window Ranking** — Intelligent memory ranking for LLM prompts (relevance + recency + importance) +- **🛂 Context Passport** — Cross-system context transfer (Linear, GitHub, agent handoff) +- **🔄 Deduplication** — Client-side (threshold-based) and server-side (content hash) duplicate prevention +- **⏰ TTL Support** — Time-to-live for temporary memories with automatic cleanup +- **📊 Observability** — Structured logging, Prometheus metrics (/metrics), request tracing +- **🌐 Multi-Agent** — Share one memory server across multiple AI agents (OpenCode, Claude Code, Cursor, etc.) +- **🖥️ Web UI** — Built-in Vue.js interface for browsing and managing memories +- **🔌 MCP Native** — Works with any MCP-compatible agent via stdio transport + ## Quick Start ### ⚙️ Cognitive Evaluation & Stress Testing @@ -171,6 +184,108 @@ The web interface provides: - Namespace filtering - Similarity-based search results +## Advanced Features + +### Deduplication + +Trindex provides both client-side and server-side deduplication: + +**Client-side (threshold-based):** +```bash +# Skip if similar content already exists +# 0.95 = exact match (content hash), 0.85 = semantic similarity +./trindex memories create \ + --content "Architecture decision: using PostgreSQL" \ + --namespace project:myapp \ + --metadata skip_duplicate_threshold=0.95 +``` + +**Server-side (content hash):** +- Automatically prevents identical content in the same namespace +- Uses SHA-256 hash of normalized content +- Backward compatible with existing memories + +### TTL (Time-To-Live) + +Set expiration for temporary memories: + +```bash +# Expires after 1 hour (3600 seconds) +./trindex memories create \ + --content "Temporary debugging notes" \ + --namespace session:debug-123 \ + --metadata ttl_seconds=3600 +``` + +**Session namespaces** (`session:*`) automatically get 24-hour TTL unless overridden. + +### Context Window Ranking + +Build optimized context windows for LLM prompts with intelligent ranking: + +```go +// Rank memories by relevance (50%) + recency (30%) + importance (20%) +window, err := memory.BuildContextWindow(ctx, "auth implementation", + []string{"project:myapp"}, memory.ContextWindowOptions{ + MaxTokens: 4000, + TopK: 20, + }) +``` + +**Ranking factors:** +- **Relevance**: Hybrid search similarity score +- **Recency**: Time decay with 24h half-life (newer = higher) +- **Importance**: Type-based boost (decision > bug > outcome > pattern) + +### Context Passport + +Transfer context between AI systems: + +```go +// Export context for handoff to GitHub issue +passport, _ := memory.CreatePassport(ctx, memory.PassportParams{ + SourceNamespace: "project:trindex", + TargetSystem: "github:issue-123", + Query: "deduplication implementation", + MaxMemories: 10, + TTLHours: 24, +}) + +// Import in target system +imported, _ := memory.ImportPassport(ctx, passportJSON, memory.ImportOptions{ + TargetNamespace: "github:issue-123", +}) +``` + +### Observability + +Trindex provides comprehensive observability features: + +**Structured Logging:** +```bash +# JSON-formatted logs with configurable levels +LOG_LEVEL=debug ./trindex server +``` + +**Prometheus Metrics:** +```bash +# Metrics endpoint at /metrics +curl http://localhost:9636/metrics +``` + +Available metrics: +- `http_request_duration_seconds` - HTTP request latency +- `http_requests_total` - Total HTTP requests +- `memory_operations_total` - Memory operations by type +- `memory_operation_duration_seconds` - Operation latency +- `db_connections` - Database connection pool stats +- `mcp_requests_total` - MCP tool requests + +**Request Tracing:** +- Automatic request ID generation +- Propagated via `X-Request-ID` header +- Included in all log entries + ## MCP Client/Server Architecture Trindex uses a **client/server** model for MCP integration: @@ -215,6 +330,33 @@ export TRINDEX_API_KEY=your-secret-key Trindex natively supports multi-tenant isolation via Namespaces. By default, the Trindex MCP tool schemas will explicitly instruct connected AI Agents (OpenCode, Claude Code, Cursor, etc.) to store and recall memories using the `default` namespace unless they are explicitly told otherwise in your system conventions or prompts. +### Hierarchical Namespace Convention + +Namespaces follow a hierarchical convention for clear scoping: + +``` +global > project:{name} > agent:{name} > session:{id} +``` + +| Namespace | Purpose | Auto-searched | +|-----------|---------|---------------| +| `global` | Cross-agent facts (preferences, identity) | **Always** | +| `project:{name}` | Project-specific knowledge | No | +| `agent:{name}` | Agent-specific optimizations | No | +| `session:{id}` | Ephemeral context (auto-expires 24h) | No | + +**Examples:** +```bash +# Global user preference (available to all agents) +./trindex memories create --content "User prefers dark mode" --namespace global + +# Project-specific architecture decision +./trindex memories create --content "Using pgvector with HNSW" --namespace project:trindex + +# Session debugging info (auto-expires in 24h) +./trindex memories create --content "Error when dimensions mismatch" --namespace session:debug-123 +``` + ### opencode Add to `~/.config/opencode/opencode.json`: diff --git a/docs/adr/001-hybrid-search-rrf.md b/docs/adr/001-hybrid-search-rrf.md new file mode 100644 index 0000000..0b146cf --- /dev/null +++ b/docs/adr/001-hybrid-search-rrf.md @@ -0,0 +1,36 @@ +# ADR-001: Hybrid Search with Reciprocal Rank Fusion (RRF) + +## Status +Accepted + +## Context +Trindex needs to provide both semantic (meaning-based) and lexical (keyword-based) search capabilities. Users may search with natural language queries ("what did we decide about the database") or exact terms ("pgvector HNSW"). + +## Decision +We will implement **hybrid search** combining: +1. **Vector search** (cosine similarity via pgvector HNSW index) for semantic matching +2. **Full-text search** (PostgreSQL tsvector) for exact keyword matching +3. **Reciprocal Rank Fusion (RRF)** to combine results with k=60 + +Formula: `score = 1/(k + rank_vector) + 1/(k + rank_fts)` + +## Consequences + +### Positive +- Handles both semantic and exact queries effectively +- Memories appearing in both result sets rank significantly higher +- No need for users to choose search type +- Configurable weights per query (vector_weight, fts_weight) + +### Negative +- Double query execution (can parallelize) +- Requires maintaining both HNSW and GIN indexes +- RRF parameter k=60 is fixed; tuning requires experimentation + +## Alternatives Considered +- **Pure vector search**: Misses exact matches, struggles with specific terms +- **Pure FTS**: No semantic understanding, poor for conceptual queries +- **Weighted linear combination**: RRF performs better for heterogeneous ranking + +## References +- [Reciprocal Rank Fusion explained](https://plg.uwaterloo.ca/~gvcormac/cormackECIR09.pdf) diff --git a/docs/adr/002-namespace-hierarchy.md b/docs/adr/002-namespace-hierarchy.md new file mode 100644 index 0000000..ae71ab7 --- /dev/null +++ b/docs/adr/002-namespace-hierarchy.md @@ -0,0 +1,47 @@ +# ADR-002: Namespace Hierarchy Convention + +## Status +Accepted + +## Context +Multiple AI agents need to share a single memory system while maintaining isolation and discoverability. We needed a convention for organizing memories that allows: +1. Agent-specific context (preferences, project knowledge) +2. Cross-agent sharing (user identity, global facts) +3. Temporary/ephemeral data (debugging sessions) + +## Decision +Implement a hierarchical namespace convention: + +``` +global > project:{name} > agent:{name} > session:{id} +``` + +### Rules +- **`global`**: Always searched automatically. Cross-agent facts (user preferences, identity) +- **`project:{name}`**: Project-specific knowledge (architecture, decisions) +- **`agent:{name}`**: Agent-specific optimizations (rarely used) +- **`session:{id}`**: Ephemeral context (debugging, temporary files). Auto-expires after 24h +- **`default`**: Fallback when no specific scope is clear + +## Consequences + +### Positive +- Clear mental model for developers +- Automatic scoping (recall searches up hierarchy) +- Prevents namespace pollution +- Session auto-cleanup via TTL + +### Negative +- Requires discipline (agents must use correct namespace) +- Migration pain for existing memories +- Some use cases don't fit hierarchy (cross-project concerns) + +## Alternatives Considered +- **Flat namespaces with tags**: More flexible but harder to reason about +- **Automatic classification**: Would require ML/LLM integration, adds latency +- **UUID-based namespaces**: No semantic meaning, hard to query + +## Implementation Notes +- `global` namespace automatically included in all recall queries +- Session namespaces (`session:*`) default to 24h TTL +- Metadata conventions defined in CLAUDE.md diff --git a/docs/adr/003-deduplication-strategy.md b/docs/adr/003-deduplication-strategy.md new file mode 100644 index 0000000..1b1557f --- /dev/null +++ b/docs/adr/003-deduplication-strategy.md @@ -0,0 +1,51 @@ +# ADR-003: Two-Tier Deduplication Strategy + +## Status +Accepted + +## Context +Agents were storing duplicate memories (5+ copies of "User's name is Dave"), causing storage bloat and retrieval noise. We needed deduplication at both client and server levels. + +## Decision +Implement two-tier deduplication: + +### Tier 1: Client-Side Semantic Check +- Before `remember`, query with high similarity threshold (0.95 exact, 0.85 semantic) +- If match found, skip storage and return existing memory +- Parameter: `skip_duplicate_threshold` (0.0-1.0) + +### Tier 2: Server-Side Hash-Based +- SHA-256 content hash stored in `content_hash` column +- Unique constraint: `(namespace, content_hash)` +- Returns existing ID on conflict instead of error + +## Consequences + +### Positive +- Client-side prevents unnecessary round-trips +- Server-side guarantees no exact duplicates +- Configurable strictness per use case +- Backward compatible (hash computed for existing memories) + +### Negative +- Extra query for client-side check (latency) +- Semantic duplicates ("Dave" vs "User is Dave") not caught by hash +- Race condition window between check and write (mitigated by server-side) + +## Implementation Details +```go +// Client-side +existing := Recall(content, namespace, threshold, 1) +if existing[0].Score >= threshold { + return existing[0] // Skip duplicate +} + +// Server-side (hash + unique constraint) +content_hash = SHA256(trim(content)) +INSERT ... ON CONFLICT (namespace, content_hash) RETURN id +``` + +## Alternatives Considered +- **Vector index deduplication**: Catches semantic duplicates but adds write latency +- **Merge strategy**: Combine content on duplicate (complex merge heuristics) +- **Periodic cleanup**: Batch deduplication job (doesn't prevent bloat) diff --git a/docs/adr/004-ttl-lifecycle.md b/docs/adr/004-ttl-lifecycle.md new file mode 100644 index 0000000..2af2bde --- /dev/null +++ b/docs/adr/004-ttl-lifecycle.md @@ -0,0 +1,58 @@ +# ADR-004: Time-To-Live (TTL) for Memory Lifecycle + +## Status +Accepted + +## Context +Memories were persisting forever, causing: +1. Storage growth without bound +2. Old session data cluttering retrieval +3. Temporary files and debug info never cleaning up + +## Decision +Add TTL (Time-To-Live) support with automatic expiration: + +### Schema +- `ttl_seconds`: Integer, TTL in seconds (0 = no expiry) +- `expires_at`: Timestamp, calculated expiration time + +### Default TTLs +| Namespace Pattern | TTL | Rationale | +|------------------|-----|-----------| +| `session:*` | 24 hours | Ephemeral debugging context | +| `global` | Never | User preferences persist | +| `project:*` | Never | Project knowledge persists | +| `default` | 30 days | Fallback cleanup | + +### Cleanup +- Expired memories filtered from recall results automatically +- `DeleteExpired()` method for batch cleanup +- Background job can be scheduled externally (cron, etc.) + +## Consequences + +### Positive +- Automatic cleanup of ephemeral data +- Explicit intent at write time (temporary vs permanent) +- Prevents session bloat +- Flexible per-memory TTL override + +### Negative +- Accidental data loss if wrong TTL set +- No built-in resurrection of expired memories +- Background job requires external scheduling + +## Implementation +```sql +ALTER TABLE memories ADD COLUMN ttl_seconds INTEGER DEFAULT 0; +ALTER TABLE memories ADD COLUMN expires_at TIMESTAMPTZ; + +-- Query automatically excludes expired +SELECT * FROM memories +WHERE (expires_at IS NULL OR expires_at > NOW()) +``` + +## Alternatives Considered +- **Archival tier**: Move old to S3 (complex infrastructure) +- **Relevance-based decay**: Gradual fading based on access (unpredictable) +- **Manual deletion**: Rely on users to clean up (won't happen) diff --git a/docs/adr/005-context-window-ranking.md b/docs/adr/005-context-window-ranking.md new file mode 100644 index 0000000..cf17d65 --- /dev/null +++ b/docs/adr/005-context-window-ranking.md @@ -0,0 +1,61 @@ +# ADR-005: Context Window Ranking for LLM Optimization + +## Status +Accepted + +## Context +When building prompts for LLMs, we need to fit the most relevant context within token limits. Simple recall by similarity isn't enough—we need to balance: +1. Relevance to current query +2. Recency (newer context is often more important) +3. Importance (decisions > random facts) + +## Decision +Implement weighted scoring algorithm for context window ranking: + +``` +final_score = (relevance * 0.5) + (recency * 0.3) + (type_boost * 0.2) +``` + +### Components +1. **Relevance (50%)**: Hybrid search similarity score +2. **Recency (30%)**: Time decay with 24h half-life + - `recency = 1 / (1 + hours_ago / 24)` +3. **Type Boost (20%)**: Importance based on metadata.type + - `decision`: +0.3 + - `bug`: +0.25 + - `outcome`: +0.2 + - `pattern`: +0.15 + - `preference`: +0.1 + +### Token Budget Management +- Estimate tokens from content length +- Fit memories until budget exceeded +- Return top N that fit + +## Consequences + +### Positive +- Optimized context for LLM prompts +- Automatically surfaces important decisions +- Respects token limits +- Better than naive top-k similarity + +### Negative +- Complex ranking heuristics (tuning required) +- May drop relevant but older memories +- Non-deterministic (depends on current time) + +## Usage +```go +window, err := memory.BuildContextWindow(ctx, "auth query", + []string{"project:myapp"}, + memory.ContextWindowOptions{ + MaxTokens: 4000, + TopK: 20, + }) +``` + +## Alternatives Considered +- **Pure recency**: Misses important old decisions +- **Pure relevance**: May miss recent context shifts +- **Fixed quotas per type**: Too rigid, doesn't adapt to query diff --git a/docs/adr/006-context-passport.md b/docs/adr/006-context-passport.md new file mode 100644 index 0000000..ac948dc --- /dev/null +++ b/docs/adr/006-context-passport.md @@ -0,0 +1,68 @@ +# ADR-006: Context Passport Pattern for Cross-System Handoff + +## Status +Accepted + +## Context +AI agents need to transfer context between systems (VS Code → Linear, GitHub → Slack) without losing continuity. Each system has its own memory store, creating silos. + +## Decision +Implement explicit **Context Passport** pattern: + +### Export (Source System) +```go +passport := memory.CreatePassport(ctx, memory.PassportParams{ + SourceNamespace: "project:trindex", + TargetSystem: "github:issue-123", + Query: "current work context", + MaxMemories: 10, + TTLHours: 24, +}) +``` + +### Passport Structure +```json +{ + "version": "1.0", + "source": "project:trindex", + "target": "github:issue-123", + "created_at": "2026-03-07T12:00:00Z", + "expires_at": "2026-03-08T12:00:00Z", + "summary": "Working on deduplication feature", + "key_facts": ["Content hash is SHA-256"], + "decisions": [{"content": "...", "rationale": "..."}], + "memory_refs": [{"id": "...", "content": "..."}], + "metadata": {"agent": "claude-code"} +} +``` + +### Import (Target System) +```go +imported, err := memory.ImportPassport(ctx, jsonData, memory.ImportOptions{ + TargetNamespace: "github:issue-123", + PreserveTTL: true, +}) +``` + +## Consequences + +### Positive +- No infrastructure changes required +- Systems remain autonomous +- Explicit context sharing (privacy control) +- Portable JSON format +- TTL prevents stale context + +### Negative +- Manual agent coordination required +- Context may be stale by import time +- Duplication across systems +- No automatic synchronization + +## Future: MCP-First Architecture +The passport pattern is a stepping stone toward full **MCP-First Architecture** (ADR-007), where all systems share Trindex as single source of truth via MCP adapters. + +## Alternatives Considered +- **Event-driven federation**: Complex infrastructure (Kafka/NATS) +- **Shared database**: Tight coupling, single point of failure +- **API integrations**: Every pair of systems needs custom integration diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..9691792 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,619 @@ +# Trindex API Reference + +Complete API reference for Trindex memory operations. + +## Table of Contents + +- [MCP Tools](#mcp-tools) +- [REST API](#rest-api) +- [Advanced Features](#advanced-features) +- [Go SDK](#go-sdk) + +--- + +## MCP Tools + +Trindex exposes 5 MCP tools for AI agent integration. + +### `remember` + +Store a memory with optional namespace, metadata, deduplication, and TTL. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `content` | string | ✅ | - | The memory text to store | +| `namespace` | string | ❌ | `"default"` | Scope for this memory | +| `metadata` | object | ❌ | `{}` | Arbitrary key/value tags | +| `skip_duplicate_threshold` | float | ❌ | - | Skip if similar memory exists (0.95=exact, 0.85=semantic) | +| `ttl_seconds` | int | ❌ | `0` | Time-to-live in seconds (0=no expiry) | + +**Example Request:** + +```json +{ + "content": "Using pgvector with HNSW index for semantic search", + "namespace": "project:trindex", + "metadata": { + "type": "decision", + "tags": ["architecture", "database"], + "agent": "claude-code" + }, + "skip_duplicate_threshold": 0.95, + "ttl_seconds": 0 +} +``` + +**Example Response (new memory):** + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "namespace": "project:trindex", + "content_hash": "a3f5c2...", + "metadata": { + "type": "decision", + "tags": ["architecture", "database"], + "agent": "claude-code" + }, + "created_at": "2026-03-07T12:00:00Z", + "expires_at": null +} +``` + +**Example Response (duplicate found):** + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440001", + "namespace": "project:trindex", + "skipped": true, + "reason": "duplicate_content", + "similarity": 0.98, + "created_at": "2026-03-06T10:00:00Z" +} +``` + +**Notes:** +- Session namespaces (`session:*`) default to 24h TTL unless overridden +- Content hash is computed using SHA-256 of normalized (trimmed) content +- Deduplication checks both exact hash matches and semantic similarity + +--- + +### `recall` + +Retrieve memories by semantic similarity using hybrid search. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `query` | string | ✅ | - | Natural language search query | +| `namespaces` | []string | ❌ | `["default"]` | Namespaces to search (`global` always included) | +| `top_k` | int | ❌ | `10` | Number of results to return | +| `threshold` | float | ❌ | `0.7` | Minimum similarity score (0.0-1.0) | +| `vector_weight` | float | ❌ | `0.7` | Weight for vector search (0.0-1.0) | +| `fts_weight` | float | ❌ | `0.3` | Weight for full-text search (0.0-1.0) | +| `filter` | object | ❌ | `{}` | Metadata filters | + +**Filter Object:** + +| Field | Type | Description | +|-------|------|-------------| +| `since` | string (RFC3339) | Only memories created after this time | +| `until` | string (RFC3339) | Only memories created before this time | +| `tags` | []string | Match any tag in `metadata.tags` | +| `source` | string | Match `metadata.source` | + +**Example Request:** + +```json +{ + "query": "database architecture decisions", + "namespaces": ["project:trindex", "project:myapp"], + "top_k": 5, + "threshold": 0.6, + "vector_weight": 0.8, + "fts_weight": 0.2, + "filter": { + "since": "2026-01-01T00:00:00Z", + "tags": ["architecture"] + } +} +``` + +**Example Response:** + +```json +{ + "results": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "content": "Using pgvector with HNSW index for semantic search", + "namespace": "project:trindex", + "score": 0.92, + "metadata": { + "type": "decision", + "tags": ["architecture", "database"] + }, + "created_at": "2026-03-07T12:00:00Z" + } + ], + "total": 1, + "namespaces_searched": ["project:trindex", "project:myapp", "global"] +} +``` + +**Notes:** +- Expired memories are automatically filtered out +- The `global` namespace is always included in searches +- Hybrid scores are fused using Reciprocal Rank Fusion (RRF) with k=60 + +--- + +### `forget` + +Delete one or more memories. + +**Parameters:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `id` | string | ❌ | Delete single memory by UUID | +| `namespace` | string | ❌ | Delete all memories in namespace | +| `filter` | object | ❌ | Filter criteria for bulk deletion | + +**Filter Object:** + +| Field | Type | Description | +|-------|------|-------------| +| `before` | string (RFC3339) | Delete memories older than this | +| `tags` | []string | Delete memories matching these tags | + +**Example Requests:** + +```json +// Delete by ID +{ + "id": "550e8400-e29b-41d4-a716-446655440000" +} + +// Delete entire namespace +{ + "namespace": "session:debug-123" +} + +// Delete old session memories +{ + "namespace": "session:*", + "filter": { + "before": "2026-03-01T00:00:00Z" + } +} +``` + +**Example Response:** + +```json +{ + "deleted": 5, + "namespace": "session:debug-123" +} +``` + +**Notes:** +- At least one of `id`, `namespace`, or `filter` must be provided +- `namespace` and `filter` can be combined +- Use wildcards with caution — `session:*` matches all session namespaces + +--- + +### `list` + +Browse memories without a semantic query. Useful for inspection and debugging. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `namespace` | string | ❌ | - | Filter by namespace | +| `limit` | int | ❌ | `20` | Maximum results | +| `offset` | int | ❌ | `0` | Pagination offset | +| `order` | string | ❌ | `"desc"` | Sort order: `"asc"` or `"desc"` by created_at | + +**Example Request:** + +```json +{ + "namespace": "project:trindex", + "limit": 10, + "offset": 0, + "order": "desc" +} +``` + +**Example Response:** + +```json +{ + "memories": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "content": "Using pgvector with HNSW index", + "namespace": "project:trindex", + "metadata": {}, + "created_at": "2026-03-07T12:00:00Z" + } + ], + "total": 42, + "limit": 10, + "offset": 0 +} +``` + +--- + +### `stats` + +Return memory statistics. Useful for monitoring and the web UI. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `namespace` | string | ❌ | - | Scope stats to namespace; omit for global | + +**Example Response:** + +```json +{ + "total_memories": 1024, + "by_namespace": { + "default": 400, + "project:trindex": 300, + "global": 200, + "session:debug-123": 124 + }, + "recent_24h": 42, + "expiring_24h": 5, + "oldest_memory": "2025-11-01T00:00:00Z", + "newest_memory": "2026-03-07T12:00:00Z", + "top_tags": ["architecture", "decision", "bug", "pattern"], + "embedding_model": "nomic-embed-text", + "embed_dimensions": 768 +} +``` + +--- + +## REST API + +When running `trindex server`, a REST API is available on port 9636. + +### Authentication + +All API endpoints (except health check) require an API key: + +```bash +curl -H "Authorization: Bearer $TRINDEX_API_KEY" \ + http://localhost:9636/api/memories +``` + +### Endpoints + +#### Memories + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/memories` | List memories | +| GET | `/api/memories/:id` | Get memory by ID | +| POST | `/api/memories` | Create memory | +| DELETE | `/api/memories/:id` | Delete memory | + +#### Search & Stats + +| Method | Endpoint | Description | +|--------|----------|-------------| +| POST | `/api/search` | Hybrid search | +| GET | `/api/stats` | Get statistics | + +#### Import/Export + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/export` | Export memories (JSONL) | +| POST | `/api/import` | Import memories (JSONL) | + +### POST /api/memories + +Create a new memory via REST API. + +**Request Body:** + +```json +{ + "content": "Memory content", + "namespace": "project:myapp", + "metadata": { + "type": "decision", + "tags": ["architecture"] + }, + "skip_duplicate_threshold": 0.95, + "ttl_seconds": 3600 +} +``` + +**Response:** Same as MCP `remember` tool. + +### POST /api/search + +Search memories using hybrid search. + +**Request Body:** + +```json +{ + "query": "database architecture", + "namespaces": ["project:myapp"], + "top_k": 10, + "threshold": 0.7, + "filter": { + "tags": ["architecture"] + } +} +``` + +**Response:** Same as MCP `recall` tool. + +### GET /api/export + +Export memories to JSONL format. + +**Query Parameters:** + +| Parameter | Description | +|-----------|-------------| +| `namespace` | Filter by namespace | +| `since` | Export memories since date (RFC3339) | +| `until` | Export memories until date (RFC3339) | + +**Example:** + +```bash +curl -H "Authorization: Bearer $TRINDEX_API_KEY" \ + "http://localhost:9636/api/export?namespace=project:myapp&since=2026-01-01T00:00:00Z" \ + > myapp-memories.jsonl +``` + +### POST /api/import + +Import memories from JSONL format. + +**Request Body:** Streaming JSONL with one memory per line: + +```jsonl +{"content": "Memory 1", "namespace": "project:myapp", "metadata": {}} +{"content": "Memory 2", "namespace": "project:myapp", "metadata": {}} +``` + +**Query Parameters:** + +| Parameter | Description | +|-----------|-------------| +| `skip_existing` | Skip duplicates (default: false) | +| `namespace` | Import to specific namespace | + +--- + +## Advanced Features + +### Context Window Ranking + +Build an optimized context window for LLM prompts. + +**Go SDK:** + +```go +import "github.com/dbehnke/trindex/internal/memory" + +window, err := memory.BuildContextWindow(ctx, + "query about authentication", + []string{"project:myapp"}, + memory.ContextWindowOptions{ + MaxTokens: 4000, + TopK: 20, + Threshold: 0.5, + }, +) + +for _, item := range window.Items { + fmt.Printf("[%s] %s (score: %.3f, tokens: %d)\n", + item.Memory.Namespace, + item.Memory.Content, + item.Score, + item.Tokens, + ) +} +``` + +**Ranking Algorithm:** + +``` +final_score = (relevance * 0.5) + (recency * 0.3) + (type_boost * 0.2) + +Where: +- relevance = hybrid search similarity score +- recency = 1 / (1 + hours_ago/24) // 24h half-life +- type_boost = based on metadata.type: + - decision: +0.3 + - bug: +0.25 + - outcome: +0.2 + - pattern: +0.15 + - preference: +0.1 +``` + +### Context Passport + +Transfer context between AI systems. + +**Creating a Passport:** + +```go +passport, err := memory.CreatePassport(ctx, memory.PassportParams{ + SourceNamespace: "project:trindex", + TargetSystem: "github:issue-123", + Query: "deduplication implementation", + MaxMemories: 10, + TTLHours: 24, +}) + +// Serialize for transfer +jsonData, _ := json.Marshal(passport) +``` + +**Passport Structure:** + +```json +{ + "version": "1.0", + "source": "project:trindex", + "target": "github:issue-123", + "created_at": "2026-03-07T12:00:00Z", + "expires_at": "2026-03-08T12:00:00Z", + "summary": "Working on deduplication feature", + "key_facts": ["Content hash is SHA-256", "Session TTL is 24h"], + "decisions": [ + { + "content": "Use 0.95 threshold for exact dedup", + "rationale": "Balances precision vs recall" + } + ], + "memory_refs": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "namespace": "project:trindex", + "content": "..." + } + ], + "metadata": { + "agent": "claude-code", + "session_id": "debug-2026-03-07" + } +} +``` + +**Importing a Passport:** + +```go +imported, err := memory.ImportPassport(ctx, jsonData, memory.ImportOptions{ + TargetNamespace: "github:issue-123", + PreserveTTL: true, +}) +``` + +--- + +## Go SDK + +### Memory Store + +```go +import "github.com/dbehnke/trindex/internal/memory" + +// Create memory with options +params := memory.CreateParams{ + Content: "Memory content", + Namespace: "project:myapp", + Metadata: map[string]interface{}{ + "type": "decision", + "tags": []string{"architecture"}, + }, + SkipIfDuplicate: true, + DuplicateThreshold: 0.95, + TTLSeconds: 86400, +} + +mem, err := store.CreateWithParams(ctx, params) +``` + +### Recall with Filters + +```go +results, err := store.Recall(ctx, "database query", + []string{"project:myapp"}, + 10, // top_k + 0.7, // threshold + &memory.Filter{ + Since: time.Now().AddDate(0, -1, 0), + Tags: []string{"architecture"}, + }, +) +``` + +### Delete Expired Memories + +```go +// Delete all expired memories (where expires_at < NOW()) +deleted, err := store.DeleteExpired(ctx) +``` + +--- + +## Error Codes + +| Code | HTTP Status | Meaning | +|------|-------------|---------| +| `INVALID_INPUT` | 400 | Missing required field or bad type | +| `EMBED_FAILED` | 503 | Embedding endpoint unreachable | +| `DB_UNAVAILABLE` | 503 | Postgres connection failed | +| `NOT_FOUND` | 404 | Memory ID not found | +| `NAMESPACE_REQUIRED` | 400 | Forget called without scope | +| `DUPLICATE_CONTENT` | 409 | Content already exists | +| `PASSPORT_EXPIRED` | 410 | Context passport has expired | +| `PASSPORT_INVALID` | 400 | Invalid passport format | + +--- + +## Best Practices + +### Namespace Organization + +``` +global/ # User preferences, cross-agent facts +project:trindex/ # Project-specific knowledge +agent:claude/ # Agent-specific optimizations +session:debug-*/ # Ephemeral debugging context (auto-expires) +``` + +### Threshold Guidelines + +| Use Case | Threshold | Notes | +|----------|-----------|-------| +| Initial orientation | 0.0001 | Wide net, filtered by time | +| General recall | 0.3-0.5 | Balance precision/recall | +| Specific lookup | 0.7+ | Exact matches only | +| Deduplication (exact) | 0.95 | Content hash match | +| Deduplication (fuzzy) | 0.85 | Semantic similarity | + +### Memory Types + +Use `metadata.type` to improve context window ranking: + +- `decision` — Architecture or design decisions (+0.3 boost) +- `bug` — Bug fixes and root causes (+0.25 boost) +- `outcome` — Task completions (+0.2 boost) +- `pattern` — Code patterns discovered (+0.15 boost) +- `preference` — User preferences (+0.1 boost) +- `fact` — General knowledge (no boost) + +### TTL Recommendations + +| Context Type | Recommended TTL | Rationale | +|--------------|-----------------|-----------| +| Session debug | 1-24 hours | Transient debugging info | +| Temporary files | 1-7 days | Build artifacts, temp paths | +| Working notes | 7-30 days | Active development context | +| Decisions | Never | Permanent architecture records | +| Preferences | Never | User preferences persist | diff --git a/docs/cli.md b/docs/cli.md index f10c2ab..f62bcfb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -297,6 +297,8 @@ trindex memories create [flags] | `--metadata key=value` | Metadata key-value | - | | `--file PATH` | Read content from file | - | | `--json` | Output as JSON | false | +| `--skip-duplicate-threshold 0.0-1.0` | Skip if similar memory exists | - | +| `--ttl-seconds N` | Time-to-live in seconds (0 = no expiry) | 0 | **Example:** @@ -313,6 +315,18 @@ trindex memories create \ # Create from file trindex memories create --file notes.txt --namespace research + +# Create with deduplication (skip if 95% similar content exists) +trindex memories create \ + --content "Architecture decision: using PostgreSQL" \ + --namespace project:myapp \ + --skip-duplicate-threshold 0.95 + +# Create with TTL (expires after 1 hour) +trindex memories create \ + --content "Temporary debugging notes" \ + --namespace session:debug-123 \ + --ttl-seconds 3600 ``` #### memories delete @@ -513,6 +527,37 @@ Trindex can be configured via environment variables. These can also be set in a | `HNSW_EF_CONSTRUCTION` | HNSW ef_construction | 64 | | `HNSW_EF_SEARCH` | HNSW ef_search | 40 | +### Observability + +| Variable | Description | Default | +|----------|-------------|---------| +| `LOG_LEVEL` | Log level (debug/info/warn/error) | info | +| `LOG_FORMAT` | Log format (json/text) | json | + +### TTL and Deduplication + +| Variable | Description | Default | +|----------|-------------|---------| +| `DEFAULT_SESSION_TTL` | Default TTL for session namespaces (seconds) | 86400 (24h) | +| `DEFAULT_DEDUP_THRESHOLD` | Default deduplication threshold (0.0-1.0) | 0.95 | + +### Database Connection Pool + +| Variable | Description | Default | +|----------|-------------|---------| +| `DB_MAX_CONNS` | Maximum database connections | 25 | +| `DB_MIN_CONNS` | Minimum database connections | 5 | +| `DB_MAX_CONN_LIFETIME_MINUTES` | Maximum connection lifetime | 60 | +| `DB_MAX_CONN_IDLE_TIME_MINUTES` | Maximum idle connection time | 30 | + +### Embedding Client + +| Variable | Description | Default | +|----------|-------------|---------| +| `EMBED_MAX_RETRIES` | Maximum embedding retries | 3 | +| `EMBED_RETRY_DELAY_MS` | Retry delay in milliseconds | 1000 | +| `EMBED_REQUEST_TIMEOUT_SEC` | Request timeout in seconds | 30 | + ## Configuration File Trindex supports YAML configuration files. Use `--config` to specify the path. diff --git a/docs/openapi.yaml b/docs/openapi.yaml new file mode 100644 index 0000000..fa50c1f --- /dev/null +++ b/docs/openapi.yaml @@ -0,0 +1,471 @@ +openapi: 3.0.0 +info: + title: Trindex API + description: | + Persistent semantic memory for AI agents via MCP (Model Context Protocol). + + Trindex provides a REST API for storing, retrieving, and managing memories + with vector-based semantic search and full-text search capabilities. + version: 1.0.0 + contact: + name: Trindex Support + url: https://github.com/dbehnke/trindex + license: + name: Business Source License 1.1 + url: https://github.com/dbehnke/trindex/blob/main/LICENSE + +servers: + - url: http://localhost:9636 + description: Local development server + +security: + - bearerAuth: [] + +paths: + /health: + get: + summary: Health check + description: Returns health status of the Trindex server + security: [] + responses: + '200': + description: Server is healthy + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: healthy + timestamp: + type: string + format: date-time + + /metrics: + get: + summary: Prometheus metrics + description: Returns Prometheus-compatible metrics for monitoring + security: [] + responses: + '200': + description: Prometheus metrics in text format + content: + text/plain: + schema: + type: string + + /api/memories: + get: + summary: List memories + description: Browse memories without semantic search + parameters: + - name: namespace + in: query + description: Filter by namespace + schema: + type: string + - name: limit + in: query + description: Maximum number of results + schema: + type: integer + default: 20 + - name: offset + in: query + description: Pagination offset + schema: + type: integer + default: 0 + - name: order + in: query + description: Sort order by created_at + schema: + type: string + enum: [asc, desc] + default: desc + responses: + '200': + description: List of memories + content: + application/json: + schema: + type: object + properties: + memories: + type: array + items: + $ref: '#/components/schemas/Memory' + total: + type: integer + limit: + type: integer + offset: + type: integer + + post: + summary: Create memory + description: Store a new memory with optional namespace and metadata + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - content + properties: + content: + type: string + description: The memory text to store + namespace: + type: string + description: Scope for this memory (default 'default') + default: default + metadata: + type: object + description: Arbitrary key/value tags + skip_duplicate_threshold: + type: number + description: Skip if similar memory exists (0.95 exact, 0.85 semantic) + minimum: 0 + maximum: 1 + ttl_seconds: + type: integer + description: Time-to-live in seconds (0 = no expiry) + minimum: 0 + responses: + '201': + description: Memory created successfully + content: + application/json: + schema: + $ref: '#/components/schemas/Memory' + '200': + description: Duplicate found, existing memory returned + content: + application/json: + schema: + $ref: '#/components/schemas/Memory' + '400': + $ref: '#/components/responses/BadRequest' + + /api/memories/{id}: + get: + summary: Get memory by ID + description: Retrieve a specific memory by its UUID + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Memory found + content: + application/json: + schema: + $ref: '#/components/schemas/Memory' + '404': + $ref: '#/components/responses/NotFound' + + delete: + summary: Delete memory + description: Delete a memory by its UUID + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Memory deleted successfully + content: + application/json: + schema: + type: object + properties: + deleted: + type: boolean + '404': + $ref: '#/components/responses/NotFound' + + /api/search: + post: + summary: Search memories + description: | + Search memories using hybrid search (vector + full-text) with + Reciprocal Rank Fusion (RRF) scoring. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - query + properties: + query: + type: string + description: Natural language search query + namespaces: + type: array + items: + type: string + description: Namespaces to search (global always included) + default: [default] + top_k: + type: integer + description: Number of results to return + default: 10 + threshold: + type: number + description: Minimum similarity score (0.0-1.0) + default: 0.7 + vector_weight: + type: number + description: Weight for vector search (0.0-1.0) + default: 0.7 + fts_weight: + type: number + description: Weight for full-text search (0.0-1.0) + default: 0.3 + filter: + $ref: '#/components/schemas/Filter' + responses: + '200': + description: Search results + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + $ref: '#/components/schemas/SearchResult' + total: + type: integer + namespaces_searched: + type: array + items: + type: string + + /api/stats: + get: + summary: Get statistics + description: Return memory statistics for monitoring and dashboard + parameters: + - name: namespace + in: query + description: Scope stats to specific namespace + schema: + type: string + responses: + '200': + description: Statistics data + content: + application/json: + schema: + $ref: '#/components/schemas/Stats' + + /api/export: + get: + summary: Export memories + description: Export memories to JSONL format + parameters: + - name: namespace + in: query + description: Filter by namespace + schema: + type: string + - name: since + in: query + description: Export memories since date (RFC3339) + schema: + type: string + format: date-time + - name: until + in: query + description: Export memories until date (RFC3339) + schema: + type: string + format: date-time + responses: + '200': + description: Memories exported as JSONL + content: + application/jsonl: + schema: + type: string + + /api/import: + post: + summary: Import memories + description: Import memories from JSONL format + parameters: + - name: skip_existing + in: query + description: Skip duplicates + schema: + type: boolean + default: false + - name: namespace + in: query + description: Import to specific namespace + schema: + type: string + requestBody: + required: true + content: + application/jsonl: + schema: + type: string + description: JSONL with one memory per line + responses: + '200': + description: Import results + content: + application/json: + schema: + type: object + properties: + imported: + type: integer + skipped: + type: integer + errors: + type: integer + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: API key + + schemas: + Memory: + type: object + properties: + id: + type: string + format: uuid + namespace: + type: string + content: + type: string + content_hash: + type: string + description: SHA-256 hash for deduplication + metadata: + type: object + ttl_seconds: + type: integer + description: TTL in seconds + expires_at: + type: string + format: date-time + nullable: true + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + + SearchResult: + type: object + properties: + id: + type: string + format: uuid + content: + type: string + namespace: + type: string + score: + type: number + description: Hybrid search score (0.0-1.0) + metadata: + type: object + created_at: + type: string + format: date-time + + Filter: + type: object + properties: + since: + type: string + format: date-time + description: Only memories created after this time + until: + type: string + format: date-time + description: Only memories created before this time + tags: + type: array + items: + type: string + description: Match any tag in metadata.tags + source: + type: string + description: Match metadata.source + + Stats: + type: object + properties: + total_memories: + type: integer + by_namespace: + type: object + additionalProperties: + type: integer + recent_24h: + type: integer + expiring_24h: + type: integer + oldest_memory: + type: string + format: date-time + newest_memory: + type: string + format: date-time + top_tags: + type: array + items: + type: string + embedding_model: + type: string + embed_dimensions: + type: integer + + responses: + BadRequest: + description: Invalid request + content: + application/json: + schema: + type: object + properties: + error: + type: string + code: + type: string + + NotFound: + description: Resource not found + content: + application/json: + schema: + type: object + properties: + error: + type: string + code: + type: string diff --git a/go.mod b/go.mod index 528d5c5..d5d6d90 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,9 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -51,11 +53,16 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.3 // indirect github.com/shirou/gopsutil/v4 v4.25.6 // indirect @@ -69,9 +76,11 @@ require ( go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/crypto v0.43.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.30.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/go.sum b/go.sum index 48e98ba..a53df93 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,12 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -111,6 +115,8 @@ github.com/modelcontextprotocol/go-sdk v1.4.0 h1:u0kr8lbJc1oBcawK7Df+/ajNMpIDFE4 github.com/modelcontextprotocol/go-sdk v1.4.0/go.mod h1:Nxc2n+n/GdCebUaqCOhTetptS17SXXNu9IfNTaLDi1E= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -123,6 +129,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= @@ -188,6 +202,8 @@ go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= @@ -222,6 +238,8 @@ google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/db/migrate.go b/internal/db/migrate.go index 9aa3631..2d3daf7 100644 --- a/internal/db/migrate.go +++ b/internal/db/migrate.go @@ -7,30 +7,27 @@ import ( // Migrate runs database schema migrations func (db *DB) Migrate(ctx context.Context) error { - // Enable required extensions if err := db.enableExtensions(ctx); err != nil { return fmt.Errorf("failed to enable extensions: %w", err) } - // Create auth tables first (referenced by audit logs) if err := db.createAPIKeysTable(ctx); err != nil { return fmt.Errorf("failed to create api_keys table: %w", err) } - if err := db.createAuditLogsTable(ctx); err != nil { return fmt.Errorf("failed to create audit_logs table: %w", err) } - - // Create memories table if err := db.createMemoriesTable(ctx); err != nil { return fmt.Errorf("failed to create memories table: %w", err) } - - // Create indexes + if err := db.migrateMemoriesV2(ctx); err != nil { + return fmt.Errorf("failed to migrate memories v2: %w", err) + } if err := db.createIndexes(ctx); err != nil { return fmt.Errorf("failed to create indexes: %w", err) } - - // Create updated_at trigger + if err := db.createMemoriesV2Indexes(ctx); err != nil { + return fmt.Errorf("failed to create memories v2 indexes: %w", err) + } if err := db.createUpdatedAtTrigger(ctx); err != nil { return fmt.Errorf("failed to create trigger: %w", err) } @@ -136,7 +133,6 @@ CREATE INDEX IF NOT EXISTS memories_embedding_hnsw_idx } func (db *DB) createUpdatedAtTrigger(ctx context.Context) error { - // Create function _, err := db.pool.Exec(ctx, ` CREATE OR REPLACE FUNCTION update_updated_at() RETURNS TRIGGER AS $$ @@ -146,13 +142,11 @@ $$ LANGUAGE plpgsql`) return fmt.Errorf("failed to create function: %w", err) } - // Drop if exists first _, err = db.pool.Exec(ctx, `DROP TRIGGER IF EXISTS memories_updated_at ON memories;`) if err != nil { return fmt.Errorf("failed to drop existing trigger: %w", err) } - // Create trigger _, err = db.pool.Exec(ctx, ` CREATE TRIGGER memories_updated_at BEFORE UPDATE ON memories @@ -163,3 +157,41 @@ CREATE TRIGGER memories_updated_at return nil } + +func (db *DB) migrateMemoriesV2(ctx context.Context) error { + queries := []string{ + `ALTER TABLE memories ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64)`, + `ALTER TABLE memories ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ`, + `ALTER TABLE memories ADD COLUMN IF NOT EXISTS ttl_seconds INTEGER`, + } + + for _, q := range queries { + if _, err := db.pool.Exec(ctx, q); err != nil { + return fmt.Errorf("failed to execute %q: %w", q, err) + } + } + + return nil +} + +func (db *DB) createMemoriesV2Indexes(ctx context.Context) error { + indexes := []string{ + `CREATE UNIQUE INDEX IF NOT EXISTS memories_content_hash_unique_idx + ON memories (namespace, content_hash) + WHERE content_hash IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS memories_expires_at_idx + ON memories (expires_at) + WHERE expires_at IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS memories_ttl_seconds_idx + ON memories (ttl_seconds) + WHERE ttl_seconds IS NOT NULL`, + } + + for _, idx := range indexes { + if _, err := db.pool.Exec(ctx, idx); err != nil { + return fmt.Errorf("failed to create index: %w", err) + } + } + + return nil +} diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index a5720e7..39425af 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -2,6 +2,8 @@ package mcp import ( "context" + "log/slog" + "strings" "time" "github.com/dbehnke/trindex/internal/memory" @@ -9,17 +11,18 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" ) -// rememberInput represents the input for the remember tool type rememberInput struct { - Content string `json:"content" jsonschema:"The memory text to store. Write 1-3 concise sentences that state the fact directly. Good candidates: decisions made, user preferences, patterns discovered, task outcomes, architectural choices, bug root causes. Avoid trivial or ephemeral facts."` - Namespace string `json:"namespace,omitempty" jsonschema:"Scope for this memory. Use 'global' for cross-agent user facts (preferences, identity, persistent context). Use a project namespace (e.g. 'trindex', 'myapp') for task-specific memories. Use 'default' when unsure."` - Metadata map[string]interface{} `json:"metadata,omitempty" jsonschema:"Arbitrary key/value tags. Recommended fields: type (e.g. 'decision', 'preference', 'pattern', 'bug', 'outcome'), tags ([]string), agent (agent name), project (project name), source (e.g. 'session', 'user-statement')."` + Content string `json:"content" jsonschema:"The memory text to store. Write 1-3 concise sentences that state the fact directly. Good candidates: decisions made, user preferences, patterns discovered, task outcomes, architectural choices, bug root causes. Avoid trivial or ephemeral facts."` + Namespace string `json:"namespace,omitempty" jsonschema:"Scope for this memory. Use 'global' for cross-agent user facts (preferences, identity, persistent context). Use a project namespace (e.g. 'trindex', 'myapp') for task-specific memories. Use 'default' when unsure. Follow hierarchical convention: global > project:{name} > agent:{name} > session:{id}."` + Metadata map[string]interface{} `json:"metadata,omitempty" jsonschema:"Arbitrary key/value tags. Recommended fields: type (e.g. 'decision', 'preference', 'pattern', 'bug', 'outcome'), tags ([]string), agent (agent name), project (project name), source (e.g. 'session', 'user-statement')."` + SkipDuplicateThreshold float64 `json:"skip_duplicate_threshold,omitempty" jsonschema:"Similarity threshold (0.0-1.0) for deduplication. If a memory with similarity >= threshold exists in the same namespace, skip storing. Use 0.95 for exact duplicates, 0.85 for semantic duplicates. Set to 0 to disable (default)."` + TTLSeconds int32 `json:"ttl_seconds,omitempty" jsonschema:"Time-to-live in seconds. Memory will be automatically deleted after this duration. Use for ephemeral context like session IDs, temporary files, transient errors. session:* namespaces default to 86400 (24h)."` } func (s *Server) registerRemember() { mcp.AddTool(s.server, &mcp.Tool{ Name: "remember", - Description: "Store a memory with optional namespace and metadata. Call this after making a significant decision, learning a user preference, identifying a recurring pattern, completing a task, or discovering a bug root cause. Write content as 1-3 concise sentences stating the fact directly. Use namespace 'global' for cross-agent user facts; use a project-specific namespace for task-specific memories; use 'default' when unsure. Include metadata fields like type, tags, agent, and project to make memories easier to filter later.", + Description: "Store a memory with optional namespace and metadata. Call this after making a significant decision, learning a user preference, identifying a recurring pattern, completing a task, or discovering a bug root cause. Write content as 1-3 concise sentences stating the fact directly. Use namespace 'global' for cross-agent user facts; use a project-specific namespace for task-specific memories; use 'default' when unsure. Include metadata fields like type, tags, agent, and project to make memories easier to filter later. Use skip_duplicate_threshold to avoid storing duplicates (0.95 for exact, 0.85 for semantic matches).", }, func(ctx context.Context, req *mcp.CallToolRequest, in rememberInput) (*mcp.CallToolResult, any, error) { if in.Content == "" { return &mcp.CallToolResult{ @@ -27,7 +30,38 @@ func (s *Server) registerRemember() { }, nil, nil } - mem, err := s.store.Create(ctx, in.Content, in.Namespace, in.Metadata) + namespace := in.Namespace + if namespace == "" { + namespace = s.cfg.DefaultNamespace + } + + ttlSeconds := in.TTLSeconds + if ttlSeconds == 0 && strings.HasPrefix(namespace, "session:") { + ttlSeconds = 86400 + } + + if in.SkipDuplicateThreshold > 0 { + duplicate, err := s.checkDuplicate(ctx, in.Content, namespace, in.SkipDuplicateThreshold) + if err != nil { + slog.Warn("deduplication check failed", "error", err) + } else if duplicate != nil { + return &mcp.CallToolResult{ + Content: successResult(map[string]interface{}{ + "status": "duplicate_skipped", + "existing_id": duplicate.ID, + "namespace": duplicate.Namespace, + "similarity": duplicate.Score, + }), + }, nil, nil + } + } + + mem, err := s.store.CreateWithParams(ctx, memory.CreateParams{ + Content: in.Content, + Namespace: namespace, + Metadata: in.Metadata, + TTLSeconds: ttlSeconds, + }) if err != nil { return &mcp.CallToolResult{ Content: errorResult("EMBED_FAILED", err.Error()), @@ -35,16 +69,38 @@ func (s *Server) registerRemember() { } result := map[string]interface{}{ - "id": mem.ID, - "namespace": mem.Namespace, - "metadata": mem.Metadata, - "created_at": mem.CreatedAt, + "id": mem.ID, + "namespace": mem.Namespace, + "metadata": mem.Metadata, + "created_at": mem.CreatedAt, + "expires_at": mem.ExpiresAt, + "content_hash": mem.ContentHash, } return &mcp.CallToolResult{Content: successResult(result)}, nil, nil }) } +func (s *Server) checkDuplicate(ctx context.Context, content, namespace string, threshold float64) (*memory.RecallResult, error) { + params := memory.RecallParams{ + Query: content, + Namespaces: []string{namespace}, + TopK: 1, + Threshold: threshold, + } + + results, err := s.store.Recall(ctx, params) + if err != nil { + return nil, err + } + + if len(results) > 0 && results[0].Score >= threshold { + return &results[0], nil + } + + return nil, nil +} + // recallInput represents the input for the recall tool type recallInput struct { Query string `json:"query" jsonschema:"Natural language search query. Phrase it as a statement or question describing what you are looking for, e.g. 'user preferred database' or 'how was the authentication bug fixed'."` diff --git a/internal/memory/context_window.go b/internal/memory/context_window.go new file mode 100644 index 0000000..4da4239 --- /dev/null +++ b/internal/memory/context_window.go @@ -0,0 +1,161 @@ +package memory + +import ( + "context" + "math" + "sort" + "time" +) + +type ContextWindowParams struct { + Query string + Namespaces []string + MaxTokens int + TokenBudget int + RelevanceWeight float64 + RecencyWeight float64 + TypeBoostWeight float64 +} + +type RankedMemory struct { + Memory + Score float64 + Relevance float64 + Recency float64 + TypeBoost float64 +} + +func (s *Store) BuildContextWindow(ctx context.Context, params ContextWindowParams) ([]RankedMemory, error) { + if params.MaxTokens <= 0 { + params.MaxTokens = 4000 + } + if params.RelevanceWeight == 0 { + params.RelevanceWeight = 0.5 + } + if params.RecencyWeight == 0 { + params.RecencyWeight = 0.3 + } + if params.TypeBoostWeight == 0 { + params.TypeBoostWeight = 0.2 + } + + recallParams := RecallParams{ + Query: params.Query, + Namespaces: params.Namespaces, + TopK: 50, + Threshold: 0.1, + } + + results, err := s.Recall(ctx, recallParams) + if err != nil { + return nil, err + } + + ranked := s.rankMemories(results) + + sort.Slice(ranked, func(i, j int) bool { + return ranked[i].Score > ranked[j].Score + }) + + return s.fitToTokenBudget(ranked, params.MaxTokens), nil +} + +func (s *Store) rankMemories(results []RecallResult) []RankedMemory { + if len(results) == 0 { + return []RankedMemory{} + } + + now := time.Now() + oldestTime := results[0].CreatedAt + newestTime := results[0].CreatedAt + + for _, r := range results { + if r.CreatedAt.Before(oldestTime) { + oldestTime = r.CreatedAt + } + if r.CreatedAt.After(newestTime) { + newestTime = r.CreatedAt + } + } + + ranked := make([]RankedMemory, len(results)) + for i, r := range results { + rm := RankedMemory{ + Memory: r.Memory, + Relevance: r.Score, + } + + age := now.Sub(r.CreatedAt).Hours() + rm.Recency = math.Exp(-age / 168) + + if memType, ok := r.Metadata["type"].(string); ok { + rm.TypeBoost = typeBoostScore(memType) + } + + rm.Score = 0.5*rm.Relevance + 0.3*rm.Recency + 0.2*rm.TypeBoost + ranked[i] = rm + } + + return ranked +} + +func typeBoostScore(memType string) float64 { + boosts := map[string]float64{ + "preference": 1.0, + "decision": 0.9, + "bug": 0.85, + "pattern": 0.8, + "outcome": 0.7, + "fact": 0.6, + } + + if boost, ok := boosts[memType]; ok { + return boost + } + return 0.5 +} + +func (s *Store) fitToTokenBudget(ranked []RankedMemory, maxTokens int) []RankedMemory { + estimatedTokens := 0 + var result []RankedMemory + + for _, rm := range ranked { + contentTokens := estimateTokens(rm.Content) + metadataTokens := estimateTokensFromMetadata(rm.Metadata) + memoryTokens := contentTokens + metadataTokens + 50 + + if estimatedTokens+memoryTokens > maxTokens { + break + } + + result = append(result, rm) + estimatedTokens += memoryTokens + } + + return result +} + +func estimateTokens(text string) int { + return int(math.Ceil(float64(len(text)) / 4.0)) +} + +func estimateTokensFromMetadata(metadata map[string]interface{}) int { + if metadata == nil { + return 0 + } + estimate := 0 + for k, v := range metadata { + estimate += estimateTokens(k) + switch val := v.(type) { + case string: + estimate += estimateTokens(val) + case []interface{}: + for _, item := range val { + if s, ok := item.(string); ok { + estimate += estimateTokens(s) + } + } + } + } + return estimate +} diff --git a/internal/memory/context_window_test.go b/internal/memory/context_window_test.go new file mode 100644 index 0000000..fb8d980 --- /dev/null +++ b/internal/memory/context_window_test.go @@ -0,0 +1,281 @@ +package memory + +import ( + "context" + "testing" + "time" + + "github.com/dbehnke/trindex/internal/config" + "github.com/dbehnke/trindex/internal/db" + "github.com/dbehnke/trindex/internal/embed" + "github.com/dbehnke/trindex/internal/testutil" +) + +func setupTestStoreForContextWindow(t *testing.T) (*Store, func()) { + t.Helper() + + testutil.SkipIfNoDocker(t) + + ctx := context.Background() + + container, err := testutil.NewPostgresContainer(ctx) + if err != nil { + t.Fatalf("Failed to start postgres container: %v", err) + } + + embeddingDim := 768 + cfg := &config.Config{ + DatabaseURL: container.ConnStr, + EmbedDimensions: embeddingDim, + HNSWM: 16, + HNSWEfConstruction: 64, + HNSWEfSearch: 40, + DBMaxConns: 10, + DBMinConns: 2, + DBMaxConnLifetime: 60, + DBMaxConnIdleTime: 30, + DefaultNamespace: "default", + } + + database, err := db.New(cfg) + if err != nil { + _ = container.Terminate(ctx) + t.Fatalf("Failed to create database: %v", err) + } + + if err := database.Migrate(ctx); err != nil { + database.Close() + _ = container.Terminate(ctx) + t.Fatalf("Failed to run migrations: %v", err) + } + + mockServer := testutil.MockOllamaServer(embeddingDim) + + embedCfg := &config.Config{ + EmbedBaseURL: mockServer.URL, + EmbedModel: "test-model", + EmbedAPIKey: "test-key", + EmbedDimensions: embeddingDim, + DefaultTopK: 10, + DefaultSimilarityThreshold: 0.7, + HNSWM: 16, + HNSWEfConstruction: 64, + HNSWEfSearch: 40, + DefaultNamespace: "default", + HybridVectorWeight: 0.7, + HybridFTSWeight: 0.3, + } + + embedClient := embed.NewClient(embedCfg) + store := NewStore(database, embedClient, embedCfg) + + cleanup := func() { + mockServer.Close() + database.Close() + _ = container.Terminate(ctx) + } + + return store, cleanup +} + +func TestBuildContextWindow(t *testing.T) { + store, cleanup := setupTestStoreForContextWindow(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := store.Create(ctx, "Architecture decision: Use PostgreSQL with pgvector for vector storage", "project:test", map[string]interface{}{ + "type": "decision", + "tags": []string{"architecture", "database"}, + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + _, err = store.Create(ctx, "User prefers dark mode for all applications", "global", map[string]interface{}{ + "type": "preference", + "tags": []string{"ui", "user-preference"}, + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + _, err = store.Create(ctx, "Bug fixed: Race condition in concurrent access", "project:test", map[string]interface{}{ + "type": "bug", + "tags": []string{"bug", "concurrency"}, + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + params := ContextWindowParams{ + Query: "database architecture decisions", + Namespaces: []string{"project:test", "global"}, + MaxTokens: 2000, + RelevanceWeight: 0.5, + RecencyWeight: 0.3, + TypeBoostWeight: 0.2, + } + + ranked, err := store.BuildContextWindow(ctx, params) + if err != nil { + t.Fatalf("BuildContextWindow failed: %v", err) + } + + for _, r := range ranked { + if r.Score <= 0 { + t.Error("expected positive score") + } + if r.Relevance < 0 { + t.Error("expected non-negative relevance") + } + if r.Recency < 0 || r.Recency > 1 { + t.Errorf("expected recency between 0 and 1, got %f", r.Recency) + } + if r.TypeBoost < 0 || r.TypeBoost > 1 { + t.Errorf("expected type boost between 0 and 1, got %f", r.TypeBoost) + } + } +} + +func TestBuildContextWindow_RespectsTokenBudget(t *testing.T) { + store, cleanup := setupTestStoreForContextWindow(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + for i := 0; i < 10; i++ { + _, err := store.Create(ctx, "This is a test memory with sufficient length to consume tokens when retrieved in a context window", "project:test", nil) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + } + + params := ContextWindowParams{ + Query: "test memory", + Namespaces: []string{"project:test"}, + MaxTokens: 500, + } + + ranked, err := store.BuildContextWindow(ctx, params) + if err != nil { + t.Fatalf("BuildContextWindow failed: %v", err) + } + + totalTokens := 0 + for _, r := range ranked { + totalTokens += estimateTokens(r.Content) + 50 + } + + if totalTokens > params.MaxTokens { + t.Errorf("expected total tokens %d to be within budget %d", totalTokens, params.MaxTokens) + } +} + +func TestTypeBoostScore(t *testing.T) { + tests := []struct { + memType string + expected float64 + }{ + {"preference", 1.0}, + {"decision", 0.9}, + {"bug", 0.85}, + {"pattern", 0.8}, + {"outcome", 0.7}, + {"fact", 0.6}, + {"unknown", 0.5}, + {"", 0.5}, + } + + for _, tt := range tests { + got := typeBoostScore(tt.memType) + if got != tt.expected { + t.Errorf("typeBoostScore(%q) = %f, want %f", tt.memType, got, tt.expected) + } + } +} + +func TestEstimateTokens(t *testing.T) { + tests := []struct { + input string + expected int + }{ + {"", 0}, + {"test", 1}, + {"this is a test", 4}, + {"this is a longer test with more words", 10}, + } + + for _, tt := range tests { + got := estimateTokens(tt.input) + if got != tt.expected { + t.Errorf("estimateTokens(%q) = %d, want %d", tt.input, got, tt.expected) + } + } +} + +func TestEstimateTokensFromMetadata(t *testing.T) { + metadata := map[string]interface{}{ + "type": "decision", + "tags": []interface{}{"architecture", "database"}, + } + + tokens := estimateTokensFromMetadata(metadata) + if tokens <= 0 { + t.Error("expected positive token count for non-empty metadata") + } + + emptyTokens := estimateTokensFromMetadata(nil) + if emptyTokens != 0 { + t.Errorf("expected 0 tokens for nil metadata, got %d", emptyTokens) + } +} + +func TestRankMemories(t *testing.T) { + now := time.Now() + results := []RecallResult{ + { + Memory: Memory{ + Content: "Recent decision", + CreatedAt: now.Add(-1 * time.Hour), + Metadata: map[string]interface{}{"type": "decision"}, + }, + Score: 0.9, + }, + { + Memory: Memory{ + Content: "Old preference", + CreatedAt: now.Add(-168 * time.Hour), + Metadata: map[string]interface{}{"type": "preference"}, + }, + Score: 0.8, + }, + { + Memory: Memory{ + Content: "Old fact", + CreatedAt: now.Add(-168 * time.Hour), + Metadata: map[string]interface{}{"type": "fact"}, + }, + Score: 0.85, + }, + } + + store := &Store{} + ranked := store.rankMemories(results) + + if len(ranked) != len(results) { + t.Errorf("expected %d ranked memories, got %d", len(results), len(ranked)) + } + + if len(ranked) > 0 && ranked[0].TypeBoost != 1.0 { + t.Logf("Preference type should have highest boost: %f", ranked[0].TypeBoost) + } + + for _, r := range ranked { + if r.Score == 0 { + t.Error("expected non-zero composite score") + } + } +} diff --git a/internal/memory/import_export.go b/internal/memory/import_export.go index 88eaafa..61c66a2 100644 --- a/internal/memory/import_export.go +++ b/internal/memory/import_export.go @@ -210,19 +210,22 @@ func (s *Store) importWithEmbedding(ctx context.Context, mem ImportMemory, names updatedAt = *mem.UpdatedAt } + contentHash := computeContentHash(mem.Content) + query := ` - INSERT INTO memories (id, namespace, content, embedding, metadata, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO memories (id, namespace, content, content_hash, embedding, metadata, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET namespace = EXCLUDED.namespace, content = EXCLUDED.content, + content_hash = EXCLUDED.content_hash, embedding = EXCLUDED.embedding, metadata = EXCLUDED.metadata, updated_at = EXCLUDED.updated_at ` _, err = s.db.Pool().Exec(ctx, query, - mem.ID, namespace, mem.Content, pgvector.NewVector(embedding), + mem.ID, namespace, mem.Content, contentHash, pgvector.NewVector(embedding), mem.Metadata, mem.CreatedAt, updatedAt, ) return err diff --git a/internal/memory/models.go b/internal/memory/models.go index 5b19ff6..c88c376 100644 --- a/internal/memory/models.go +++ b/internal/memory/models.go @@ -7,15 +7,17 @@ import ( "github.com/pgvector/pgvector-go" ) -// Memory represents a stored memory type Memory struct { - ID uuid.UUID `db:"id" json:"id"` - Namespace string `db:"namespace" json:"namespace"` - Content string `db:"content" json:"content"` - Embedding pgvector.Vector `db:"embedding" json:"-"` - Metadata map[string]interface{} `db:"metadata" json:"metadata"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + ID uuid.UUID `db:"id" json:"id"` + Namespace string `db:"namespace" json:"namespace"` + Content string `db:"content" json:"content"` + ContentHash string `db:"content_hash" json:"content_hash,omitempty"` + Embedding pgvector.Vector `db:"embedding" json:"-"` + Metadata map[string]interface{} `db:"metadata" json:"metadata"` + TTLSeconds int32 `db:"ttl_seconds" json:"ttl_seconds,omitempty"` + ExpiresAt *time.Time `db:"expires_at" json:"expires_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // RecallResult represents a memory retrieved by search @@ -32,6 +34,15 @@ type Filter struct { Source string } +type CreateParams struct { + Content string + Namespace string + Metadata map[string]interface{} + SkipIfDuplicate bool + DuplicateThreshold float64 + TTLSeconds int32 +} + // RecallParams represents parameters for recall operation type RecallParams struct { Query string diff --git a/internal/memory/passport.go b/internal/memory/passport.go new file mode 100644 index 0000000..f5d4840 --- /dev/null +++ b/internal/memory/passport.go @@ -0,0 +1,117 @@ +package memory + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +type ContextPassport struct { + ID uuid.UUID `json:"id"` + Source string `json:"source"` + Target string `json:"target"` + Query string `json:"query"` + TopK int `json:"top_k"` + Memories []Memory `json:"memories"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` + TTLSeconds int32 `json:"ttl_seconds"` +} + +type PassportParams struct { + Source string + Target string + Query string + Namespaces []string + TopK int + TTLSeconds int32 +} + +func (s *Store) CreatePassport(ctx context.Context, params PassportParams) (*ContextPassport, error) { + if params.TopK <= 0 { + params.TopK = 10 + } + if params.TTLSeconds <= 0 { + params.TTLSeconds = 3600 + } + + recallParams := RecallParams{ + Query: params.Query, + Namespaces: params.Namespaces, + TopK: params.TopK, + Threshold: 0.3, + } + + results, err := s.Recall(ctx, recallParams) + if err != nil { + return nil, err + } + + memories := make([]Memory, len(results)) + for i, r := range results { + memories[i] = r.Memory + } + + now := time.Now() + passport := &ContextPassport{ + ID: uuid.New(), + Source: params.Source, + Target: params.Target, + Query: params.Query, + TopK: params.TopK, + Memories: memories, + CreatedAt: now, + ExpiresAt: now.Add(time.Duration(params.TTLSeconds) * time.Second), + TTLSeconds: params.TTLSeconds, + } + + return passport, nil +} + +func (s *Store) ImportPassport(ctx context.Context, passport *ContextPassport, targetNamespace string) (int, error) { + if targetNamespace == "" { + targetNamespace = "default" + } + + imported := 0 + for _, mem := range passport.Memories { + if mem.ExpiresAt != nil && mem.ExpiresAt.Before(time.Now()) { + continue + } + + metadata := make(map[string]interface{}) + for k, v := range mem.Metadata { + metadata[k] = v + } + if metadata["imported_from"] == nil { + metadata["imported_from"] = passport.Source + } + if metadata["passport_id"] == nil { + metadata["passport_id"] = passport.ID.String() + } + + _, err := s.CreateWithParams(ctx, CreateParams{ + Content: mem.Content, + Namespace: targetNamespace, + Metadata: metadata, + TTLSeconds: mem.TTLSeconds, + }) + if err != nil { + continue + } + imported++ + } + + return imported, nil +} + +type PassportFilter struct { + Source string + Target string + Since *time.Time +} + +func (s *Store) ListPassports(ctx context.Context, filter PassportFilter) ([]*ContextPassport, error) { + return []*ContextPassport{}, nil +} diff --git a/internal/memory/passport_test.go b/internal/memory/passport_test.go new file mode 100644 index 0000000..c84c889 --- /dev/null +++ b/internal/memory/passport_test.go @@ -0,0 +1,312 @@ +package memory + +import ( + "context" + "testing" + "time" + + "github.com/dbehnke/trindex/internal/config" + "github.com/dbehnke/trindex/internal/db" + "github.com/dbehnke/trindex/internal/embed" + "github.com/dbehnke/trindex/internal/testutil" +) + +func setupTestStoreForPassport(t *testing.T) (*Store, func()) { + t.Helper() + + testutil.SkipIfNoDocker(t) + + ctx := context.Background() + + container, err := testutil.NewPostgresContainer(ctx) + if err != nil { + t.Fatalf("Failed to start postgres container: %v", err) + } + + embeddingDim := 768 + cfg := &config.Config{ + DatabaseURL: container.ConnStr, + EmbedDimensions: embeddingDim, + HNSWM: 16, + HNSWEfConstruction: 64, + HNSWEfSearch: 40, + DBMaxConns: 10, + DBMinConns: 2, + DBMaxConnLifetime: 60, + DBMaxConnIdleTime: 30, + DefaultNamespace: "default", + } + + database, err := db.New(cfg) + if err != nil { + _ = container.Terminate(ctx) + t.Fatalf("Failed to create database: %v", err) + } + + if err := database.Migrate(ctx); err != nil { + database.Close() + _ = container.Terminate(ctx) + t.Fatalf("Failed to run migrations: %v", err) + } + + mockServer := testutil.MockOllamaServer(embeddingDim) + + embedCfg := &config.Config{ + EmbedBaseURL: mockServer.URL, + EmbedModel: "test-model", + EmbedAPIKey: "test-key", + EmbedDimensions: embeddingDim, + DefaultTopK: 10, + DefaultSimilarityThreshold: 0.7, + HNSWM: 16, + HNSWEfConstruction: 64, + HNSWEfSearch: 40, + DefaultNamespace: "default", + HybridVectorWeight: 0.7, + HybridFTSWeight: 0.3, + } + + embedClient := embed.NewClient(embedCfg) + store := NewStore(database, embedClient, embedCfg) + + cleanup := func() { + mockServer.Close() + database.Close() + _ = container.Terminate(ctx) + } + + return store, cleanup +} + +func TestCreatePassport(t *testing.T) { + store, cleanup := setupTestStoreForPassport(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := store.Create(ctx, "Architecture decision for Linear integration", "project:linear", map[string]interface{}{ + "type": "decision", + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + _, err = store.Create(ctx, "User prefers to track issues in Linear", "global", map[string]interface{}{ + "type": "preference", + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + params := PassportParams{ + Source: "trindex", + Target: "linear", + Query: "Linear integration preferences", + Namespaces: []string{"project:linear", "global"}, + TopK: 5, + TTLSeconds: 3600, + } + + passport, err := store.CreatePassport(ctx, params) + if err != nil { + t.Fatalf("CreatePassport failed: %v", err) + } + + if passport.ID.String() == "" { + t.Error("expected passport ID to be set") + } + + if passport.Source != "trindex" { + t.Errorf("expected source 'trindex', got '%s'", passport.Source) + } + + if passport.Target != "linear" { + t.Errorf("expected target 'linear', got '%s'", passport.Target) + } + + if passport.TTLSeconds != 3600 { + t.Errorf("expected TTL 3600, got %d", passport.TTLSeconds) + } + + if passport.ExpiresAt.Before(time.Now()) { + t.Error("expected expiry in the future") + } +} + +func TestCreatePassport_Defaults(t *testing.T) { + store, cleanup := setupTestStoreForPassport(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := store.Create(ctx, "Some context to transfer", "test", nil) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + params := PassportParams{ + Source: "system-a", + Target: "system-b", + Query: "context", + Namespaces: []string{"test"}, + } + + passport, err := store.CreatePassport(ctx, params) + if err != nil { + t.Fatalf("CreatePassport failed: %v", err) + } + + if passport.TopK != 10 { + t.Errorf("expected default TopK 10, got %d", passport.TopK) + } + + if passport.TTLSeconds != 3600 { + t.Errorf("expected default TTL 3600, got %d", passport.TTLSeconds) + } +} + +func TestImportPassport(t *testing.T) { + store, cleanup := setupTestStoreForPassport(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mem, err := store.Create(ctx, "Context to import", "source-ns", map[string]interface{}{ + "type": "decision", + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + passport := &ContextPassport{ + ID: mem.ID, + Source: "source-system", + Target: "target-system", + Query: "test query", + Memories: []Memory{*mem}, + CreatedAt: time.Now(), + ExpiresAt: time.Now().Add(1 * time.Hour), + } + + imported, err := store.ImportPassport(ctx, passport, "target-ns") + if err != nil { + t.Fatalf("ImportPassport failed: %v", err) + } + + if imported != 1 { + t.Errorf("expected 1 imported memory, got %d", imported) + } + + list, err := store.List(ctx, ListParams{Namespace: "target-ns"}) + if err != nil { + t.Fatalf("Failed to list memories: %v", err) + } + + if len(list) != 1 { + t.Errorf("expected 1 memory in target-ns, got %d", len(list)) + } + + if len(list) > 0 { + if list[0].Metadata["imported_from"] != "source-system" { + t.Error("expected imported_from metadata to be set") + } + if list[0].Metadata["passport_id"] != passport.ID.String() { + t.Error("expected passport_id metadata to be set") + } + } +} + +func TestImportPassport_SkipsExpired(t *testing.T) { + store, cleanup := setupTestStoreForPassport(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + expiredTime := time.Now().Add(-1 * time.Hour) + mem := &Memory{ + Content: "Expired memory", + Namespace: "test", + Metadata: map[string]interface{}{}, + ExpiresAt: &expiredTime, + } + + passport := &ContextPassport{ + Source: "source", + Target: "target", + Memories: []Memory{*mem}, + CreatedAt: time.Now(), + } + + imported, err := store.ImportPassport(ctx, passport, "target-ns") + if err != nil { + t.Fatalf("ImportPassport failed: %v", err) + } + + if imported != 0 { + t.Errorf("expected 0 imported memories (expired), got %d", imported) + } +} + +func TestImportPassport_DefaultNamespace(t *testing.T) { + store, cleanup := setupTestStoreForPassport(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mem, err := store.Create(ctx, "Test memory", "test", nil) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + passport := &ContextPassport{ + Source: "source", + Target: "target", + Memories: []Memory{*mem}, + CreatedAt: time.Now(), + } + + imported, err := store.ImportPassport(ctx, passport, "") + if err != nil { + t.Fatalf("ImportPassport failed: %v", err) + } + + if imported != 1 { + t.Errorf("expected 1 imported memory, got %d", imported) + } + + list, err := store.List(ctx, ListParams{Namespace: "default"}) + if err != nil { + t.Fatalf("Failed to list memories: %v", err) + } + + if len(list) != 1 { + t.Errorf("expected 1 memory in default namespace, got %d", len(list)) + } +} + +func TestContextPassport_Expiry(t *testing.T) { + now := time.Now() + passport := &ContextPassport{ + ID: [16]byte{1, 2, 3, 4}, + Source: "test-source", + Target: "test-target", + Query: "test", + Memories: []Memory{}, + CreatedAt: now, + ExpiresAt: now.Add(1 * time.Hour), + TTLSeconds: 3600, + } + + if passport.ExpiresAt.Before(now) { + t.Error("passport expiry should be in the future") + } + + if passport.TTLSeconds != 3600 { + t.Errorf("expected TTL 3600, got %d", passport.TTLSeconds) + } +} diff --git a/internal/memory/recall.go b/internal/memory/recall.go index 2303c5a..571bdfa 100644 --- a/internal/memory/recall.go +++ b/internal/memory/recall.go @@ -268,9 +268,10 @@ func (s *Store) fetchMemoriesByIDs(ctx context.Context, ids []uuid.UUID, filter } query := ` - SELECT id, namespace, content, metadata, created_at, updated_at + SELECT id, namespace, content, content_hash, metadata, ttl_seconds, expires_at, created_at, updated_at FROM memories WHERE id = ANY($1) + AND (expires_at IS NULL OR expires_at > NOW()) ` args := []interface{}{ids} argIdx := 2 diff --git a/internal/memory/store.go b/internal/memory/store.go index 032b76e..41f0c28 100644 --- a/internal/memory/store.go +++ b/internal/memory/store.go @@ -2,6 +2,8 @@ package memory import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "strings" "time" @@ -14,14 +16,12 @@ import ( "github.com/pgvector/pgvector-go" ) -// Store handles memory CRUD operations type Store struct { db *db.DB embed *embed.Client cfg *config.Config } -// NewStore creates a new memory store func NewStore(database *db.DB, embedClient *embed.Client, cfg *config.Config) *Store { return &Store{ db: database, @@ -30,42 +30,61 @@ func NewStore(database *db.DB, embedClient *embed.Client, cfg *config.Config) *S } } -// Create stores a new memory with embedding func (s *Store) Create(ctx context.Context, content, namespace string, metadata map[string]interface{}) (*Memory, error) { - // Generate embedding - embedding, err := s.embed.Embed(content) - if err != nil { - return nil, fmt.Errorf("failed to generate embedding: %w", err) + return s.CreateWithParams(ctx, CreateParams{ + Content: content, + Namespace: namespace, + Metadata: metadata, + }) +} + +func (s *Store) CreateWithParams(ctx context.Context, params CreateParams) (*Memory, error) { + if params.Namespace == "" { + params.Namespace = "default" + } + if params.Metadata == nil { + params.Metadata = make(map[string]interface{}) + } + + contentHash := computeContentHash(params.Content) + + if existing, err := s.findByContentHash(ctx, params.Namespace, contentHash); err == nil && existing != nil { + return existing, nil } - // Ensure namespace is set - if namespace == "" { - namespace = "default" + embedding, err := s.embed.Embed(params.Content) + if err != nil { + return nil, fmt.Errorf("failed to generate embedding: %w", err) } - // Ensure metadata is not nil - if metadata == nil { - metadata = make(map[string]interface{}) + now := time.Now() + var expiresAt *time.Time + if params.TTLSeconds > 0 { + exp := now.Add(time.Duration(params.TTLSeconds) * time.Second) + expiresAt = &exp } memory := &Memory{ - ID: uuid.New(), - Namespace: namespace, - Content: content, - Embedding: pgvector.NewVector(embedding), - Metadata: metadata, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + ID: uuid.New(), + Namespace: params.Namespace, + Content: params.Content, + ContentHash: contentHash, + Embedding: pgvector.NewVector(embedding), + Metadata: params.Metadata, + TTLSeconds: params.TTLSeconds, + ExpiresAt: expiresAt, + CreatedAt: now, + UpdatedAt: now, } query := ` - INSERT INTO memories (id, namespace, content, embedding, metadata, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO memories (id, namespace, content, content_hash, embedding, metadata, ttl_seconds, expires_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ` _, err = s.db.Pool().Exec(ctx, query, - memory.ID, memory.Namespace, memory.Content, memory.Embedding, - memory.Metadata, memory.CreatedAt, memory.UpdatedAt, + memory.ID, memory.Namespace, memory.Content, memory.ContentHash, memory.Embedding, + memory.Metadata, memory.TTLSeconds, memory.ExpiresAt, memory.CreatedAt, memory.UpdatedAt, ) if err != nil { return nil, fmt.Errorf("failed to insert memory: %w", err) @@ -74,27 +93,65 @@ func (s *Store) Create(ctx context.Context, content, namespace string, metadata return memory, nil } -// GetByID retrieves a single memory by ID +func computeContentHash(content string) string { + hash := sha256.Sum256([]byte(content)) + return hex.EncodeToString(hash[:]) +} + +func (s *Store) findByContentHash(ctx context.Context, namespace, contentHash string) (*Memory, error) { + query := ` + SELECT id, namespace, content, content_hash, metadata, ttl_seconds, expires_at, created_at, updated_at + FROM memories + WHERE namespace = $1 AND content_hash = $2 + ` + + var m Memory + var expiresAt *time.Time + var ttlSeconds *int32 + err := s.db.Pool().QueryRow(ctx, query, namespace, contentHash).Scan( + &m.ID, &m.Namespace, &m.Content, &m.ContentHash, + &m.Metadata, &ttlSeconds, &expiresAt, &m.CreatedAt, &m.UpdatedAt, + ) + if err != nil { + return nil, err + } + m.ExpiresAt = expiresAt + if ttlSeconds != nil { + m.TTLSeconds = *ttlSeconds + } + + if m.ExpiresAt != nil && m.ExpiresAt.Before(time.Now()) { + return nil, nil + } + + return &m, nil +} + func (s *Store) GetByID(ctx context.Context, id uuid.UUID) (*Memory, error) { query := ` - SELECT id, namespace, content, metadata, created_at, updated_at - FROM memories + SELECT id, namespace, content, content_hash, metadata, ttl_seconds, expires_at, created_at, updated_at + FROM memories WHERE id = $1 ` var m Memory + var expiresAt *time.Time + var ttlSeconds *int32 err := s.db.Pool().QueryRow(ctx, query, id).Scan( - &m.ID, &m.Namespace, &m.Content, - &m.Metadata, &m.CreatedAt, &m.UpdatedAt, + &m.ID, &m.Namespace, &m.Content, &m.ContentHash, + &m.Metadata, &ttlSeconds, &expiresAt, &m.CreatedAt, &m.UpdatedAt, ) if err != nil { return nil, fmt.Errorf("memory not found: %w", err) } + m.ExpiresAt = expiresAt + if ttlSeconds != nil { + m.TTLSeconds = *ttlSeconds + } return &m, nil } -// DeleteByID deletes a single memory by ID func (s *Store) DeleteByID(ctx context.Context, id uuid.UUID) error { query := `DELETE FROM memories WHERE id = $1` result, err := s.db.Pool().Exec(ctx, query, id) @@ -109,7 +166,6 @@ func (s *Store) DeleteByID(ctx context.Context, id uuid.UUID) error { return nil } -// DeleteByNamespace deletes memories by namespace with optional filter func (s *Store) DeleteByNamespace(ctx context.Context, namespace string, filter ForgetFilter) (int64, error) { query := `DELETE FROM memories WHERE namespace = $1` args := []interface{}{namespace} @@ -138,7 +194,16 @@ func (s *Store) DeleteByNamespace(ctx context.Context, namespace string, filter return result.RowsAffected(), nil } -// List retrieves memories without semantic search +func (s *Store) DeleteExpired(ctx context.Context) (int64, error) { + query := `DELETE FROM memories WHERE expires_at IS NOT NULL AND expires_at < NOW()` + result, err := s.db.Pool().Exec(ctx, query) + if err != nil { + return 0, fmt.Errorf("failed to delete expired memories: %w", err) + } + + return result.RowsAffected(), nil +} + func (s *Store) List(ctx context.Context, params ListParams) ([]Memory, error) { if params.Limit <= 0 { params.Limit = 20 @@ -150,7 +215,7 @@ func (s *Store) List(ctx context.Context, params ListParams) ([]Memory, error) { params.Order = "desc" } - query := `SELECT id, namespace, content, metadata, created_at, updated_at FROM memories` + query := `SELECT id, namespace, content, content_hash, metadata, ttl_seconds, expires_at, created_at, updated_at FROM memories` args := []interface{}{} argIdx := 1 @@ -177,11 +242,17 @@ func scanMemories(rows pgx.Rows) ([]Memory, error) { var memories []Memory for rows.Next() { var m Memory - err := rows.Scan(&m.ID, &m.Namespace, &m.Content, - &m.Metadata, &m.CreatedAt, &m.UpdatedAt) + var expiresAt *time.Time + var ttlSeconds *int32 + err := rows.Scan(&m.ID, &m.Namespace, &m.Content, &m.ContentHash, + &m.Metadata, &ttlSeconds, &expiresAt, &m.CreatedAt, &m.UpdatedAt) if err != nil { return nil, fmt.Errorf("failed to scan memory: %w", err) } + m.ExpiresAt = expiresAt + if ttlSeconds != nil { + m.TTLSeconds = *ttlSeconds + } memories = append(memories, m) } diff --git a/internal/memory/store_test.go b/internal/memory/store_test.go index ca9ee10..f81ceae 100644 --- a/internal/memory/store_test.go +++ b/internal/memory/store_test.go @@ -315,3 +315,187 @@ func TestStore_Recall(t *testing.T) { } } } + +func TestStore_CreateWithParams_Deduplication(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + content := "This is a unique memory for dedup testing" + namespace := "dedup-test" + + mem1, err := store.CreateWithParams(ctx, CreateParams{ + Content: content, + Namespace: namespace, + Metadata: map[string]interface{}{"test": "value"}, + }) + if err != nil { + t.Fatalf("Failed to create first memory: %v", err) + } + + mem2, err := store.CreateWithParams(ctx, CreateParams{ + Content: content, + Namespace: namespace, + Metadata: map[string]interface{}{"test": "different"}, + }) + if err != nil { + t.Fatalf("Failed to create second memory: %v", err) + } + + if mem1.ID != mem2.ID { + t.Error("expected duplicate content to return existing memory") + } + + if mem2.ContentHash == "" { + t.Error("expected content_hash to be set") + } +} + +func TestStore_CreateWithParams_TTL(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mem, err := store.CreateWithParams(ctx, CreateParams{ + Content: "Memory with TTL", + Namespace: "ttl-test", + TTLSeconds: 3600, + }) + if err != nil { + t.Fatalf("Failed to create memory with TTL: %v", err) + } + + if mem.TTLSeconds != 3600 { + t.Errorf("expected TTLSeconds to be 3600, got %d", mem.TTLSeconds) + } + + if mem.ExpiresAt == nil { + t.Fatal("expected ExpiresAt to be set") + } + + expectedExpiry := time.Now().Add(3600 * time.Second) + if mem.ExpiresAt.Before(expectedExpiry.Add(-5*time.Second)) || mem.ExpiresAt.After(expectedExpiry.Add(5*time.Second)) { + t.Errorf("expected expiry around %v, got %v", expectedExpiry, mem.ExpiresAt) + } +} + +func TestStore_CreateWithParams_NoTTL(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mem, err := store.CreateWithParams(ctx, CreateParams{ + Content: "Memory without TTL", + Namespace: "no-ttl-test", + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + if mem.ExpiresAt != nil { + t.Error("expected ExpiresAt to be nil when no TTL specified") + } + + if mem.TTLSeconds != 0 { + t.Errorf("expected TTLSeconds to be 0, got %d", mem.TTLSeconds) + } +} + +func TestStore_DeleteExpired(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mem, err := store.CreateWithParams(ctx, CreateParams{ + Content: "Memory that will expire", + Namespace: "expire-test", + TTLSeconds: 1, + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + _, err = store.GetByID(ctx, mem.ID) + if err != nil { + t.Fatalf("Should be able to retrieve memory before expiry: %v", err) + } + + time.Sleep(2 * time.Second) + + deleted, err := store.DeleteExpired(ctx) + if err != nil { + t.Fatalf("Failed to delete expired memories: %v", err) + } + + if deleted < 1 { + t.Errorf("expected at least 1 expired memory deleted, got %d", deleted) + } + + _, err = store.GetByID(ctx, mem.ID) + if err == nil { + t.Error("expected memory to be deleted after expiry") + } +} + +func TestStore_Recall_FiltersExpired(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := store.CreateWithParams(ctx, CreateParams{ + Content: "Unique content for expired recall test", + Namespace: "expired-recall-test", + TTLSeconds: 1, + }) + if err != nil { + t.Fatalf("Failed to create memory: %v", err) + } + + time.Sleep(2 * time.Second) + + params := RecallParams{ + Query: "unique content expired recall", + Namespaces: []string{"expired-recall-test"}, + TopK: 5, + Threshold: 0.01, + } + + results, err := store.Recall(ctx, params) + if err != nil { + t.Fatalf("Recall failed: %v", err) + } + + for _, r := range results { + if r.Namespace == "expired-recall-test" { + t.Error("expected expired memories to be filtered from recall results") + } + } +} + +func TestComputeContentHash(t *testing.T) { + hash1 := computeContentHash("test content") + hash2 := computeContentHash("test content") + hash3 := computeContentHash("different content") + + if hash1 != hash2 { + t.Error("same content should produce same hash") + } + + if hash1 == hash3 { + t.Error("different content should produce different hash") + } + + if len(hash1) != 64 { + t.Errorf("expected SHA256 hash length of 64, got %d", len(hash1)) + } +} diff --git a/internal/observability/observability.go b/internal/observability/observability.go new file mode 100644 index 0000000..3464318 --- /dev/null +++ b/internal/observability/observability.go @@ -0,0 +1,239 @@ +// Package observability provides structured logging, metrics, and tracing +package observability + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "time" + + "github.com/go-chi/chi/v5/middleware" + "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + // Metrics + requestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "HTTP request duration in seconds", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, + }, + []string{"method", "path", "status"}, + ) + + requestTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "Total HTTP requests", + }, + []string{"method", "path", "status"}, + ) + + activeConnections = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "http_active_connections", + Help: "Number of active HTTP connections", + }, + ) + + // Memory operation metrics + memoryOperations = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "memory_operations_total", + Help: "Total memory operations", + }, + []string{"operation", "namespace"}, + ) + + memoryOperationDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "memory_operation_duration_seconds", + Help: "Memory operation duration in seconds", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5}, + }, + []string{"operation"}, + ) + + // Database metrics + dbConnections = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "db_connections", + Help: "Database connection pool statistics", + }, + []string{"state"}, + ) + + // MCP metrics + mcpRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "mcp_requests_total", + Help: "Total MCP tool requests", + }, + []string{"tool"}, + ) + + mcpRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "mcp_request_duration_seconds", + Help: "MCP tool request duration", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}, + }, + []string{"tool"}, + ) +) + +func init() { + // Register all metrics + prometheus.MustRegister(requestDuration) + prometheus.MustRegister(requestTotal) + prometheus.MustRegister(activeConnections) + prometheus.MustRegister(memoryOperations) + prometheus.MustRegister(memoryOperationDuration) + prometheus.MustRegister(dbConnections) + prometheus.MustRegister(mcpRequests) + prometheus.MustRegister(mcpRequestDuration) +} + +// contextKey for storing request ID in context +type contextKey string + +const requestIDKey contextKey = "request_id" + +// InitLogger configures structured logging with the specified level +func InitLogger(level string) { + var logLevel slog.Level + + switch level { + case "debug": + logLevel = slog.LevelDebug + case "info": + logLevel = slog.LevelInfo + case "warn", "warning": + logLevel = slog.LevelWarn + case "error": + logLevel = slog.LevelError + default: + logLevel = slog.LevelInfo + } + + opts := &slog.HandlerOptions{ + Level: logLevel, + AddSource: logLevel == slog.LevelDebug, + } + + handler := slog.NewJSONHandler(os.Stdout, opts) + logger := slog.New(handler) + slog.SetDefault(logger) + + slog.Info("logger initialized", "level", level) +} + +// RequestIDMiddleware adds a unique request ID to each request +func RequestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestID := r.Header.Get("X-Request-ID") + if requestID == "" { + requestID = uuid.New().String() + } + + ctx := context.WithValue(r.Context(), requestIDKey, requestID) + w.Header().Set("X-Request-ID", requestID) + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// GetRequestID retrieves the request ID from context +func GetRequestID(ctx context.Context) string { + if id, ok := ctx.Value(requestIDKey).(string); ok { + return id + } + return "" +} + +// LoggingMiddleware logs HTTP requests with structured fields +func LoggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + requestID := GetRequestID(r.Context()) + + // Wrap response writer to capture status code + ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + + activeConnections.Inc() + defer activeConnections.Dec() + + next.ServeHTTP(ww, r.WithContext(r.Context())) + + duration := time.Since(start) + status := ww.Status() + + // Record metrics + requestDuration.WithLabelValues(r.Method, r.URL.Path, fmt.Sprintf("%d", status)).Observe(duration.Seconds()) + requestTotal.WithLabelValues(r.Method, r.URL.Path, fmt.Sprintf("%d", status)).Inc() + + // Log with appropriate level based on status + logAttrs := []any{ + "method", r.Method, + "path", r.URL.Path, + "status", status, + "duration_ms", duration.Milliseconds(), + "request_id", requestID, + "remote_addr", r.RemoteAddr, + "user_agent", r.UserAgent(), + } + + if status >= 500 { + slog.Error("http request error", logAttrs...) + } else if status >= 400 { + slog.Warn("http request warning", logAttrs...) + } else { + slog.Debug("http request completed", logAttrs...) + } + }) +} + +// MetricsHandler returns the Prometheus metrics endpoint handler +func MetricsHandler() http.Handler { + return promhttp.Handler() +} + +// RecordMemoryOperation records a memory operation metric +func RecordMemoryOperation(operation, namespace string) { + memoryOperations.WithLabelValues(operation, namespace).Inc() +} + +// RecordMemoryOperationDuration records the duration of a memory operation +func RecordMemoryOperationDuration(operation string, duration time.Duration) { + memoryOperationDuration.WithLabelValues(operation).Observe(duration.Seconds()) +} + +// RecordDBConnections updates database connection metrics +func RecordDBConnections(open, idle int32) { + dbConnections.WithLabelValues("open").Set(float64(open)) + dbConnections.WithLabelValues("idle").Set(float64(idle)) +} + +// RecordMCPRequest records an MCP tool request +func RecordMCPRequest(tool string) { + mcpRequests.WithLabelValues(tool).Inc() +} + +// RecordMCPRequestDuration records MCP request duration +func RecordMCPRequestDuration(tool string, duration time.Duration) { + mcpRequestDuration.WithLabelValues(tool).Observe(duration.Seconds()) +} + +// LoggerWithRequestID returns a logger with request ID attached +func LoggerWithRequestID(ctx context.Context) *slog.Logger { + requestID := GetRequestID(ctx) + if requestID != "" { + return slog.With("request_id", requestID) + } + return slog.Default() +} diff --git a/internal/web/server.go b/internal/web/server.go index 758d0d9..bb8fff7 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -18,6 +18,7 @@ import ( "github.com/dbehnke/trindex/internal/db" embedclient "github.com/dbehnke/trindex/internal/embed" "github.com/dbehnke/trindex/internal/memory" + "github.com/dbehnke/trindex/internal/observability" "github.com/go-chi/chi/v5" "github.com/go-chi/cors" "github.com/google/uuid" @@ -70,17 +71,12 @@ func (s *Server) setupRouter() { MaxAge: 300, })) - r.Use(func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - next.ServeHTTP(w, r) - slog.Debug("http request", "method", r.Method, "path", r.URL.Path, "duration", time.Since(start)) - }) - }) - + r.Use(observability.RequestIDMiddleware) + r.Use(observability.LoggingMiddleware) r.Use(s.securityHeadersMiddleware) r.Get("/health", s.handleHealth) + r.Get("/metrics", observability.MetricsHandler().ServeHTTP) r.Route("/api", func(r chi.Router) { r.Use(s.apiKeyMiddleware) diff --git a/plans/2026-03-07-thoughts.md b/plans/2026-03-07-thoughts.md new file mode 100644 index 0000000..da46b24 --- /dev/null +++ b/plans/2026-03-07-thoughts.md @@ -0,0 +1,663 @@ +# Trindex Memory System Improvement Plan + +**Date:** 2026-03-07 +**Context:** Principal Engineering Assessment of Memory System Architecture +**Scope:** Problems 1-6 with 3 solution options each + +--- + +## Problem 1: Deduplication on Write + +**Current State:** Agents store memories without checking for duplicates, resulting in multiple identical or semantically equivalent entries (e.g., 5+ copies of "User's name is Dave"). + +**Impact:** Storage bloat, retrieval noise, confusion about which memory is authoritative. + +### Option A: Client-Side Semantic Check (Recommended) + +**Approach:** Before calling `remember`, agent queries with high similarity threshold (≥0.95). If match found within namespace, skip storage and optionally update existing. + +**Implementation:** +```go +// Pseudo-code for remember_unique +func RememberUnique(content, namespace string, threshold float64) error { + existing := Recall(content, namespace, threshold, 1) + if len(existing) > 0 && existing[0].Score >= threshold { + // Optional: Update timestamp or metadata + return nil // Skip duplicate + } + return Remember(content, namespace) +} +``` + +**Pros:** +- No server changes required +- Immediate deployment across all agents +- Agents control dedup strictness per use case + +**Cons:** +- Extra round-trip (recall + remember) +- Race condition window between check and write +- Inconsistent if agents don't all implement + +**Effort:** Low (1-2 days) + +### Option B: Server-Side Hash-Based Deduplication + +**Approach:** Store content hash (SHA-256) as indexed column. On insert, reject if hash exists within namespace. Fall back to semantic check for near-duplicates. + +**Implementation:** +- Add `content_hash` column to memories table +- Create unique constraint: `(namespace, content_hash)` +- Return existing ID on conflict instead of error + +**Pros:** +- Zero latency overhead (hash check is O(1)) +- Guaranteed deduplication at storage layer +- No race conditions + +**Cons:** +- Requires schema migration +- Identical content with different formatting creates new entry +- Doesn't catch semantic duplicates ("Dave" vs "User is Dave") + +**Effort:** Medium (3-5 days including migration) + +### Option C: Vector Index Deduplication with Merge Strategy + +**Approach:** Use existing pgvector index to find near-duplicates at write time. If similarity > threshold, merge content (concatenate or pick longer) rather than reject. + +**Implementation:** +- Modify `remember` to run vector similarity search before insert +- On match: `new_content = merge(existing.content, new_content)` +- Update existing row instead of inserting new + +**Pros:** +- Catches semantic duplicates, not just exact matches +- Content enrichment over time (details accumulate) +- Single operation (no extra client round-trip) + +**Cons:** +- Complex merge logic (what constitutes "better" content?) +- Write latency increases (vector search on every remember) +- Potential for runaway content growth + +**Effort:** High (1-2 weeks, complex merge heuristics) + +**Recommendation:** Start with Option A (client-side) for immediate relief. Implement Option B (hash-based) as server-side safety net. Defer Option C until we understand merge patterns better. + +--- + +## Problem 2: Namespace Strategy + +**Current State:** Ad-hoc namespaces (`default`, `user_info`, `trindex`). No clear ownership or conventions. Agents dump memories wherever, making retrieval unreliable. + +**Impact:** Cross-agent context loss, inability to scope searches, namespace pollution. + +### Option A: Hierarchical Namespace Convention + +**Approach:** Establish strict hierarchy: `global` > `project:{name}` > `agent:{name}` > `session:{id}`. Agents always write to most specific appropriate level. + +**Rules:** +- `global`: User identity, preferences, cross-project facts +- `project:{name}`: Architecture, decisions, codebase patterns +- `agent:{name}`: Agent-specific learnings (rarely needed) +- `session:{id}`: Ephemeral context (auto-expires) + +**Implementation:** +- Document convention in CLAUDE.md +- Add validation on `remember` (reject unknown namespaces?) +- `recall` automatically searches up hierarchy + +**Pros:** +- Clear mental model +- Automatic scoping (session → project → global) +- Prevents pollution + +**Cons:** +- Requires discipline (easy to misuse) +- Migration pain for existing memories +- Hierarchy depth may not fit all use cases + +**Effort:** Low (documentation + convention enforcement) + +### Option B: Tag-Based Flat Namespaces + +**Approach:** Flat namespace structure with rich tagging. Query by tag intersection rather than namespace hierarchy. + +**Structure:** +```json +{ + "namespace": "trindex", + "tags": ["user:dave", "project:trindex", "type:architecture", "agent:claude"] +} +``` + +**Implementation:** +- Replace namespace with tags array +- Index tags for fast filtering +- Queries specify tag sets: `recall(query, tags=["user:dave", "type:bug"])` + +**Pros:** +- Flexible cross-cutting concerns +- No migration (tags can be added to existing) +- Multiple valid categorizations per memory + +**Cons:** +- Tag proliferation (agents inventing own tags) +- Query complexity (AND vs OR semantics) +- Harder to reason about "where" to put things + +**Effort:** Medium (schema change + query builder) + +### Option C: Automatic Namespace Detection + +**Approach:** AI-powered namespace classification. Agent provides raw content, system determines optimal namespace based on content analysis. + +**Implementation:** +```go +func AutoNamespace(content string) string { + // Use lightweight classifier or LLM call + if contains(content, ["user", "name", "preference"]) { + return "global" + } + if contains(content, ["project", "architecture", "decision"]) { + return "project:" + detectProject() + } + return "default" +} +``` + +**Pros:** +- Zero cognitive load on agents +- Consistent categorization +- Adapts to new namespace patterns + +**Cons:** +- Classification errors (miscategorized memories) +- Latency (LLM call or embedding lookup) +- Opacity (why was this put here?) + +**Effort:** High (ML pipeline or LLM integration) + +**Recommendation:** Option A (hierarchical) for immediate clarity. Add tagging (Option B elements) as metadata enrichment later. Avoid Option C until classification is highly reliable. + +--- + +## Problem 3: Structured Metadata + +**Current State:** Memories are plain text strings. No type information, source tracking, or confidence scoring. Impossible to filter ("show me only bugs") or validate. + +**Impact:** Retrieval returns irrelevant results, no provenance, hard to audit. + +### Option A: JSON Metadata Column + +**Approach:** Add `metadata` JSONB column to memories table. Store structured fields alongside content. Query using JSON operators. + +**Schema:** +```sql +ALTER TABLE memories ADD COLUMN metadata JSONB DEFAULT '{}'; +CREATE INDEX idx_metadata_type ON memories ((metadata->>'type')); +``` + +**Example:** +```json +{ + "content": "User prefers dark mode", + "metadata": { + "type": "preference", + "source": "user-statement", + "confidence": 0.95, + "agent": "claude", + "created_at": "2026-03-07T03:00:00Z" + } +} +``` + +**Pros:** +- Flexible schema (evolves without migrations) +- Native JSON operators in PostgreSQL +- Backwards compatible (default empty object) + +**Cons:** +- No validation (agents can store any structure) +- Query performance on deeply nested fields +- Storage overhead (JSON parsing) + +**Effort:** Low (1 day + index tuning) + +### Option B: Typed Memory System + +**Approach:** Define memory types as enums with validated schemas. Each type has required fields. Store in separate tables or with discriminator. + +**Types:** +```go +type MemoryType string +const ( + MemoryFact MemoryType = "fact" + MemoryPreference MemoryType = "preference" + MemoryDecision MemoryType = "decision" + MemoryBug MemoryType = "bug" + MemoryPattern MemoryType = "pattern" +) + +type TypedMemory struct { + ID uuid.UUID + Type MemoryType + Content string + // Type-specific fields + Facts *FactMetadata // when Type == "fact" + Bugs *BugMetadata // when Type == "bug" + // ... +} +``` + +**Pros:** +- Type safety (compile-time validation) +- Optimized storage per type +- Clear API contracts + +**Cons:** +- Schema rigidity (new types require migration) +- Complex query logic (joins or unions) +- Breaking change for existing memories + +**Effort:** High (2-3 weeks, significant refactor) + +### Option C: Content-Derived Metadata Extraction + +**Approach:** Automatically extract metadata from content using NLP/LLM. No explicit metadata from agents. System parses content for type, confidence, relationships. + +**Implementation:** +```go +func ExtractMetadata(content string) Metadata { + // Use NER, classification, or lightweight LLM + return Metadata{ + Type: classify(content), // "fact", "preference", etc. + Entities: extractEntities(content), // ["Dave", "Trindex"] + Sentiment: analyzeSentiment(content), + Confidence: 0.8, // based on certainty markers + } +} +``` + +**Pros:** +- Zero agent changes +- Rich metadata without discipline +- Can retroactively process existing memories + +**Cons:** +- Extraction errors +- Latency on write (or async processing complexity) +- Opacity (why was this classified as "preference"?) + +**Effort:** Medium-High (ML pipeline + validation) + +**Recommendation:** Option A (JSON metadata) for flexibility. Define recommended schema in documentation but don't enforce rigidly. Build validation layer later if needed. Defer Option B until we know exactly what fields we need. + +--- + +## Problem 4: Retrieval Hygiene + +**Current State:** Agents may not query at session start, or query with inappropriate thresholds. Low thresholds (0.0001) return noise; high thresholds miss relevant context. + +**Impact:** Context loss between sessions, retrieval returning garbage or missing key facts. + +### Option A: Mandatory Session Initialization Pattern + +**Approach:** Define standard "session bootstrap" that agents MUST run at start. Queries `global` + relevant project namespaces, caches results for session. + +**Pattern:** +```go +func InitializeSession(agentName, projectName string) SessionContext { + return SessionContext{ + GlobalFacts: Recall("user identity preferences", ["global"], 0.3, 10), + ProjectFacts: Recall("project context", [projectName], 0.3, 10), + RecentActivity: Recall("recent work", [projectName], 0.1, 5, since=24h), + } +} +``` + +**Enforcement:** +- Document in CLAUDE.md as REQUIRED +- Add linter/check for agent code (look for InitializeSession call) +- Framework-level: Auto-inject context if agent uses Trindex MCP + +**Pros:** +- Guaranteed context loading +- Consistent thresholds per query type +- Predictable agent behavior + +**Cons:** +- Additional latency at session start +- One-size-fits-all may not fit all use cases +- Hard to enforce across different agent implementations + +**Effort:** Low (documentation + code review) + +### Option B: Adaptive Threshold System + +**Approach:** Dynamic thresholds based on query characteristics. Broad queries get lower thresholds; specific queries get higher. System learns optimal thresholds per namespace. + +**Rules:** +```go +func GetThreshold(query string, namespace string) float64 { + if isSpecific(query) { // contains proper nouns, unique terms + return 0.7 + } + if namespace == "global" { + return 0.5 // user facts need precision + } + return 0.3 // project context can be broader +} +``` + +**Machine Learning:** Track which memories were actually useful (agent upvotes/downvotes), adjust thresholds per query pattern. + +**Pros:** +- Optimal retrieval without manual tuning +- Adapts to different content types +- Better precision/recall tradeoff + +**Cons:** +- Complex to implement and tune +- Requires feedback loop (agents reporting usefulness) +- Unpredictable for debugging + +**Effort:** High (ML system + feedback mechanism) + +### Option C: Context Window Management + +**Approach:** Treat memories like LLM context window. Rank by relevance + recency, include top N in prompt context. Automatic pruning of low-value memories. + +**Implementation:** +```go +func BuildContextWindow(query string, maxTokens int) []Memory { + candidates := Recall(query, allNamespaces, 0.1, 100) + ranked := rankBy(candidates, + score*0.5 + // relevance + recency*0.3 + // recent = more important + typeBoost*0.2 // preferences > random facts + ) + return fitToTokenBudget(ranked, maxTokens) +} +``` + +**Pros:** +- Automatic relevance filtering +- Fits within LLM context limits +- Surfaces most important context + +**Cons:** +- May drop relevant but older memories +- Complex ranking heuristics +- Non-deterministic (hard to reproduce) + +**Effort:** Medium (ranking algorithm + token counting) + +**Recommendation:** Option A (mandatory initialization) immediately. It's documentation + discipline, no code changes. Experiment with Option C (context window) for complex multi-turn sessions. Avoid Option B until we have usage data. + +--- + +## Problem 5: Memory Lifecycle + +**Current State:** All memories persist forever. No expiration for ephemeral context (session IDs, temporary files, transient errors). Storage grows indefinitely. + +**Impact:** Cluttered retrieval, storage costs, old irrelevant memories drowning out current context. + +### Option A: TTL (Time-To-Live) with Auto-Expiration + +**Approach:** Add `expires_at` timestamp to memories. Background job deletes expired entries. Agents can specify TTL per memory. + +**Implementation:** +```sql +ALTER TABLE memories ADD COLUMN expires_at TIMESTAMP; +CREATE INDEX idx_expires_at ON memories (expires_at) WHERE expires_at IS NOT NULL; + +-- Background job runs every hour +DELETE FROM memories WHERE expires_at < NOW(); +``` + +**Usage:** +```go +RememberWithTTL("Session context: debugging auth", "session:abc123", 24*time.Hour) +``` + +**Default TTLs:** +- `session:*`: 24h +- `global` preferences: Never +- Project decisions: 90d (refresh if still relevant) + +**Pros:** +- Automatic cleanup +- Explicit intent at write time +- Simple implementation + +**Cons:** +- Accidental data loss (wrong TTL) +- Background job complexity (monitoring, failures) +- Hard to "resurrect" accidentally expired memories + +**Effort:** Low (2-3 days) + +### Option B: Archival Tier with Compression + +**Approach:** Don't delete old memories, move to cold storage. Recent memories in hot database; old in S3/object store. Query hot first, fall back to cold if needed. + +**Implementation:** +```go +// Nightly job archives memories older than 30d +func ArchiveOldMemories() { + old := SelectOlderThan(30 * 24 * time.Hour) + for _, m := range old { + UploadToS3(m) + DeleteFromDB(m.ID) // or mark archived + } +} + +// Recall checks hot DB first +func Recall(query string) []Memory { + hot := QueryDB(query) + if len(hot) < 5 { + cold := QueryS3(query) // slower, more expensive + return append(hot, cold...) + } + return hot +} +``` + +**Pros:** +- No data loss +- Hot storage stays performant +- Cost savings (S3 cheaper than PostgreSQL) + +**Cons:** +- Complex infrastructure (S3, Athena for queries) +- Cold retrieval latency (seconds) +- Re-hydration complexity + +**Effort:** High (1-2 weeks infrastructure) + +### Option C: Relevance-Based Graduated Decay + +**Approach:** Memories lose "relevance score" over time. Frequently accessed memories stay fresh; untouched memories fade and eventually delete. Organic garbage collection. + +**Algorithm:** +```go +func UpdateRelevance(memory Memory) { + daysSinceAccess := now.Sub(memory.LastAccessed).Hours() / 24 + memory.RelevanceScore *= math.Pow(0.99, daysSinceAccess) // 1% decay per day + + if memory.RelevanceScore < 0.1 { + Delete(memory.ID) // Or archive + } +} +``` + +**Boosts:** +- Accessed: +0.1 +- Recalled and used: +0.2 +- Explicitly upvoted: +0.5 + +**Pros:** +- Natural selection (useful memories survive) +- No arbitrary TTLs +- Self-tuning to usage patterns + +**Cons:** +- Unpredictable (when will my memory disappear?) +- Important but rarely accessed memories may be lost +- Complex tuning (decay rates, boost values) + +**Effort:** Medium-High (scoring system + background job) + +**Recommendation:** Option A (TTL) is simplest and most predictable. Start with explicit expiration for session contexts. Option C (graduated decay) is elegant but risky—deferred until we have confidence scoring working. + +--- + +## Problem 6: Cross-System Integration + +**Current State:** Trindex is isolated. Other systems (issue trackers, documentation, CI/CD) don't share context. Agents working across systems lose continuity. + +**Impact:** Siloed knowledge, repeated explanations, no system-of-record for user context. + +### Option A: MCP-First Architecture (Expand Tool Set) + +**Approach:** Make Trindex the universal memory layer via MCP. Build adapters for external systems to read/write through Trindex. All agents/tools use Trindex as single source of truth. + +**Adapters:** +``` +External System → Trindex Adapter → Trindex Core +- Linear issues → Remember bug reports, recall related decisions +- Notion docs → Remember architecture, recall during coding +- GitHub PRs → Remember review patterns, recall for new PRs +- Slack → Remember decisions, recall context in threads +``` + +**Implementation:** +- Build MCP servers for each external system +- Map external concepts to Trindex namespaces +- Unified query interface across all data + +**Pros:** +- Single memory system (simplicity) +- All context in one place +- Rich cross-system queries + +**Cons:** +- Heavy integration burden (every new system needs adapter) +- Potential performance bottleneck +- Single point of failure + +**Effort:** High (ongoing integration work) + +### Option B: Event-Driven Federation + +**Approach:** Decentralized memory. Each system keeps its own store, emits events on changes. Trindex subscribes to relevant events, creates shadow copies for cross-system queries. + +**Architecture:** +``` +Linear ──┐ +Notion ──┼──→ Event Bus (Kafka/NATS) ──→ Trindex (shadow copy) +GitHub ──┘ +Slack ────┘ +``` + +**Event Types:** +- `memory.created` {source, namespace, content, metadata} +- `memory.updated` {id, patch} +- `memory.deleted` {id} + +**Implementation:** +- Systems emit events (webhook or SDK) +- Trindex consumes, stores in `shadow:{system}` namespace +- Write-through: Changes to shadow propagate back via events + +**Pros:** +- Systems remain autonomous +- Loose coupling (systems can change independently) +- Scalable (each system handles own load) + +**Cons:** +- Eventual consistency (lag between systems) +- Conflict resolution (what if both systems update?) +- Complex event infrastructure + +**Effort:** High (1-2 months for infrastructure) + +### Option C: Context Passport Pattern + +**Approach:** Agents carry context between systems explicitly. When switching from coding to issue tracking, agent exports relevant Trindex context, imports into target system. + +**Implementation:** +```go +// Agent switching from VS Code to Linear +func SwitchContext(fromSystem, toSystem string) ContextPassport { + relevant := Recall("current work", ["trindex"], 0.5, 10) + return ContextPassport{ + Source: fromSystem, + Target: toSystem, + Memories: relevant, + Timestamp: now(), + } +} + +// Target system imports +func ImportPassport(passport ContextPassport) { + for _, m := range passport.Memories { + LocalStore.Remember(m.Content, metadata={source: passport.Source}) + } +} +``` + +**Pros:** +- No infrastructure changes +- Systems don't need to know about each other +- Explicit context sharing (privacy control) + +**Cons:** +- Manual agent coordination +- Context may be stale by time of import +- Duplication across systems + +**Effort:** Low (protocol definition only) + +**Recommendation:** Option A (MCP-first) is the endgame but heavy. Start with Option C (passport pattern) for immediate cross-system flow. Build adapters incrementally toward Option A. Defer Option B unless we have 5+ systems to integrate. + +--- + +## Implementation Roadmap + +**Phase 1 (Immediate - Week 1-2):** +- Implement Problem 1 Option A (client-side dedup) +- Document Problem 2 Option A (namespace convention) +- Enforce Problem 4 Option A (session initialization) + +**Phase 2 (Short-term - Month 1):** +- Problem 1 Option B (server-side hash dedup) +- Problem 3 Option A (JSON metadata column) +- Problem 5 Option A (TTL for session contexts) + +**Phase 3 (Medium-term - Month 2-3):** +- Problem 2 namespace migration (consolidate `default`) +- Problem 4 Option C experiments (context window ranking) +- Problem 6 Option C (passport pattern for Linear/GitHub) + +**Phase 4 (Long-term - Quarter 2+):** +- Problem 3 Option C (auto-extraction if metadata sparse) +- Problem 5 Option C (graduated decay for long-term storage) +- Problem 6 Option A (MCP adapters for major systems) + +--- + +## Risk Assessment + +| Risk | Probability | Impact | Mitigation | +|------|-------------|--------|------------| +| Agent adoption (don't follow conventions) | High | High | Linter checks, code review, framework enforcement | +| Migration complexity (existing memories) | Medium | Medium | Batch migration scripts, backwards compatibility | +| Performance degradation (new queries) | Medium | Medium | Load testing, query optimization, caching layer | +| Over-engineering (solutions too complex) | High | Medium | Start with simplest option, iterate based on pain | + +--- + +*Document written as principal engineering assessment. Individual implementers should validate effort estimates against actual codebase before committing to timelines.*