From 7bb6e49e75863318eceb655155f191bd01f0ec3b Mon Sep 17 00:00:00 2001
From: Johnathan Neals <johnathanneals@gmail.com>
Date: Sat, 15 Aug 2026 10:54:38 -0600
Subject: [PATCH] fix: chunk long text before embedding to prevent
 context-length errors
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The embed() path sends full text to the provider with no size handling.
When input exceeds the model's context window (2048 tokens for
nomic-embed-text, 8191 for text-embedding-3-small), the API returns
400 and the memory store fails.

Adds automatic chunking at the create_embedding() level (above the
provider abstraction, so all providers benefit):
- Text exceeding 4000 chars is split into overlapping chunks
- Each chunk is embedded independently
- Chunk embeddings are mean-pooled + L2-normalized into one vector

Also routes create_embeddings() (batch) through the same guard so
both single and batch paths handle long content consistently.

The 4000-char threshold safely fits within nomic-embed-text's 2048-token
window even for dense technical content (~2-3 chars/token). The full
memory text is stored unchanged — only embedding generation is affected.
---
 src/embedder/__init__.py | 47 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/src/embedder/__init__.py b/src/embedder/__init__.py
index d4e6b1d..db420b3 100644
--- a/src/embedder/__init__.py
+++ b/src/embedder/__init__.py
@@ -458,14 +458,53 @@ def get_embedder(config_path: str = None) -> BaseEmbedder:
     return _embedder
 
 
+EMBEDDING_MAX_CHARS = 4_000
+EMBEDDING_CHUNK_OVERLAP = 500
+
+
+def _chunk_text(text: str, max_chars: int, overlap: int) -> List[str]:
+    """Split text into overlapping chunks."""
+    chunks = []
+    start = 0
+    while start < len(text):
+        chunks.append(text[start:start + max_chars])
+        start += max_chars - overlap
+    return chunks
+
+
+def _mean_pool(embeddings: List[List[float]]) -> List[float]:
+    """Mean-pool embeddings and L2-normalize for cosine similarity consistency."""
+    import math
+
+    dim = len(embeddings[0])
+    mean = [0.0] * dim
+    for emb in embeddings:
+        for i in range(dim):
+            mean[i] += emb[i]
+    norm = 0.0
+    for i in range(dim):
+        mean[i] /= len(embeddings)
+        norm += mean[i] * mean[i]
+    norm = math.sqrt(norm)
+    if norm > 0:
+        for i in range(dim):
+            mean[i] /= norm
+    return mean
+
+
 def create_embedding(text: str) -> List[float]:
-    """Convenience function to create an embedding."""
-    return get_embedder().embed(text)
+    """Create an embedding, automatically chunking long text."""
+    embedder = get_embedder()
+    if len(text) <= EMBEDDING_MAX_CHARS:
+        return embedder.embed(text)
+    chunks = _chunk_text(text, EMBEDDING_MAX_CHARS, EMBEDDING_CHUNK_OVERLAP)
+    embeddings = [embedder.embed(chunk) for chunk in chunks]
+    return _mean_pool(embeddings)
 
 
 def create_embeddings(texts: List[str]) -> List[List[float]]:
-    """Convenience function to create multiple embeddings."""
-    return get_embedder().embed_batch(texts)
+    """Create embeddings for multiple texts, chunking any that exceed the limit."""
+    return [create_embedding(text) for text in texts]
 
 
 # For backward compatibility