fix(embeddings): accept RFC 2045 last-line image folds

CHANGELOG.md

@@ -0,0 +1,27 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- `chunking_strategy=meaning_units` on `POST /v1/batch/embeddings` embeds email
+  parties, HTML blocks, embedded images, and paragraphs as separate vectors and
+  returns `chunk_units` with source offsets. Omit the field to keep the naruon
+  one-vector-per-input contract. Next action: send the raw invoice email and
+  search `chunk_units` for the invoice id.
+
+### Fixed
+
+- Meaning-unit HTML cuts keep innermost leaves, so a wrapped
+  `<div><p>Good morning</p><p>Invoice INV-…</p></div>` no longer becomes one
+  vector. RFC 2397 `data:image` units now accept charset parameters, URL-safe
+  `-_`, and RFC 2045 folded last lines (including a 76-column wrap whose
+  final line is 15 alphabet characters plus `=`). Next action: POST the raw
+  Gmail HTML or a 76-column MIME-wrapped scan with
+  `chunking_strategy=meaning_units` and search `chunk_units` — leftover
+  base64 must not be in the invoice vector.

README.md

@@ -187,7 +187,11 @@ is read from a **KV config store**, never `os.getenv`.
   embedding parts (`routing.embedding_max_tokens_per_request`, default 280,000;
   `routing.embedding_max_chars_per_part`, default 240,000) and reduces part
   vectors with a token-weighted average, so Azure/LiteLLM over-limit embedding
-  requests are split internally instead of surfacing as caller errors. It routes
+  requests are split internally instead of surfacing as caller errors. Send
+  `chunking_strategy=meaning_units` to embed email, HTML, RFC 2397/2045 image,
+  and paragraph units separately and read `chunk_units` for source offsets;
+  omit it to keep one vector per submitted string. A 76-column MIME-wrapped
+  scan must keep leftover base64 out of the invoice vector. It routes
   through the same RoutingPolicy/cost optimiser and `pg-llm-batch` embeddings
   backend (local in-process backend standalone), and records one usage-ledger row
   per original vector with the full attribution dimensions (service, team,
@@ -204,6 +208,7 @@ Grounding papers (LLM cost, routing, load balancing) live in
 
 ## Design Artifacts
 
+- [Meaning-unit chunking](docs/meaning_unit_chunking.md)
 - [Library research](docs/library_research.md)
 - [Product planning](docs/product_planning.md)
 - [Screen design](docs/screen_design.md)
@@ -284,4 +289,6 @@ python tests/test_commercial_proposal_packet.py
 python tests/test_commercial_purchase_approval_packet.py
 python tests/test_commercial_due_diligence_room.py
 python tests/test_commercial_investment_committee_memo.py
+python tests/test_meaning_unit_chunking.py
+python tests/test_embeddings_meaning_units_http_honesty.py
 ```

contextual_orchestrator/api_contract.py

@@ -440,6 +440,16 @@
                                         "description": "observability + attribution dims (service, team, group, company, provider)",
                                     },
                                     "attribution": {"type": "object"},
+                                    "chunking_strategy": {
+                                        "type": "string",
+                                        "enum": ["meaning_units"],
+                                        "description": (
+                                            "Omit to keep one vector per input (naruon contract). "
+                                            "meaning_units embeds email parties, HTML blocks, "
+                                            "embedded images, and paragraphs separately and "
+                                            "returns chunk_units with source offsets."
+                                        ),
+                                    },
                                 },
                             }
                         }
@@ -451,7 +461,7 @@
                             "Batch completed synchronously: "
                             "{batch_id, status, embeddings:[{index, embedding}], "
                             "cost_micro_usd, token_counts, total_tokens, part_count, "
-                            "input_part_counts, map_reduce}"
+                            "input_part_counts, map_reduce, optional chunk_units}"
                         )
                     },
                     "202": {"description": "Batch accepted; poll GET /v1/batch/embeddings/{batch_id}"},

contextual_orchestrator/cost_router.py

@@ -85,6 +85,7 @@ def __init__(
         self._embedding_part_counts: Dict[str, List[int]] = {}
         self._embedding_part_limits: Dict[str, Dict[str, int]] = {}
         self._embedding_documents: Dict[str, Dict[str, Any]] = {}
+        self._embedding_chunk_units: Dict[str, List[Dict[str, Any]]] = {}
 
     # ------------------------------------------------------------------
     # Provider / model resolution
@@ -286,12 +287,16 @@ def submit_embeddings_batch(
         requests, part_counts, part_limits = self._build_embedding_requests(
             inputs, model=model, attribution=shared_attribution
         )
-        job = self.embedding_batch_backend.submit(requests, metadata=metadata)
+        submit_metadata = dict(metadata or {})
+        chunk_units = submit_metadata.pop("chunk_units", None)
+        job = self.embedding_batch_backend.submit(requests, metadata=submit_metadata or None)
         self._embedding_jobs[job.job_id] = job
         self._embedding_requests[job.job_id] = requests
         self._embedding_input_counts[job.job_id] = len(inputs)
         self._embedding_part_counts[job.job_id] = part_counts
         self._embedding_part_limits[job.job_id] = part_limits
+        if chunk_units:
+            self._embedding_chunk_units[job.job_id] = list(chunk_units)
         return job
 
     def _build_embedding_requests(
@@ -471,12 +476,16 @@ def embeddings_batch_document(self, batch_id: str) -> Dict[str, Any]:
         job = self._require_embedding_job(batch_id)
         status = self.embedding_batch_backend.poll(job)
         if not status.get("is_complete"):
-            return {
+            pending = {
                 "batch_id": batch_id,
                 "status": status.get("status") or job.status,
                 "backend": job.backend,
                 "embeddings": None,
             }
+            pending_units = self._embedding_chunk_units.get(batch_id)
+            if pending_units:
+                pending["chunk_units"] = pending_units
+            return pending
 
         items: List[EmbeddingBatchResultItem] = self.embedding_batch_backend.retrieve(job)
         requests = self._embedding_requests.get(batch_id, [])
@@ -561,6 +570,9 @@ def embeddings_batch_document(self, batch_id: str) -> Dict[str, Any]:
             "currency_code": currency_code,
             "cost_micro_usd": int(round(total_cost_amount * 1_000_000)),
         }
+        chunk_units = self._embedding_chunk_units.get(batch_id)
+        if chunk_units:
+            document["chunk_units"] = chunk_units
         self._embedding_documents[batch_id] = document
         return document