From 9de476209a6e584cefd06463692f39dfa4bcb366 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 01:52:35 +0000 Subject: [PATCH 01/13] fix(threading): collapse interior whitespace in Message-ID normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `normalize_message_id` stripped surrounding whitespace and angle brackets but left interior whitespace intact. A Message-ID (RFC 5322 §3.6.4) carries no interior whitespace, but header unfolding (RFC 5322 §2.2.3) can leave interior spaces/tabs when a folded header is rejoined, e.g. `` unfolds to ``. The old normalizer returned `abc@ example.com` for the folded form and `abc@example.com` for the clean form, so the same message arriving with a different fold boundary produced two distinct keys — `threading_service` (the sole thread-id owner) and `email_dedupe_service` would then split one message into two threads / fail to de-duplicate it. Collapse all interior whitespace after bracket stripping so folded and unfolded forms of the same Message-ID normalize equal. Valid Message-IDs contain no interior whitespace, so this is a no-op for well-formed input and a correctness fix for folded/malformed input; it also flows through `extract_reference_ids`, keeping References-header matching fold-insensitive. Tests: added `normalize_message_id` coverage (brackets/outer whitespace, empty/ None, the interior-unfolding-whitespace cases) and an `extract_reference_ids` case that de-dupes a reference split over a fold boundary. Verified the pure functions directly and lint-clean under the CI-pinned ruff 0.15.21 (the full pytest suite needs the app's asyncpg/crypto stack, unavailable in this sandbox; CI runs it). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/services/threading_service.py | 13 ++++++-- backend/tests/test_threading_service.py | 40 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/backend/services/threading_service.py b/backend/services/threading_service.py index 01c738432..97132addd 100644 --- a/backend/services/threading_service.py +++ b/backend/services/threading_service.py @@ -31,11 +31,20 @@ def generate_email_fingerprint( def normalize_message_id(value: str | None) -> str | None: - """Return the canonical persisted form for a Message-ID-like header.""" + """Return the canonical persisted form for a Message-ID-like header. + + A Message-ID (RFC 5322 section 3.6.4) carries no interior whitespace, but + header unfolding (RFC 5322 section 2.2.3) can leave interior spaces or tabs + when a folded header is rejoined -- e.g. ```` unfolds + to ````. Collapsing all interior whitespace keeps the + folded and unfolded forms of the same Message-ID equal, so de-duplication + and threading never split one message into two over a fold boundary. + """ if value is None: return None - normalized = str(value).strip().strip("<>").strip() + stripped = str(value).strip().strip("<>") + normalized = "".join(stripped.split()) return normalized or None diff --git a/backend/tests/test_threading_service.py b/backend/tests/test_threading_service.py index 9c0d03450..2b57a5464 100644 --- a/backend/tests/test_threading_service.py +++ b/backend/tests/test_threading_service.py @@ -1,6 +1,10 @@ import pytest -from services.threading_service import assign_thread_id +from services.threading_service import ( + assign_thread_id, + extract_reference_ids, + normalize_message_id, +) class _Result: @@ -143,3 +147,37 @@ async def test_existing_thread_lookup_is_scoped_to_owner_and_organization(): query_text = str(session.queries[-1]).lower() assert "email_records.user_id" in query_text assert "email_records.organization_id" in query_text + + +def test_normalize_message_id_strips_brackets_and_outer_whitespace(): + assert normalize_message_id("") == "abc@example.com" + assert normalize_message_id(" ") == "abc@example.com" + assert normalize_message_id("< abc@example.com >") == "abc@example.com" + assert normalize_message_id("<>") == "abc@example.com" + assert normalize_message_id("abc@example.com") == "abc@example.com" + + +def test_normalize_message_id_handles_empty_and_none(): + assert normalize_message_id(None) is None + assert normalize_message_id("") is None + assert normalize_message_id(" ") is None + assert normalize_message_id("<>") is None + + +def test_normalize_message_id_collapses_interior_unfolding_whitespace(): + # RFC 5322 section 2.2.3 header unfolding can leave interior whitespace when + # a folded Message-ID is rejoined; RFC 5322 section 3.6.4 msg-id carries + # none, so the folded and unfolded forms must normalize to the same value or + # dedup/threading would treat one message as two. + canonical = normalize_message_id("") + assert normalize_message_id("") == canonical + assert normalize_message_id("") == canonical + assert normalize_message_id("") == canonical + assert normalize_message_id("") == canonical + + +def test_extract_reference_ids_normalizes_folded_whitespace_and_dedupes(): + header = " \r\n " + # The first two are the same id split over a fold boundary, so only two + # distinct references remain, in header order. + assert extract_reference_ids(header) == ["a@x.com", "b@x.com"] From 266cd0ec24e9615aa3cbf14963baf81fe98e8e40 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 02:07:08 +0000 Subject: [PATCH 02/13] fix(email): normalize unknown-zone (-0000) Date to timezone-aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_extract_date` returned `parsedate_to_datetime`'s value directly. RFC 5322 §3.3 defines a "-0000" zone as "time zone unknown", for which `parsedate_to_datetime` returns a NAIVE datetime, whereas every other branch — a real offset, and the `datetime.now(timezone.utc)` fallback for missing/ malformed dates — yields a timezone-AWARE datetime. That inconsistency is a latent crash: sorting or comparing a `-0000` message's date against any other message's date raises `TypeError: can't compare offset-naive and offset-aware datetimes`, and a naive value misbinds the instant when persisted to a `timestamptz` column. `-0000` is common in real mail (list servers / MTAs that decline to assert a meaningful zone). Normalize the unknown zone to UTC so `_extract_date` always returns an aware datetime. Added `test_parse_eml_unknown_timezone_date_is_timezone_aware`; verified the full `tests/test_email_parser.py` (15 passed) and lint-clean under the CI-pinned ruff 0.15.21. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/services/email_parser.py | 8 ++++++++ backend/tests/test_email_parser.py | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/backend/services/email_parser.py b/backend/services/email_parser.py index 2d32454ad..b57f29d29 100644 --- a/backend/services/email_parser.py +++ b/backend/services/email_parser.py @@ -136,6 +136,14 @@ def _extract_date(msg: Message) -> datetime.datetime: if not parsed_date: parsed_date = datetime.datetime.now(datetime.timezone.utc) + elif parsed_date.tzinfo is None: + # RFC 5322 section 3.3: a "-0000" zone means the time zone is unknown, + # for which parsedate_to_datetime returns a naive datetime. Every other + # branch here yields a timezone-aware datetime, and mixing naive with + # aware datetimes raises TypeError on comparison/sorting and misbinds the + # instant when stored in a timestamptz column. Treat the unknown zone as + # UTC so the returned value is always timezone-aware. + parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc) return parsed_date diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index e44d03127..6e8980562 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -325,6 +325,33 @@ def test_parse_eml_missing_and_malformed_date(): os.unlink(temp_path2) +def test_parse_eml_unknown_timezone_date_is_timezone_aware(): + # RFC 5322 section 3.3: a "-0000" zone means the time zone is unknown, for + # which parsedate_to_datetime returns a *naive* datetime. Every other parse + # path yields an aware datetime, so the parser must normalize this to aware + # too -- otherwise sorting/comparing it against another message's date raises + # "can't compare offset-naive and offset-aware datetimes" and it misbinds the + # instant in a timestamptz column. + eml_content = b"""Message-ID: +From: test@test.com +To: recipient@test.com +Subject: Unknown zone +Date: Mon, 27 Apr 2026 10:00:00 -0000 + +Test.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".eml") as f: + f.write(eml_content) + temp_path = f.name + + try: + parsed = parse_eml(temp_path) + assert parsed["date"].tzinfo is not None + # must not raise offset-naive/aware TypeError + assert parsed["date"] <= datetime.datetime.now(datetime.timezone.utc) + finally: + os.unlink(temp_path) + + def test_parse_eml_io_error(): with pytest.raises(EmailParseError): parse_eml("/path/to/nonexistent/file.eml") From bb8880c7e3a0c16a446d91893baafdbd28bca384 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 04:13:55 +0000 Subject: [PATCH 03/13] fix(threading): parse In-Reply-To as RFC 5322 1*msg-id (multi-id + CFWS) assign_thread_id() parsed In-Reply-To with normalize_message_id(), which treats the whole header as a single opaque Message-ID: it strips only the outer <> and collapses interior whitespace. RFC 5322 section 3.6.4 defines In-Reply-To (like References) as 1*msg-id -- one or more angle-bracketed ids, each optionally wrapped in CFWS -- so two real-world shapes produced a garbage candidate that matched no existing thread and silently split the reply onto its own thread: " " -> "a@x> (sent from ...)" -> "parent@x>(sentfrom...)" (trailing CFWS) Parse In-Reply-To with extract_reference_ids() (already used for References, and already fold/CFWS/dedupe-correct), add every extracted parent id as a thread-lookup candidate, and use the first parent id as the deterministic root fallback. A single clean id is unaffected -- extract_reference_ids("") == ["parent@x"] -- so all existing threading behavior is preserved; only the previously-broken multi-id and CFWS forms change. Verification (backend/, --noconftest to skip the openai-dependent app import): - tests/test_threading_service.py: 13 passed (10 existing + 3 new TDD: multi-id match, multi-id root fallback, CFWS comment). - tests/test_threading_perf.py: 2 passed (no threading regression). - ruff check services/threading_service.py tests/test_threading_service.py: clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/services/threading_service.py | 31 ++++++++----- backend/tests/test_threading_service.py | 62 +++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/backend/services/threading_service.py b/backend/services/threading_service.py index 97132addd..c500e4d35 100644 --- a/backend/services/threading_service.py +++ b/backend/services/threading_service.py @@ -49,7 +49,14 @@ def normalize_message_id(value: str | None) -> str | None: def extract_reference_ids(value: str | None) -> list[str]: - """Extract canonical message IDs from a References header in header order.""" + """Extract canonical message IDs from a ``1*msg-id`` header in header order. + + RFC 5322 defines both References (section 3.6.4) and In-Reply-To + (section 3.6.4) as ``1*msg-id`` -- one or more angle-bracketed Message-IDs, + each optionally surrounded by CFWS -- so this extractor applies to either + header. Ids are canonicalized with :func:`normalize_message_id` and + de-duplicated while preserving header order. + """ if not value: return [] @@ -116,19 +123,21 @@ async def assign_thread_id( Determine the thread_id for a new email based on in_reply_to and references. If no existing match is found, generate a new thread_id. """ - in_reply_to = normalize_message_id(email_data.get("in_reply_to")) + # In-Reply-To (RFC 5322 section 3.6.4) is 1*msg-id, exactly like References, + # and each id may be wrapped in CFWS. Parse it with the same multi-id + # extractor rather than treating the whole header as one opaque Message-ID, + # so a reply that names several parents -- or a single id trailed by a + # comment -- still threads onto an existing ancestor instead of splitting off. + in_reply_to_ids = extract_reference_ids(email_data.get("in_reply_to")) references = extract_reference_ids(email_data.get("references")) existing_candidates = [] # Optimization: Use a set for O(1) membership checks to prevent O(n^2) deduplication of candidates seen = set() - if in_reply_to: - existing_candidates.append(in_reply_to) - seen.add(in_reply_to) - for ref in references: - if ref not in seen: - seen.add(ref) - existing_candidates.append(ref) + for candidate in (*in_reply_to_ids, *references): + if candidate not in seen: + seen.add(candidate) + existing_candidates.append(candidate) if existing_candidates: thread_ids_by_message_id = await _find_existing_thread_ids( @@ -147,8 +156,8 @@ async def assign_thread_id( if references: return references[0] - if in_reply_to: - return in_reply_to + if in_reply_to_ids: + return in_reply_to_ids[0] msg_id = normalize_message_id(email_data.get("message_id")) if msg_id: diff --git a/backend/tests/test_threading_service.py b/backend/tests/test_threading_service.py index 2b57a5464..b66043ccd 100644 --- a/backend/tests/test_threading_service.py +++ b/backend/tests/test_threading_service.py @@ -181,3 +181,65 @@ def test_extract_reference_ids_normalizes_folded_whitespace_and_dedupes(): # The first two are the same id split over a fold boundary, so only two # distinct references remain, in header order. assert extract_reference_ids(header) == ["a@x.com", "b@x.com"] + + +@pytest.mark.asyncio +async def test_multi_id_in_reply_to_threads_on_any_existing_parent(): + # RFC 5322 section 3.6.4: In-Reply-To is 1*msg-id, so it may carry more than + # one parent id (a reply that joins two messages). Threading must consider + # every parent, not treat the whole header as one opaque id -- otherwise a + # multi-id In-Reply-To never matches an existing thread and the reply splits + # off on its own. + session = _SequentialSession([[("", "thread-xyz")]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": " ", + "references": None, + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "thread-xyz" + + +@pytest.mark.asyncio +async def test_multi_id_in_reply_to_fallback_uses_first_parent_as_root(): + session = _SequentialSession([[]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": " ", + "references": None, + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "first@example.com" + + +@pytest.mark.asyncio +async def test_in_reply_to_with_cfws_comment_extracts_bare_msg_id(): + # RFC 5322 sections 3.6.4 / 3.2.2 permit CFWS (e.g. a trailing comment) around + # a msg-id. The comment text must not leak into the id, or the reply is + # threaded/deduped against a garbage id and splits from its parent thread. + session = _SequentialSession([[("", "thread-123")]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": " (sent from my phone)", + "references": None, + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "thread-123" From db600e39db5a86bd9d75a718e2fc9147360f7aed Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 04:43:07 +0000 Subject: [PATCH 04/13] test(email): raise threading/parser coverage on the RFC-5322 ingest path Follow-up coverage for the email-ingest fixes on this branch. Adds tests for previously-unexercised paths (measured with coverage --branch on the two modules, --noconftest to skip the openai-dependent app import): threading_service.py 89% -> 93% - generate_email_fingerprint: was entirely untested. Now covers determinism, SHA-256 hex shape, lower-case + outer-whitespace collapse, None-component tolerance, and per-field sensitivity. - assign_thread_id uuid4 root: an email with no in_reply_to / references / message_id now asserts a fresh 32-hex uuid root with no DB lookup. - extract_reference_ids: a bracketed but whitespace-only id ("< >") is dropped rather than carried as an empty candidate. - assign_thread_id later-candidate: the lookup loop skips an unmatched first candidate (unimported immediate parent) and returns a matched older reference's thread instead of the deterministic root. email_parser.py 88% -> 90% - _extract_thread_id: a whitespace-only References/In-Reply-To header (splits to no token) now asserts fall-through to the next source (In-Reply-To, then Message-ID) rather than a blank thread id (branches 155->158 and 161->164). Test-only; no production code changed. 33 passed; ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/tests/test_email_parser.py | 21 +++++++ backend/tests/test_threading_service.py | 76 +++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index 6e8980562..1a5d2d4e1 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -408,6 +408,27 @@ def test_extract_thread_id_uses_first_reference_from_long_header(): assert _extract_thread_id(msg, "") == "" +def test_extract_thread_id_falls_through_whitespace_only_headers(): + # A References/In-Reply-To header that unfolds to only whitespace is present + # but yields no token when split; _extract_thread_id must fall through to the + # next source rather than return a blank thread id. + fell_to_in_reply_to = Message() + fell_to_in_reply_to["References"] = " " + fell_to_in_reply_to["In-Reply-To"] = "" + assert ( + _extract_thread_id(fell_to_in_reply_to, "") + == "" + ) + + fell_to_message_id = Message() + fell_to_message_id["References"] = " " + fell_to_message_id["In-Reply-To"] = " \t " + assert ( + _extract_thread_id(fell_to_message_id, "") + == "" + ) + + def test_parse_eml_extracts_reply_to_header(): eml_content = b"""Message-ID: From: Sender Name diff --git a/backend/tests/test_threading_service.py b/backend/tests/test_threading_service.py index b66043ccd..ed63f7b91 100644 --- a/backend/tests/test_threading_service.py +++ b/backend/tests/test_threading_service.py @@ -3,6 +3,7 @@ from services.threading_service import ( assign_thread_id, extract_reference_ids, + generate_email_fingerprint, normalize_message_id, ) @@ -183,6 +184,81 @@ def test_extract_reference_ids_normalizes_folded_whitespace_and_dedupes(): assert extract_reference_ids(header) == ["a@x.com", "b@x.com"] +def test_extract_reference_ids_drops_bracketed_whitespace_only_ids(): + # "< >" / "<\t>" are bracketed but whitespace-only: they normalize to nothing + # and must be dropped, not carried as empty thread candidates. + assert extract_reference_ids("< > <\t>") == ["a@x.com"] + + +@pytest.mark.asyncio +async def test_assign_thread_id_uses_a_later_candidate_when_the_first_has_no_thread(): + # The immediate parent (in_reply_to) is not yet imported, but an older + # reference is: the lookup loop must skip the unmatched first candidate and + # return the matched later one, not fall through to the deterministic root. + session = _SequentialSession([[("", "thread-older")]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": "", + "references": "", + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "thread-older" + + +def test_generate_email_fingerprint_is_deterministic_case_insensitive_and_field_sensitive(): + baseline = generate_email_fingerprint( + "Quarterly plan", "Mon, 01 Jun 2026 09:00:00 +0000", "a@x.com", "b@y.com" + ) + # 1. deterministic + SHA-256 hex digest + assert baseline == generate_email_fingerprint( + "Quarterly plan", "Mon, 01 Jun 2026 09:00:00 +0000", "a@x.com", "b@y.com" + ) + assert len(baseline) == 64 + assert all(character in "0123456789abcdef" for character in baseline) + # 2. lower-cased + outer-whitespace-stripped components collapse to one key + assert ( + generate_email_fingerprint( + " QUARTERLY PLAN ", "Mon, 01 Jun 2026 09:00:00 +0000", "A@X.com", " b@Y.com " + ) + == baseline + ) + # 3. None components are treated as empty (no crash) and stay distinct + all_empty = generate_email_fingerprint(None, None, None, None) + assert len(all_empty) == 64 + assert all_empty != baseline + # 4. any changed component changes the fingerprint (no field is dropped) + assert ( + generate_email_fingerprint( + "Quarterly plan", "Mon, 01 Jun 2026 09:00:00 +0000", "a@x.com", "c@z.com" + ) + != baseline + ) + + +@pytest.mark.asyncio +async def test_assign_thread_id_generates_fresh_uuid_when_no_identifiers_present(): + # An email with no in_reply_to, no references, and no message_id has nothing + # to thread on, so a fresh uuid4 root is minted and no lookup is issued. + session = _SequentialSession([]) + + thread_id = await assign_thread_id( + session, + {"message_id": None, "in_reply_to": None, "references": None}, + user_id="testuser", + organization_id="org-acme", + ) + + assert len(thread_id) == 32 + assert all(character in "0123456789abcdef" for character in thread_id) + assert session.execute_count == 0 + + @pytest.mark.asyncio async def test_multi_id_in_reply_to_threads_on_any_existing_parent(): # RFC 5322 section 3.6.4: In-Reply-To is 1*msg-id, so it may carry more than From 09211773969206317be7c5788088a2d60e44f32e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 04:53:45 +0000 Subject: [PATCH 05/13] test(email): cover parser edge/error paths (email_parser 90% -> 98%) Continues securing coverage on the RFC-5322 ingest path with additive, production-untouched tests for the remaining uncovered lines (measured with coverage --branch, --noconftest to skip the openai-dependent app): - _sanitize_address_display_text: a token with a display name but empty address part keeps the name (49-50); a header yielding no address at all falls back to sanitized raw text (53). - _attachment_part_content: a part whose get_content() cannot decode (unknown charset / bad transfer-encoding) falls back to the raw decoded payload, and to "" when the payload is absent (99-101). - parse_eml_bytes: parses provider bytes (happy path) and wraps an internal parse failure as the sanitized public EmailParseError rather than leaking the exception chain at the ingest boundary (217-222). email_parser.py 90% -> 98% (0 statements missed; the 3 remaining branch partials are defensive isinstance(str) guards for non-str part content). Test-only; no production change. 19 passed; ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/tests/test_email_parser.py | 54 ++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index 1a5d2d4e1..935db8c41 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -2,10 +2,17 @@ import os import tempfile from email.message import Message -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest -from services.email_parser import _extract_thread_id, _sanitize_nul, parse_eml +from services.email_parser import ( + _attachment_part_content, + _extract_thread_id, + _sanitize_address_display_text, + _sanitize_nul, + parse_eml, + parse_eml_bytes, +) from services.exceptions import EmailParseError @@ -408,6 +415,49 @@ def test_extract_thread_id_uses_first_reference_from_long_header(): assert _extract_thread_id(msg, "") == "" +def test_sanitize_address_display_text_keeps_name_only_and_falls_back_to_text(): + # A token with a display name but an empty address part keeps the name + # (rather than dropping it), and a header that yields no address at all + # falls back to the sanitized raw text. + assert _sanitize_address_display_text("Display Name <>") == "Display Name" + assert _sanitize_address_display_text("") == "" + + +def test_attachment_part_content_falls_back_to_raw_payload_on_decode_error(): + # A part whose get_content() cannot decode (unknown charset / malformed + # transfer-encoding) falls back to the raw decoded payload, and to "" when + # the payload is absent, instead of propagating the decode error. + raw_part = MagicMock() + raw_part.get_content.side_effect = LookupError("unknown charset") + raw_part.get_payload.return_value = b"raw-bytes" + assert _attachment_part_content(raw_part) == b"raw-bytes" + + empty_part = MagicMock() + empty_part.get_content.side_effect = ValueError("bad encoding") + empty_part.get_payload.return_value = None + assert _attachment_part_content(empty_part) == "" + + +def test_parse_eml_bytes_parses_provider_bytes_and_wraps_parse_errors(): + parsed = parse_eml_bytes( + b"Message-ID: \r\n" + b"From: sender@test.com\r\n" + b"To: user@test.com\r\n" + b"Subject: Bytes\r\n\r\n" + b"Body" + ) + assert parsed["message_id"] == "" + assert parsed["subject"] == "Bytes" + + # A parser failure is wrapped as the sanitized public EmailParseError rather + # than leaking the internal exception chain at the ingest boundary. + with patch( + "services.email_parser.message_from_bytes", side_effect=ValueError("boom") + ): + with pytest.raises(EmailParseError): + parse_eml_bytes(b"anything") + + def test_extract_thread_id_falls_through_whitespace_only_headers(): # A References/In-Reply-To header that unfolds to only whitespace is present # but yields no token when split; _extract_thread_id must fall through to the From feddde03539304879f23f7dd2e576a6a669c8003 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 04:56:14 +0000 Subject: [PATCH 06/13] test(threading): reach 100% coverage on threading_service Closes the last gaps in the module the In-Reply-To / Message-ID fixes on this branch touched (measured with coverage --branch, --noconftest): - extract_reference_ids: a References value with no angle brackets falls back to a whitespace split (line 65). - _find_existing_thread_ids: returns {} for no candidates without issuing a query (86); dedupes a bare id against its already-bracketed form so the shared lookup key is enqueued once (92->91); skips a stored row with a null thread_id (106) and a row whose message_id normalizes to nothing (108->104), so neither pollutes the returned map. services/threading_service.py 93% -> 100% (0 missed, 0 branch partials); services/email_parser.py stands at 98% after the prior commit. Test-only; no production change. 21 passed; ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/tests/test_threading_service.py | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/backend/tests/test_threading_service.py b/backend/tests/test_threading_service.py index ed63f7b91..2372ceaf1 100644 --- a/backend/tests/test_threading_service.py +++ b/backend/tests/test_threading_service.py @@ -1,6 +1,7 @@ import pytest from services.threading_service import ( + _find_existing_thread_ids, assign_thread_id, extract_reference_ids, generate_email_fingerprint, @@ -190,6 +191,58 @@ def test_extract_reference_ids_drops_bracketed_whitespace_only_ids(): assert extract_reference_ids("< > <\t>") == ["a@x.com"] +def test_extract_reference_ids_falls_back_to_whitespace_split_without_brackets(): + # A References value with no angle brackets (some non-conforming clients) + # falls back to a whitespace split rather than yielding nothing. + assert extract_reference_ids("a@x.com b@x.com") == ["a@x.com", "b@x.com"] + + +@pytest.mark.asyncio +async def test_find_existing_thread_ids_returns_empty_without_candidates(): + session = _SequentialSession([]) + result = await _find_existing_thread_ids( + session, [], user_id="testuser", organization_id="org-acme" + ) + assert result == {} + assert session.execute_count == 0 + + +@pytest.mark.asyncio +async def test_find_existing_thread_ids_dedupes_overlapping_bracket_targets(): + # A bare id and its already-bracketed form collapse to one target set, so the + # shared "" lookup key is not enqueued twice. + session = _QueryCapturingSession([[("", "thread-a")]]) + result = await _find_existing_thread_ids( + session, + ["", "a@x.com"], + user_id="testuser", + organization_id="org-acme", + ) + assert result == {"a@x.com": "thread-a"} + + +@pytest.mark.asyncio +async def test_find_existing_thread_ids_skips_rows_with_blank_thread_or_message_id(): + # A stored row with no thread_id is skipped, and a row whose message_id + # normalizes to nothing is skipped; neither pollutes the returned map. + session = _SequentialSession( + [ + [ + ("", None), + ("", "thread-b"), + ("", "thread-c"), + ] + ] + ) + result = await _find_existing_thread_ids( + session, + ["a@x.com", "c@x.com"], + user_id="testuser", + organization_id="org-acme", + ) + assert result == {"c@x.com": "thread-c"} + + @pytest.mark.asyncio async def test_assign_thread_id_uses_a_later_candidate_when_the_first_has_no_thread(): # The immediate parent (in_reply_to) is not yet imported, but an older From 5cea7ba3a83217e7665f5ae2f72688689e070af2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 05:08:48 +0000 Subject: [PATCH 07/13] docs(threading): add RFC 5322 standards-basis pack for email-ingest work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ground the #1192 email-ingest correctness changes (Message-ID interior-whitespace normalization, unknown-zone -0000 date normalization, In-Reply-To 1*msg-id multi-parent + CFWS parsing) in the message-format standard, anchoring each fix to a specific RFC 5322 clause (§2.2.3 header unfolding, §3.6.4 msg-id, §3.2.2 CFWS, §3.3 unknown zone). Also record the precision-first threading design rationale with verified citations (Kooti et al., 2015; Mohiuddin, Joty, & Nguyen, 2018; Zhang et al., 2021), following the existing docs/research// convention. RFC text and PDFs are referenced by canonical URL and bookmarked in the alphaXiv library; they are not committed because the sandbox proxy blocks rfc-editor.org and arxiv.org (disclosed in the pack's Preservation notes). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- .../research/email-ingest-threading/README.md | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/research/email-ingest-threading/README.md diff --git a/docs/research/email-ingest-threading/README.md b/docs/research/email-ingest-threading/README.md new file mode 100644 index 000000000..436f04e0f --- /dev/null +++ b/docs/research/email-ingest-threading/README.md @@ -0,0 +1,92 @@ +# Email Ingest & Threading — Standards Basis and Design Rationale + +This pack grounds the RFC 5322 email-ingest correctness work on +`ContextualWisdomLab/naruon#1192` +(`backend/services/threading_service.py`, `backend/services/email_parser.py`): +Message-ID interior-whitespace normalization, unknown-zone (`-0000`) date +normalization, and In-Reply-To `1*msg-id` (multi-parent + CFWS) parsing. + +## Standards basis (RFC 5322, Internet Message Format) + +Each fix is anchored to a specific clause of the message-format standard: + +- **Header unfolding** — RFC 5322 §2.2.3. When a folded header is rejoined, + interior whitespace/tabs can survive. `normalize_message_id` collapses that + interior whitespace so the folded and unfolded forms of one Message-ID map to + a single de-dup/threading key. +- **Message-ID** — RFC 5322 §3.6.4 (`msg-id = "<" id-left "@" id-right ">"`). + A well-formed Message-ID carries no interior whitespace, so collapsing it is a + no-op for conforming input and only repairs unfolded input. +- **In-Reply-To / References** — RFC 5322 §3.6.4 defines both as `1*msg-id` + (one or more angle-bracketed ids, each optionally wrapped in CFWS, §3.2.2). + `assign_thread_id` therefore parses In-Reply-To with the same multi-id + extractor used for References, so a reply naming several parents — or a single + id trailed by a comment — threads onto an existing ancestor instead of + splitting off on a garbage key. +- **Date / unknown zone** — RFC 5322 §3.3 defines a `-0000` zone as "time zone + unknown". `email.utils.parsedate_to_datetime` returns a naive datetime for + that case; `_extract_date` treats the unknown zone as UTC so every ingested + date is timezone-aware and safe to sort and to store in a `timestamptz` + column. + +## Design rationale — header-based, precision-first threading + +Naruon reconstructs conversations from the RFC 5322 reference graph +(References / In-Reply-To), deterministically, and deliberately does **not** +fall back to subject- or content-based grouping. This is a precision/recall +trade-off, not an oversight, and the rejected alternative is grounded in the +conversation-threading literature: + +- Content- and coherence-model approaches to thread reconstruction are + probabilistic and improve *recall* on broken reference chains, but carry an + inherent false-merge (precision) cost: Mohiuddin, Joty, and Nguyen (2018) + reconstruct thread trees by scoring candidate structures with a neural + coherence model, and even their best model reaches only ~30% thread-level + reconstruction accuracy — useful for recall on missing links, but far from the + certainty a mailbox view requires. +- Email threads are also large and topically heterogeneous in practice (Kooti, + Aiello, Grbovic, Lerman, & Mantrach, 2015, characterize replying behavior + over 16 billion messages; Zhang, Celikyilmaz, Gao, & Bansal, 2021, curate + 2,549 real email threads for EmailSum), so a wrong content-based merge is both + likely and costly at scale. +- Naruon therefore optimizes for *precision* — never merging unrelated messages — + because a wrong merge silently corrupts a user's mailbox view. The invariant is + pinned by `test_forwarded_subject_alone_does_not_merge_unrelated_thread`. +- The #1192 In-Reply-To fix improves *recall on the header-complete path* (it + no longer drops multi-parent / CFWS replies) with **zero** precision cost, + because it stays entirely within the deterministic header graph. + +The cited papers are bookmarked in the shared alphaXiv library folder +"CWL · Naruon email/threading standards grounding". + +## References (APA 7) + +- Resnick, P. (Ed.). (2008). *Internet message format* (RFC 5322). RFC Editor. + https://www.rfc-editor.org/rfc/rfc5322.txt +- Kooti, F., Aiello, L. M., Grbovic, M., Lerman, K., & Mantrach, A. (2015). + *Evolution of conversations in the age of email overload* [Preprint]. arXiv. + https://arxiv.org/abs/1504.00704 +- Mohiuddin, T., Joty, S., & Nguyen, D. T. (2018). *Coherence modeling of + asynchronous conversations: A neural entity grid approach* [Preprint]. arXiv. + https://arxiv.org/abs/1805.02275 +- Zhang, S., Celikyilmaz, A., Gao, J., & Bansal, M. (2021). *EmailSum: + Abstractive email thread summarization* [Preprint]. arXiv. + https://arxiv.org/abs/2107.14691 + +## Preservation notes + +- Original standard text and paper PDFs are referenced by their canonical + RFC Editor / arXiv URLs above and are bookmarked in the alphaXiv library + folder named in the design-rationale section. They are not committed here + because this sandbox's outbound proxy blocks `rfc-editor.org` and `arxiv.org`; + when a network-enabled run can fetch them, drop the RFC text into + `standards/` and the PDFs into `pdfs/` following the sibling packs' layout. +- Git LFS is intentionally not used, consistent with the other + `docs/research/*` packs. + +## Governance notes + +- Work item: `ContextualWisdomLab/naruon#1192` (RFC 5322 email-ingest + correctness + coverage). +- Verification: threading/parser suites pass with `--noconftest`; + `threading_service.py` at 100% and `email_parser.py` at 98% branch coverage. From 74d138806cf58783abc3902887521227b5457f18 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 06:37:52 +0000 Subject: [PATCH 08/13] fix(email): store non-ASCII sender/recipient names decoded, not re-encoded _sanitize_address_display_text formatted each parsed address with email.utils.formataddr. The From/To/Reply-To headers arrive already header-decoded (policy.default decodes RFC 2047 encoded-words), but formataddr re-encodes any non-ASCII display name back into an =?utf-8?...?= encoded-word, so every non-ASCII sender/recipient/reply-to name (e.g. Korean) was stored and displayed as garbled machine bytes rather than readable text. The subject path was unaffected because it does not round-trip through formataddr. Replace formataddr with _format_display_address, which mirrors formataddr's RFC 5322 quoting/escaping for display-name specials (so ", "-joined multi-address values stay unambiguous and embedded quotes are escaped) but keeps the decoded name literal -- these values are stored for human display, not re-emitted as message headers. Tests: end-to-end parse of RFC 2047-encoded Korean From/To/Reply-To/Subject storing decoded text; unit coverage of the helper's quoting, escaping, empty name, and non-ASCII literal paths. email_parser branch coverage stays at 98% (new helper fully covered); ruff clean; 22 passed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/services/email_parser.py | 31 +++++++++++- backend/tests/test_email_parser.py | 75 ++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/backend/services/email_parser.py b/backend/services/email_parser.py index b57f29d29..be8bee1c4 100644 --- a/backend/services/email_parser.py +++ b/backend/services/email_parser.py @@ -2,7 +2,8 @@ from email.message import Message from pathlib import Path import datetime -from email.utils import formataddr, getaddresses +import re +from email.utils import getaddresses from email.utils import parsedate_to_datetime from typing import NotRequired, TypedDict from .attachment_parser import parse_email_attachment @@ -39,13 +40,39 @@ def _sanitize_display_text(text: str) -> str: return strip_html_markup(_sanitize_nul(text)) +# Mirror email.utils.formataddr's RFC 5322 display-name quoting: the specials +# that force a quoted-string, and the characters escaped inside one. +_ADDRESS_SPECIALS_RE = re.compile(r'[()<>@,;:\\".\[\]]') +_ADDRESS_QUOTED_ESCAPE_RE = re.compile(r'["\\]') + + +def _format_display_address(display_name: str, address: str) -> str: + """Formats an already-decoded display name and address for storage. + + Mirrors ``email.utils.formataddr`` quoting for RFC 5322 special characters + but keeps ``display_name`` literal instead of re-encoding a non-ASCII name + as an RFC 2047 encoded-word. The ``From``/``To``/``Reply-To`` headers arrive + already header-decoded (``policy.default``), and these values are stored for + human display, not re-emitted as message headers, so ``formataddr`` would + corrupt a decoded name (e.g. Korean) back into ``=?utf-8?b?...?=``. + """ + if not display_name: + return address + if _ADDRESS_SPECIALS_RE.search(display_name): + escaped_name = _ADDRESS_QUOTED_ESCAPE_RE.sub(r"\\\g<0>", display_name) + return f'"{escaped_name}" <{address}>' + return f"{display_name} <{address}>" + + def _sanitize_address_display_text(text: str) -> str: sanitized_parts: list[str] = [] for display_name, address in getaddresses([text]): safe_display_name = _sanitize_display_text(display_name).strip() safe_address = _sanitize_nul(address).strip() if safe_address: - sanitized_parts.append(formataddr((safe_display_name, safe_address))) + sanitized_parts.append( + _format_display_address(safe_display_name, safe_address) + ) elif safe_display_name: sanitized_parts.append(safe_display_name) if sanitized_parts: diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index 935db8c41..736680f5d 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -1,3 +1,4 @@ +import base64 import datetime import os import tempfile @@ -8,6 +9,7 @@ from services.email_parser import ( _attachment_part_content, _extract_thread_id, + _format_display_address, _sanitize_address_display_text, _sanitize_nul, parse_eml, @@ -116,6 +118,79 @@ def test_parse_eml_strips_active_html_from_address_display_fields(): os.unlink(temp_path) +def test_parse_eml_stores_non_ascii_display_names_decoded(): + # RFC 2047: non-ASCII From/To/Reply-To display names arrive as encoded-words + # (e.g. =?UTF-8?B?...?=). policy.default header-decodes them; the stored + # display fields must keep the decoded text rather than re-encoding it back + # into an encoded-word (formataddr's behavior), which would render every + # non-ASCII sender/recipient as garbled =?utf-8?...?= bytes in the UI. + from_name = "박성호" + to_name = "김천" + reply_name = "응답" + subject_text = "회 테스트" + + def encoded_word(text: str) -> bytes: + token = base64.b64encode(text.encode("utf-8")).decode("ascii") + return f"=?UTF-8?B?{token}?=".encode("ascii") + + eml_content = ( + b"Message-ID: \r\n" + b"From: " + encoded_word(from_name) + b" \r\n" + b"To: " + encoded_word(to_name) + b" \r\n" + b"Reply-To: " + encoded_word(reply_name) + b" \r\n" + b"Subject: " + encoded_word(subject_text) + b"\r\n" + b"Date: Mon, 27 Apr 2026 10:00:00 +0000\r\n" + b"\r\n" + b"Plain body" + ) + + with tempfile.NamedTemporaryFile(delete=False, suffix=".eml") as f: + f.write(eml_content) + temp_path = f.name + + try: + parsed = parse_eml(temp_path) + assert parsed["sender"] == f"{from_name} " + assert parsed["recipients"] == f"{to_name} " + assert parsed["reply_to"] == f"{reply_name} " + assert parsed["subject"] == subject_text + assert "=?" not in parsed["sender"] + assert "=?" not in parsed["recipients"] + finally: + os.unlink(temp_path) + + +def test_sanitize_address_display_text_keeps_decoded_unicode_and_quotes_specials(): + # A decoded non-ASCII name stays literal (formataddr would re-encode it). + assert ( + _sanitize_address_display_text("박성호 ") + == "박성호 " + ) + # A display name containing an RFC 5322 special is quoted so a ", "-joined + # multi-address value stays unambiguous. + assert ( + _sanitize_address_display_text('"Doe, John" ') + == '"Doe, John" ' + ) + # Multiple addresses with mixed scripts are each formatted and comma-joined. + assert ( + _sanitize_address_display_text("박성호 , Bob ") + == "박성호 , Bob " + ) + + +def test_format_display_address_escapes_quotes_and_handles_empty_name(): + # No display name -> bare address. + assert _format_display_address("", "a@x.com") == "a@x.com" + # Non-ASCII name kept literal. + assert _format_display_address("박성호", "s@x.com") == "박성호 " + # Embedded quotes/backslashes are escaped inside the quoted-string, matching + # email.utils.formataddr's escaping. + assert ( + _format_display_address('Fancy "Q"', "q@x.com") == '"Fancy \\"Q\\"" ' + ) + + def test_parse_eml_strips_active_html_from_attachment_display_fields(): eml_content = b"""Message-ID: From: sender@test.com From 7e5406d453949621c9cc986f85f9ab44857a02b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 06:41:25 +0000 Subject: [PATCH 09/13] docs(threading): ground the RFC 2047 display-name fix in the standards pack Extend the email-ingest standards-basis pack to cover the RFC 2047 encoded-word decoding fix now on #1192: display names in From/To/Reply-To arrive header-decoded under email.policy.default and must be stored decoded, not re-encoded by email.utils.formataddr (which garbled every non-ASCII sender into =?utf-8?...?=). Adds the RFC 2047 clause to the standards basis, an APA-7 reference (Moore, 1996), and updates the work-item/verification notes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- .../research/email-ingest-threading/README.md | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/research/email-ingest-threading/README.md b/docs/research/email-ingest-threading/README.md index 436f04e0f..96bf77fbd 100644 --- a/docs/research/email-ingest-threading/README.md +++ b/docs/research/email-ingest-threading/README.md @@ -1,14 +1,15 @@ # Email Ingest & Threading — Standards Basis and Design Rationale -This pack grounds the RFC 5322 email-ingest correctness work on +This pack grounds the email-ingest correctness work on `ContextualWisdomLab/naruon#1192` (`backend/services/threading_service.py`, `backend/services/email_parser.py`): Message-ID interior-whitespace normalization, unknown-zone (`-0000`) date -normalization, and In-Reply-To `1*msg-id` (multi-parent + CFWS) parsing. +normalization, In-Reply-To `1*msg-id` (multi-parent + CFWS) parsing, and +RFC 2047 encoded-word decoding of non-ASCII display names. -## Standards basis (RFC 5322, Internet Message Format) +## Standards basis (RFC 5322 message format + RFC 2047 header encoding) -Each fix is anchored to a specific clause of the message-format standard: +Each fix is anchored to a specific clause of the relevant standard: - **Header unfolding** — RFC 5322 §2.2.3. When a folded header is rejoined, interior whitespace/tabs can survive. `normalize_message_id` collapses that @@ -28,6 +29,15 @@ Each fix is anchored to a specific clause of the message-format standard: that case; `_extract_date` treats the unknown zone as UTC so every ingested date is timezone-aware and safe to sort and to store in a `timestamptz` column. +- **Non-ASCII display names** — RFC 2047 (MIME Part Three) defines the + `=?charset?enc?text?=` encoded-word so non-ASCII text can appear in structured + headers. `From` / `To` / `Reply-To` display names arrive header-decoded under + `email.policy.default`, so `_sanitize_address_display_text` must store the + decoded name and must **not** re-encode it. `email.utils.formataddr` re-encodes + any non-ASCII display name back into an encoded-word, which stored a garbled + `=?utf-8?...?=` value for every non-ASCII (e.g. Korean) sender/recipient; + `_format_display_address` keeps the decoded name literal while preserving + formataddr's RFC 5322 quoting/escaping for display-name specials. ## Design rationale — header-based, precision-first threading @@ -63,6 +73,9 @@ The cited papers are bookmarked in the shared alphaXiv library folder - Resnick, P. (Ed.). (2008). *Internet message format* (RFC 5322). RFC Editor. https://www.rfc-editor.org/rfc/rfc5322.txt +- Moore, K. (1996). *MIME (Multipurpose Internet Mail Extensions) part three: + Message header extensions for non-ASCII text* (RFC 2047). RFC Editor. + https://www.rfc-editor.org/rfc/rfc2047.txt - Kooti, F., Aiello, L. M., Grbovic, M., Lerman, K., & Mantrach, A. (2015). *Evolution of conversations in the age of email overload* [Preprint]. arXiv. https://arxiv.org/abs/1504.00704 @@ -86,7 +99,8 @@ The cited papers are bookmarked in the shared alphaXiv library folder ## Governance notes -- Work item: `ContextualWisdomLab/naruon#1192` (RFC 5322 email-ingest +- Work item: `ContextualWisdomLab/naruon#1192` (RFC 5322 / RFC 2047 email-ingest correctness + coverage). - Verification: threading/parser suites pass with `--noconftest`; - `threading_service.py` at 100% and `email_parser.py` at 98% branch coverage. + `threading_service.py` at 100% and `email_parser.py` at 98% branch coverage + (the RFC 2047 `_format_display_address` helper is fully covered). From f642f3580aecc5fd2dfa4e8d51e3be4e97faab9d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 06:48:27 +0000 Subject: [PATCH 10/13] test(email): cover non-string part-content branches (email_parser 98% -> 100%) Add unit tests for the isinstance(part_content, str) guards in _process_multipart_body (text/plain and text/html parts) and _process_singlepart_body, exercising the case where part.get_content() returns a non-str (e.g. undecodable bytes) so the body-extraction path drops it instead of concatenating bytes. Brings email_parser.py to 100% branch coverage (the org 100% standard), completing the RFC 2047 display-name change's coverage. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/tests/test_email_parser.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index 736680f5d..397744a53 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -10,6 +10,8 @@ _attachment_part_content, _extract_thread_id, _format_display_address, + _process_multipart_body, + _process_singlepart_body, _sanitize_address_display_text, _sanitize_nul, parse_eml, @@ -191,6 +193,34 @@ def test_format_display_address_escapes_quotes_and_handles_empty_name(): ) +def test_process_multipart_body_ignores_non_string_part_content(): + # get_content() can return a non-str (e.g. undecodable bytes) even for a + # text/* part; the isinstance guard must drop it rather than concatenate + # bytes into the plain/html body. + plain_part = MagicMock() + plain_part.get_content_type.return_value = "text/plain" + plain_part.get_filename.return_value = None + plain_part.get_content.return_value = b"not-a-str" + html_part = MagicMock() + html_part.get_content_type.return_value = "text/html" + html_part.get_filename.return_value = None + html_part.get_content.return_value = b"not-a-str" + msg = MagicMock() + msg.walk.return_value = [plain_part, html_part] + + assert _process_multipart_body(msg) == ("", "", []) + + +def test_process_singlepart_body_ignores_non_string_content(): + # A single-part message whose get_content() returns a non-str yields an + # empty body rather than a stringified bytes value. + msg = MagicMock() + msg.get_content_type.return_value = "text/plain" + msg.get_content.return_value = b"not-a-str" + + assert _process_singlepart_body(msg) == ("", "", []) + + def test_parse_eml_strips_active_html_from_attachment_display_fields(): eml_content = b"""Message-ID: From: sender@test.com From 5db7cacd868d8656dc7a5bbb155643506b88eb85 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 07:03:10 +0000 Subject: [PATCH 11/13] test(email): import EmailParseError from the module under test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate the test's EmailParseError import into the services.email_parser import block (email_parser re-exports it from services.exceptions), removing the separate services.exceptions import. No behavior change — EmailParseError was already imported and the suite passed; this addresses a review note preferring the module-under-test as the single import source. Ruff clean; 24 passed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016wtuYFp4E22QnEU1bFMhsr --- backend/tests/test_email_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index 397744a53..da098bc34 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -7,6 +7,7 @@ import pytest from services.email_parser import ( + EmailParseError, _attachment_part_content, _extract_thread_id, _format_display_address, @@ -17,7 +18,6 @@ parse_eml, parse_eml_bytes, ) -from services.exceptions import EmailParseError def test_parse_eml_basic(): From bd79d7586d864846031f2bf7813a112f85e614bf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 3 Aug 2026 09:56:05 +0900 Subject: [PATCH 12/13] chore(ci): refresh required checks for PR #1192 --- .github/.ci-refresh-1192 | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/.ci-refresh-1192 diff --git a/.github/.ci-refresh-1192 b/.github/.ci-refresh-1192 new file mode 100644 index 000000000..ce6dce31e --- /dev/null +++ b/.github/.ci-refresh-1192 @@ -0,0 +1 @@ +refresh required checks for PR #1192 From cfb2b6b771eb59913075e6bf724c2356e8938e9b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 3 Aug 2026 09:56:16 +0900 Subject: [PATCH 13/13] chore(ci): remove refresh marker for PR #1192 --- .github/.ci-refresh-1192 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .github/.ci-refresh-1192 diff --git a/.github/.ci-refresh-1192 b/.github/.ci-refresh-1192 deleted file mode 100644 index ce6dce31e..000000000 --- a/.github/.ci-refresh-1192 +++ /dev/null @@ -1 +0,0 @@ -refresh required checks for PR #1192