diff --git a/polylogue/api/archive.py b/polylogue/api/archive.py index 9582b2c7eb..0f3197f224 100644 --- a/polylogue/api/archive.py +++ b/polylogue/api/archive.py @@ -5093,7 +5093,7 @@ async def get_session_events( ``world_state``/``agent_policy``/``turn_context`` policy facts (sandbox/truncation policy, rate limits, ghost commits, developer instructions), every Claude Code sidecar event (``claude_todo_state``, - ``claude_tool_result_sidecar``, ``claude_attachment``, ``micro_compaction``, + ``claude_tool_result_sidecar``, ``claude_attachment_file``, ``micro_compaction``, etc.), Hermes ``hermes_tool_availability_span``/``hermes_*`` step telemetry, and ``claude_ai_conversation_summary`` all surface through the same one reader instead of a bespoke accessor per event type. diff --git a/polylogue/sources/origin_specs.py b/polylogue/sources/origin_specs.py index e74c07caa1..c6ee7df18c 100644 --- a/polylogue/sources/origin_specs.py +++ b/polylogue/sources/origin_specs.py @@ -434,7 +434,7 @@ def _claude_code_spec() -> OriginSpec: "records -- there is no child-side wire evidence to read. tool_result outcome_unknown_reason is " "NOT_REPORTED when the Anthropic-protocol segment carries no is_error, and DISTRUSTED for the " "background-task start acknowledgement's is_error=false (see _mark_background_task_start).", - "code_parser.py's _SKIPPED_SIDECAR_RECORD_TYPES (14 sidecar record " + "code_parser.py's _NON_MESSAGE_SIDECAR_RECORD_TYPES (14 sidecar record " "types) already carries a per-type disposition with corpus counts " "in a comment block (polylogue-pbuh/parser-diff triage, " "2026-07-29) -- not converted to a DroppedValueVocabulary " diff --git a/polylogue/sources/parsers/claude/code_parser.py b/polylogue/sources/parsers/claude/code_parser.py index 9c5fcd233d..f4068e64d9 100644 --- a/polylogue/sources/parsers/claude/code_parser.py +++ b/polylogue/sources/parsers/claude/code_parser.py @@ -3,7 +3,7 @@ from __future__ import annotations import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, TypeAlias @@ -103,17 +103,25 @@ def _clean_title_text(text: str) -> str: logger = get_logger(__name__) -# ``_SKIPPED_SIDECAR_RECORD_TYPES`` marks record types that never become a +# ``_NON_MESSAGE_SIDECAR_RECORD_TYPES`` marks record types that never become a # ``ParsedMessage`` row (they are not chat content). That is still correct for -# all twelve types below. What changed (polylogue-pbuh, audited against the -# live corpus 2026-07-29, ~1.17M records total) is that most of them ARE -# evidence-bearing and are no longer silently discarded: they persist through -# ``_sidecar_evidence_payload``/the ``progress`` delegation accumulator below -# as typed ``session_events`` (index.db). ``session_events.event_type`` has no -# CHECK-constrained vocabulary (see ``storage/sqlite/archive_tiers/index.py``), -# so adding new event types here is additive data, not a schema change. +# all twelve types below -- the name used to be +# ``_SKIPPED_SIDECAR_RECORD_TYPES``, which stopped being accurate the day most +# of these started persisting as ``session_events`` (polylogue-pbuh); 13 of +# the 15 members below ARE persisted today, so "skipped" described the +# pre-polylogue-pbuh behavior, not the current one. Renamed (polylogue lane, +# audited against the live corpus 2026-07-31) with no compat alias -- this +# repo does not carry old spellings forward. What changed originally +# (polylogue-pbuh, audited 2026-07-29, ~1.17M records total) is that most of +# these ARE evidence-bearing and are no longer silently discarded: they +# persist through ``_sidecar_evidence_payload``/the ``progress`` delegation +# accumulator/the attachment-subtype dispatch below as typed ``session_events`` +# (index.db). ``session_events.event_type`` has no CHECK-constrained +# vocabulary (see ``storage/sqlite/archive_tiers/index.py``), so adding new +# event types here is additive data, not a schema change. # -# Per-type disposition (counts = live corpus, rg single pass, 2026-07-29): +# Per-type disposition (counts = live corpus, rg single pass, 2026-07-29 +# unless noted): # ai-title (18,561) EVIDENCE, wins session title (TitleSource.ORIGIN) # agent-name (5,001) EVIDENCE -> claude_agent_name event, ALSO wins # session title (TitleSource.ORIGIN, below @@ -129,10 +137,18 @@ def _clean_title_text(text: str) -> str: # bypassPermissions -- real operational signal) # last-prompt (37,799) EVIDENCE -> claude_last_prompt event # queue-operation (60,709) EVIDENCE -> claude_queue_operation event -# attachment (86,237) EVIDENCE -> claude_attachment event (20 distinct -# attachment.type payloads incl. real files, -# edited-file records, diagnostics; per-subtype -# fidelity split is a follow-up, not this pass) +# attachment (87,539 as of 2026-07-31 re-audit) EVIDENCE, subtype-dispatched -- +# see ``_ATTACHMENT_SUBTYPE_EVENT_TYPES`` +# below. The type used to collapse all 20+ +# ``attachment.type`` payloads into one +# ``claude_attachment`` event regardless of +# whether the record was a real referenced +# file or a hook success ping; a 2026-07-31 +# full-corpus enumeration found 38 distinct +# subtypes (corpus growth/CLI evolution +# since the 20-subtype estimate above), now +# routed to ~22 semantically grouped event +# types plus 4 confirmed-transient subtypes. # progress (850,678) MIXED, not uniformly evidence-bearing -- an # earlier draft of this bead's tracking issue # guessed the whole type was noise, then @@ -159,15 +175,28 @@ def _clean_title_text(text: str) -> str: # facts. Persisting them would 4-8x the # session_events row count for zero # incremental evidence. Kept transient. -# init (0 live occurrences) TRANSIENT: every corpus occurrence on record -# is a bare {"type": "init"} marker with no -# field beyond ``type``/``sessionId`` -- -# nothing to lose. -# mode (20,779) TRANSIENT: ``mode`` was the literal string -# "normal" in 100% of live records -- zero -# information content in the corpus as -# observed. Re-audit if a non-"normal" value -# is ever seen. +# OUT OF SCOPE for the 2026-07-31 lane that +# added the attachment-subtype dispatch and +# re-verified init/mode below -- this +# disposition is unchanged and was +# deliberately not revisited. +# init (0 live occurrences, re-confirmed 2026-07-31 against the full +# ~11.7K-session-file corpus) TRANSIENT: every corpus occurrence on +# record is a bare {"type": "init"} marker +# with no field beyond ``type``/ +# ``sessionId`` -- nothing to lose. Zero +# occurrences both times measured. +# mode (21,545 as of 2026-07-31 re-audit, up from 20,779 on 2026-07-29 -- +# corpus growth) TRANSIENT: ``mode`` was the literal string "normal" +# in 100% of live records both times +# measured (21,545/21,545 on the 2026-07-31 +# pass) -- zero information content in the +# corpus as observed, in contrast to the +# sibling ``permission-mode`` type above +# (kept: 5 distinct values observed). Kept +# transient on repeated evidence, not +# assumption; re-audit if a non-"normal" +# value is ever seen. # # Two more sidecar record types (parser-diff triage, 2026-07-29, ~600-file # sample of the live corpus) were not in the original twelve and fell through @@ -181,7 +210,7 @@ def _clean_title_text(text: str) -> str: # (per-file incremental backup, the # fine-grained sibling of the # whole-snapshot file-history-snapshot type) -_SKIPPED_SIDECAR_RECORD_TYPES = frozenset( +_NON_MESSAGE_SIDECAR_RECORD_TYPES = frozenset( { "init", "file-history-snapshot", @@ -202,10 +231,12 @@ def _clean_title_text(text: str) -> str: # record_type -> session_events.event_type for the sidecar types persisted # generically via ``_sidecar_evidence_payload``. ``progress``, ``ai-title``, -# and ``custom-title`` are handled separately in ``_parse_code_records`` -# (progress needs whole-session deduplication; ai-title/custom-title feed -# title resolution as well as an audit-trail event). ``init``/``mode`` map to -# nothing (transient, see above). +# ``custom-title``, and ``attachment`` are handled separately in +# ``_parse_code_records`` (progress needs whole-session deduplication; +# ai-title/custom-title feed title resolution as well as an audit-trail +# event; attachment needs subtype-dependent dispatch -- see +# ``_ATTACHMENT_SUBTYPE_EVENT_TYPES`` below). ``init``/``mode`` map to nothing +# (transient, see above). _SIDECAR_EVENT_TYPES: dict[str, str] = { "agent-name": "claude_agent_name", "pr-link": "claude_pr_link", @@ -215,18 +246,263 @@ def _clean_title_text(text: str) -> str: "permission-mode": "claude_permission_mode", "last-prompt": "claude_last_prompt", "queue-operation": "claude_queue_operation", - "attachment": "claude_attachment", "ai-title": "claude_ai_title", "custom-title": "claude_custom_title", } +# ``attachment.type`` subtype -> session_events.event_type (polylogue lane, +# 2026-07-31). Replaces the single collapsed ``claude_attachment`` bucket: a +# full-corpus enumeration (~11.7K Claude Code session files, +# ``~/.claude/projects``) found 38 distinct ``attachment.type`` payloads +# spanning genuinely different kinds of fact -- a real referenced file's +# content is not the same *kind of thing* as a hook success ping, and both +# were previously indistinguishable at the ``event_type`` level (payload had +# to be JSON-inspected to tell them apart). Grouping rule: subtypes that +# report the same real-world entity (a hook firing, a mode transition, a +# capability/tool-surface delta) share one event_type distinguished by +# payload fields, rather than one event_type per subtype -- that would have +# produced ~38 near-empty event types for what are structurally the same +# few kinds of fact. Real file/reference content gets its own dedicated +# event_type per subtype since each is a structurally distinct artifact +# (a full file vs. an edit snippet vs. a bare path reference). +# +# Counts below are live-corpus occurrences of the *subtype*, 2026-07-31, +# same corpus as the 87,539 total above: +_ATTACHMENT_SUBTYPE_EVENT_TYPES: dict[str, str] = { + # Real file/reference content -- each subtype is a structurally distinct + # artifact, so each gets its own event_type (no collapsing these into + # each other, let alone into the hook/capability buckets below). + "file": "claude_attachment_file", # 711: a real referenced file's full content + "edited_text_file": "claude_attachment_edited_file", # 2,446: editor-buffer snippet at edit time + "nested_memory": "claude_attachment_nested_memory", # 487: a nested CLAUDE.md/memory file's content + "plan_file_reference": "claude_attachment_plan_reference", # 43: a plan file's full content + "compact_file_reference": "claude_attachment_file_reference", # 460: a bare path reference, no content + "directory": "claude_attachment_directory_listing", # 16: a directory listing (path + entry names) + # Hook lifecycle -- six subtypes report the same real-world entity (a + # hook firing) distinguished only by outcome; one event_type with the + # subtype riding in payload["type"] (already present via dict(attachment)) + # avoids six near-identical event types for one entity. + "hook_success": "claude_hook_event", # 27,678 + "hook_non_blocking_error": "claude_hook_event", # 558 + "hook_blocking_error": "claude_hook_event", # 60 + "hook_cancelled": "claude_hook_event", # 303 + "hook_system_message": "claude_hook_event", # 129 + "hook_additional_context": "claude_hook_event", # 1,003 + # Agent-mode transitions (auto-mode / plan-mode enter-exit-reentry) -- one + # session-state-transition entity, five lifecycle phases. + "auto_mode": "claude_agent_mode_event", # 644 + "auto_mode_exit": "claude_agent_mode_event", # 130 + "plan_mode": "claude_agent_mode_event", # 84 + "plan_mode_exit": "claude_agent_mode_event", # 99 + "plan_mode_reentry": "claude_agent_mode_event", # 15 + # Capability/tool-surface deltas presented to the model mid-session -- + # bounded to names/counts, not the full injected instruction blocks + # (which can be large and duplicate what's already visible on the + # transcript), matching the existing bounded-summary precedent in + # ``_tool_execution_result_payload``. + "deferred_tools_delta": "claude_capability_delta", # 2,856 + "mcp_instructions_delta": "claude_capability_delta", # 1,050 + "agent_listing_delta": "claude_capability_delta", # 200 + "skill_listing": "claude_capability_snapshot", # 2,690 + "invoked_skills": "claude_capability_snapshot", # 117 + # Small, clean, high-signal single-field events -- real operational + # state, same tier of evidence as the existing ``permission-mode`` type. + "output_style": "claude_output_style", # 10,055 + "command_permissions": "claude_command_permissions", # 570 + # Queue activity -- semantically the same entity as the top-level + # ``queue-operation`` record type (see ``_SIDECAR_EVENT_TYPES`` above), + # just reached via a different code path in the provider; reuses the + # same event_type with ``operation="queued_command"`` rather than + # inventing a sibling type for the identical concept. + "queued_command": "claude_queue_operation", # 7,387 + # Task/todo evidence. + "task_status": "claude_task_status", # 80: one polled background task's status + "task_reminder": "claude_task_reminder", # 19,751: 37% carry real todo-list + # state (id/subject/status/blocks/ + # blockedBy) when non-empty -- + # NOT uniformly noise despite the + # majority being an empty-list + # reminder; see the todo_reminder + # sibling below for the contrast + # that justifies keeping this one. + # Misc small, real, evidence-bearing signals. + "diagnostics": "claude_diagnostics", # 1,583: bounded per-file diagnostic counts + "goal_status": "claude_agent_goal_status", # 367: sentinel goal condition evaluation + "date_change": "claude_date_change", # 163: session crossed a calendar day + "read_truncation_notice": "claude_read_truncation_notice", # 97: a Read tool output was truncated + "ultrathink_effort": "claude_agent_effort", # 11: explicit reasoning-effort signal + "structured_output": "claude_structured_output", # 1: rare but potentially meaningful + "max_turns_reached": "claude_max_turns_reached", # 1: session hit a turn budget +} + +# Subtypes audited and found to carry zero information content in the live +# corpus -- dropped (no event emitted), the same evidentiary bar applied to +# ``init``/``mode`` above, not an assumption: +# total_tokens_reminder (5,677) -- the literal string +# "Infinite tokens left" in every sample +# checked; zero variance observed. +# todo_reminder (5) -- always {"content": [], "itemCount": 0} in every +# occurrence observed (contrast with the sibling task_reminder above, +# which is 37% non-empty and kept); volume too small to rule out a +# future non-empty value, but zero signal in every occurrence measured. +# context_tip (11) -- CLI feature-adoption UI hints ("try /compact", +# "you have background agents stopped") aimed at the human operator's +# product experience, not evidence about the session's actual work. +# companion_intro (1) -- a novelty/branding record (pet companion name); +# not operational evidence by any reading. +_ATTACHMENT_TRANSIENT_SUBTYPES = frozenset( + { + "total_tokens_reminder", + "todo_reminder", + "context_tip", + "companion_intro", + } +) + +# Fallback for an attachment subtype not in the table above -- e.g. a new +# Claude Code CLI version introducing a 39th subtype. FAIL LOUD: still +# persisted (never silently dropped into the generic bucket, which is +# exactly the collapse this dispatch replaces), tagged with a event_type +# that is greppable/triageable on its own, distinct from every classified +# bucket above. +_ATTACHMENT_UNCLASSIFIED_EVENT_TYPE = "claude_attachment_unclassified" + + +_SKILL_LISTING_NAME_RE = re.compile(r"^-\s*([\w-]+):", re.MULTILINE) + + +def _bounded_delta_payload(attachment: Mapping[str, object]) -> dict[str, object]: + """Bound a capability-delta attachment to names/counts. + + ``deferred_tools_delta``/``mcp_instructions_delta``/``agent_listing_delta`` + carry an ``added*`` name list alongside an ``added*`` body-text list (full + tool/skill/MCP-server instruction blocks, potentially large and already + duplicated on disk or in the provider's own capability registry). Keeping + the names but dropping the body text matches the existing bounded-summary + precedent in ``_tool_execution_result_payload`` (hunk counts, not full + diffs) and ``_file_history_snapshot`` handling (file list, not file + contents). + """ + added_names: list[str] = [] + added_body_count = 0 + for key, value in attachment.items(): + if not isinstance(value, list): + continue + if key.endswith("Names") or key.endswith("Types"): + added_names.extend(str(v) for v in value if isinstance(v, str)) + elif key.endswith("Lines") or key.endswith("Blocks"): + added_body_count += len(value) + return {"added_names": added_names, "added_body_count": added_body_count} + + +def _bounded_capability_snapshot_payload(attachment: Mapping[str, object]) -> dict[str, object]: + """Bound a capability-snapshot attachment to names/counts. + + ``skill_listing`` carries one large concatenated-markdown ``content`` + string (all available skills' full descriptions); ``invoked_skills`` + carries a ``skills`` list whose ``content`` field is each skill's full + body. Both duplicate content that already exists as a skill file on + disk -- extract just the names (bounded, queryable) rather than persist + the full text verbatim into every session that loads the skill roster. + """ + skills = attachment.get("skills") + if isinstance(skills, list): + names = [str(skill.get("name")) for skill in skills if isinstance(skill, dict) and skill.get("name")] + return {"skill_names": names, "skill_count": len(skills)} + content = attachment.get("content") + if isinstance(content, str): + names = _SKILL_LISTING_NAME_RE.findall(content) + return {"skill_names": names, "skill_count": len(names)} + return {"skill_names": [], "skill_count": 0} + + +def _bounded_diagnostics_payload(attachment: Mapping[str, object]) -> dict[str, object]: + """Bound a diagnostics attachment to per-file counts, not full messages. + + ``diagnostics.files[].diagnostics[]`` carries full Pyright/LSP message + text, source ranges, and codes per finding -- unbounded in principle (as + many findings as the language server reports). Persist file-level counts + (queryable: "how many diagnostics did this session generate, on which + files") rather than the full message text, matching the + ``structured_patch_hunk_count`` precedent in ``_tool_execution_result_payload``. + """ + files = attachment.get("files") + if not isinstance(files, list): + return {"file_count": 0, "diagnostic_count": 0, "files": []} + file_summaries: list[dict[str, object]] = [] + total = 0 + for file_entry in files: + if not isinstance(file_entry, dict): + continue + diagnostics = file_entry.get("diagnostics") + count = len(diagnostics) if isinstance(diagnostics, list) else 0 + total += count + uri = file_entry.get("uri") + file_summaries.append({"uri": uri if isinstance(uri, str) else None, "diagnostic_count": count}) + return {"file_count": len(file_summaries), "diagnostic_count": total, "files": file_summaries} + + +# Subtypes whose generic ``dict(attachment)`` payload carries unbounded free +# text (full injected instruction blocks, full skill bodies, full diagnostic +# messages) -- these get a dedicated bounded builder instead of the raw +# pass-through every other subtype uses. Real file/reference content +# (file/edited_text_file/nested_memory/plan_file_reference/task_reminder) is +# deliberately NOT in this set: the full text there IS the evidence, not +# duplicated decoration around it. +_ATTACHMENT_BOUNDED_PAYLOAD_BUILDERS: dict[str, Callable[[Mapping[str, object]], dict[str, object]]] = { + "deferred_tools_delta": _bounded_delta_payload, + "mcp_instructions_delta": _bounded_delta_payload, + "agent_listing_delta": _bounded_delta_payload, + "skill_listing": _bounded_capability_snapshot_payload, + "invoked_skills": _bounded_capability_snapshot_payload, + "diagnostics": _bounded_diagnostics_payload, +} + + +def _attachment_sidecar_event(item: dict[str, object], timestamp: str | None) -> ParsedSessionEvent | None: + """Build the typed session_event for one ``attachment`` sidecar record. + + Subtype-dispatched (see ``_ATTACHMENT_SUBTYPE_EVENT_TYPES`` above) instead + of the collapsed single ``claude_attachment`` event_type this replaces. + Returns ``None`` only when the record carries no usable ``attachment`` + dict, or the subtype is confirmed-transient (``_ATTACHMENT_TRANSIENT_SUBTYPES``). + """ + attachment = item.get("attachment") + if not isinstance(attachment, dict): + return None + raw_subtype = attachment.get("type") + subtype = str(raw_subtype) if raw_subtype is not None else None + if subtype in _ATTACHMENT_TRANSIENT_SUBTYPES: + return None + event_type = ( + _ATTACHMENT_SUBTYPE_EVENT_TYPES.get(subtype, _ATTACHMENT_UNCLASSIFIED_EVENT_TYPE) + if subtype + else _ATTACHMENT_UNCLASSIFIED_EVENT_TYPE + ) + if subtype == "queued_command": + # Fold into the shared claude_queue_operation shape (operation/content) + # instead of the raw commandMode/prompt field names. + payload: dict[str, object] = { + "operation": "queued_command", + "content": attachment.get("prompt"), + "command_mode": attachment.get("commandMode"), + } + elif subtype is not None and subtype in _ATTACHMENT_BOUNDED_PAYLOAD_BUILDERS: + payload = _ATTACHMENT_BOUNDED_PAYLOAD_BUILDERS[subtype](attachment) + else: + payload = dict(attachment) + payload["summary"] = subtype or "attachment" + return ParsedSessionEvent(event_type=event_type, timestamp=timestamp, payload=payload) + def _sidecar_evidence_payload(record_type: str, item: dict[str, object]) -> dict[str, object] | None: - """Return a typed evidence payload for a skipped-as-message sidecar record. + """Return a typed evidence payload for a non-message sidecar record. Returns ``None`` when the specific record instance carries no usable - signal (e.g. an ``attachment`` record whose ``attachment`` field is - missing) so the caller can skip emitting an empty event. + signal (e.g. a ``last-prompt`` record whose ``lastPrompt`` field is + empty) so the caller can skip emitting an empty event. ``attachment`` is + handled separately by ``_attachment_sidecar_event`` (subtype-dependent + dispatch), not here. """ if record_type == "agent-name": name = _string_field(item, "agentName") @@ -278,14 +554,6 @@ def _sidecar_evidence_payload(record_type: str, item: dict[str, object]) -> dict else: payload["summary"] = operation return payload - if record_type == "attachment": - attachment = item.get("attachment") - if not isinstance(attachment, dict): - return None - payload = dict(attachment) - attachment_kind = attachment.get("type") - payload["summary"] = str(attachment_kind) if attachment_kind is not None else "attachment" - return payload if record_type == "ai-title": ai_title = _string_field(item, "aiTitle") return {"ai_title": ai_title, "summary": ai_title} if ai_title else None @@ -322,7 +590,7 @@ def _accumulate_delegation_progress( """Fold one ``progress``/``agent_progress`` record into its dispatch edge. Only ``data.type == "agent_progress"`` carries a genuine dispatcher edge - (see the classification comment above ``_SKIPPED_SIDECAR_RECORD_TYPES``); + (see the classification comment above ``_NON_MESSAGE_SIDECAR_RECORD_TYPES``); every occurrence under the same ``parentToolUseID`` is one streaming tick of the same subagent dispatch, so ticks are counted and time-bounded rather than persisted as one row apiece. @@ -1125,14 +1393,14 @@ def _parse_code_records( notification = _task_notification_from_record(item, message) # These twelve record types are never chat content -- see the - # classification comment above ``_SKIPPED_SIDECAR_RECORD_TYPES`` for - # why each one either persists as typed ``session_events`` evidence - # (polylogue-pbuh) or stays genuinely transient. ``progress`` (hook - # lifecycle pings, streaming tool-progress ticks, and the one + # classification comment above ``_NON_MESSAGE_SIDECAR_RECORD_TYPES`` + # for why each one either persists as typed ``session_events`` + # evidence (polylogue-pbuh) or stays genuinely transient. ``progress`` + # (hook lifecycle pings, streaming tool-progress ticks, and the one # evidence-bearing subtype ``agent_progress``) previously also # produced empty ``tool_result``-shaped message rows before the # record-type check below existed; see #1617 for that forensic. - if record_type in _SKIPPED_SIDECAR_RECORD_TYPES: + if record_type in _NON_MESSAGE_SIDECAR_RECORD_TYPES: if notification is not None: background_notifications.append((notification, record_uuid, timestamp)) if record_type == "progress": @@ -1170,6 +1438,17 @@ def _parse_code_records( else None, ) ) + elif record_type == "attachment": + # Subtype-dispatched -- see ``_attachment_sidecar_event`` + # and ``_ATTACHMENT_SUBTYPE_EVENT_TYPES`` above. Handled + # here rather than through the generic + # ``_SIDECAR_EVENT_TYPES``/``_sidecar_evidence_payload`` + # path below because the event_type itself depends on the + # nested ``attachment.type``, not just the outer + # record_type. + attachment_event = _attachment_sidecar_event(item, timestamp) + if attachment_event is not None: + session_events.append(attachment_event) event_type = _SIDECAR_EVENT_TYPES.get(record_type) if event_type is not None: evidence_payload = _sidecar_evidence_payload(record_type, item) diff --git a/tests/unit/sources/test_claude_code_sidecar_evidence.py b/tests/unit/sources/test_claude_code_sidecar_evidence.py index 477a739856..d7f5c15f17 100644 --- a/tests/unit/sources/test_claude_code_sidecar_evidence.py +++ b/tests/unit/sources/test_claude_code_sidecar_evidence.py @@ -2,7 +2,7 @@ Each test exercises ``polylogue.sources.parsers.claude.code_parser``, the sole production code path that decides what happens to a Claude Code JSONL -record whose ``type`` is in ``_SKIPPED_SIDECAR_RECORD_TYPES``. Before this +record whose ``type`` is in ``_NON_MESSAGE_SIDECAR_RECORD_TYPES``. Before this bead, every record type below was silently dropped by a bare frozenset membership check with no per-type rationale; these tests pin that dropping one of the evidence-bearing types (or reverting the ai-title title-override) @@ -264,10 +264,14 @@ def test_queue_operation_enqueue_persists_content_dequeue_does_not() -> None: ] -def test_attachment_persists_generic_typed_payload() -> None: - """attachment.type covers 20 distinct shapes in the live corpus; this - pins that the generic pass-through keeps the full typed payload (not an - opaque blob truncation) for at least one representative shape.""" +def test_attachment_file_subtype_gets_its_own_event_type() -> None: + """A real referenced file must not share an event_type with hook chatter. + + ``attachment.type`` covers 38 distinct shapes in the live corpus + (polylogue lane, 2026-07-31 full-corpus enumeration); this pins that a + real file attachment is routed to ``claude_attachment_file``, not the + single collapsed ``claude_attachment`` bucket the dispatch replaced. + """ parsed = parse_code( [ { @@ -281,12 +285,202 @@ def test_attachment_persists_generic_typed_payload() -> None: events = [(e.event_type, e.payload) for e in parsed.session_events] assert events == [ ( - "claude_attachment", + "claude_attachment_file", {"type": "file", "path": "/tmp/example.txt", "sizeBytes": 42, "summary": "file"}, ) ] +def test_attachment_hook_subtypes_share_one_event_type() -> None: + """Six hook-lifecycle subtypes are the same real-world entity (a hook + firing) distinguished by outcome, not six near-identical event types.""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-hook", + "attachment": {"type": "hook_success", "hookName": "SessionStart:startup"}, + }, + { + "type": "attachment", + "sessionId": "sess-hook", + "attachment": {"type": "hook_blocking_error", "hookName": "PreToolUse:Bash"}, + }, + ], + "sess-hook", + ) + event_types = [e.event_type for e in parsed.session_events] + assert event_types == ["claude_hook_event", "claude_hook_event"] + + +def test_attachment_queued_command_reuses_queue_operation_event_type() -> None: + """``queued_command`` (attachment subtype) and ``queue-operation`` + (top-level record type) describe the same message-queue entity through + two different provider code paths -- they must land on the same + event_type, not a sibling type for an identical concept.""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-queue", + "attachment": {"type": "queued_command", "prompt": "run the tests", "commandMode": "prompt"}, + } + ], + "sess-queue", + ) + events = [(e.event_type, e.payload) for e in parsed.session_events] + assert events == [ + ( + "claude_queue_operation", + { + "operation": "queued_command", + "content": "run the tests", + "command_mode": "prompt", + "summary": "queued_command", + }, + ) + ] + + +def test_attachment_transient_subtype_emits_no_event() -> None: + """Confirmed-zero-information subtypes (e.g. the constant + total_tokens_reminder text) are dropped, not persisted as noise.""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-transient", + "attachment": { + "type": "total_tokens_reminder", + "text": "Infinite tokens left", + }, + } + ], + "sess-transient", + ) + assert parsed.session_events == [] + + +def test_attachment_unrecognized_subtype_fails_loud() -> None: + """A future/unrecognized attachment subtype must still surface -- tagged + distinctly so it is triageable, not silently merged into a known bucket + or dropped on the floor.""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-unknown", + "attachment": {"type": "some_future_subtype", "value": 1}, + } + ], + "sess-unknown", + ) + events = [(e.event_type, e.payload) for e in parsed.session_events] + assert events == [ + ( + "claude_attachment_unclassified", + {"type": "some_future_subtype", "value": 1, "summary": "some_future_subtype"}, + ) + ] + + +def test_attachment_deferred_tools_delta_drops_body_text_keeps_names() -> None: + """Capability deltas keep the added tool/skill names but drop the full + injected instruction-block text (unbounded, duplicative).""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-delta", + "attachment": { + "type": "deferred_tools_delta", + "addedNames": ["WebFetch", "WebSearch"], + "addedLines": ["full description of WebFetch...", "full description of WebSearch..."], + }, + } + ], + "sess-delta", + ) + events = [(e.event_type, e.payload) for e in parsed.session_events] + assert events == [ + ( + "claude_capability_delta", + { + "added_names": ["WebFetch", "WebSearch"], + "added_body_count": 2, + "summary": "deferred_tools_delta", + }, + ) + ] + + +def test_attachment_skill_listing_extracts_names_not_full_text() -> None: + """``skill_listing`` keeps skill names, not the full concatenated + markdown description of every available skill.""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-skills", + "attachment": { + "type": "skill_listing", + "content": "- update-config: long description here\n- keybindings-help: another description", + }, + } + ], + "sess-skills", + ) + events = [(e.event_type, e.payload) for e in parsed.session_events] + assert events == [ + ( + "claude_capability_snapshot", + { + "skill_names": ["update-config", "keybindings-help"], + "skill_count": 2, + "summary": "skill_listing", + }, + ) + ] + + +def test_attachment_diagnostics_bounds_to_per_file_counts() -> None: + """``diagnostics`` keeps per-file finding counts, not the full LSP + message text/ranges/codes for every finding.""" + parsed = parse_code( + [ + { + "type": "attachment", + "sessionId": "sess-diag", + "attachment": { + "type": "diagnostics", + "files": [ + { + "uri": "/repo/foo.py", + "diagnostics": [ + {"message": "long pyright message", "severity": "Error"}, + {"message": "another long message", "severity": "Warning"}, + ], + } + ], + }, + } + ], + "sess-diag", + ) + events = [(e.event_type, e.payload) for e in parsed.session_events] + assert events == [ + ( + "claude_diagnostics", + { + "file_count": 1, + "diagnostic_count": 2, + "files": [{"uri": "/repo/foo.py", "diagnostic_count": 2}], + "summary": "diagnostics", + }, + ) + ] + + def test_progress_agent_progress_dedups_into_one_delegation_event() -> None: """Three ``agent_progress`` ticks under the same dispatching tool_use collapse into ONE ``claude_delegation_progress`` event with a tick count