diff --git a/docs/integrations/lark-kanban-control-plane-adapter.md b/docs/integrations/lark-kanban-control-plane-adapter.md index 53736bcbb0..04569c53bb 100644 --- a/docs/integrations/lark-kanban-control-plane-adapter.md +++ b/docs/integrations/lark-kanban-control-plane-adapter.md @@ -188,7 +188,7 @@ The recommended path is `setup`, using user identity by default. It preflights python3 -m loopx.cli lark-kanban doctor lark-cli auth login --domain base --recommend python3 -m loopx.cli lark-kanban setup --base-name "LoopX Kanban POC" --execute -python3 -m loopx.cli lark-kanban sync-loopx-todos --goal-id --execute +python3 -m loopx.cli lark-kanban sync-loopx-todos --goal-id --sink-visibility shared --execute ``` The local config stores the reusable Base token, table id, view ids, identity, @@ -206,7 +206,7 @@ To use someone else's shared board, store its URL or IDs: ```bash python3 -m loopx.cli lark-kanban use --base-url "" python3 -m loopx.cli lark-kanban config -python3 -m loopx.cli lark-kanban heartbeat --execute-lark +python3 -m loopx.cli lark-kanban heartbeat --execute-lark --sink-visibility shared ``` `sync-loopx-todos` reads the goal active state from the LoopX registry and @@ -218,6 +218,29 @@ cards; claimed agent todos become `Claimed`; blocked/done todos map to as a worker-launch row; the shared board must not receive raw local checkout or active-state paths. +`sync-loopx-todos` and `heartbeat` default to `--sink-visibility shared`. Every +record writer removes canonical provider-credential shapes before serialization, +including on an `owner-only` board. Shared visibility additionally applies the +local-path, private-link, and private-reference redaction boundary recursively to +nested record keys and values while preserving only structurally valid +record-link IDs. It recognizes Unix, drive-letter, UNC, and extended Windows +paths, plus credential assignments that use compound environment-variable names, +credential-bearing URL userinfo, and terminal control sequences around a token. +Configured Explore whiteboards apply the same visibility boundary to node and +edge display text before SVG or Mermaid is encoded, and shared Explore stage +documents sanitize lane labels before their `docs +update` content is built. +Redaction never runs over raw markup, so encoded tags and diagram delimiters +stay intact even when a value ends with a private link. Private-key blocks are +removed in full rather than leaving their body behind. The automatic issue-fix +Explore synchronization uses the same shared boundary. + +`sink_visibility` is a caller declaration; the adapter cannot verify the remote +Base's effective access control. Redaction is shape-based, not a complete +data-loss-prevention system: it does not classify every arbitrary high-entropy +value without a provider prefix or credential label. Never paste credentials +into board fields, and rotate any credential that may already have reached shared +storage rather than relying on later clean-up. + Issue outcomes are derived, not persisted separately. Every feasibility row is projected; a PR lifecycle row enriches it only through an explicit matching `repo` and `issue_ref`. This avoids title/branch guessing and makes the issue @@ -261,13 +284,13 @@ python3 -m loopx.cli lark-kanban doctor python3 -m loopx.cli lark-kanban setup --base-name "LoopX Kanban POC" --execute python3 -m loopx.cli lark-kanban use --base-url "" python3 -m loopx.cli lark-kanban config -python3 -m loopx.cli lark-kanban sync-loopx-todos --goal-id --execute +python3 -m loopx.cli lark-kanban sync-loopx-todos --goal-id --sink-visibility shared --execute python3 -m loopx.cli lark-kanban sync-projection --projection-file --include-done --reconcile-source --source-snapshot-complete python3 -m loopx.cli lark-kanban plan-create --base-name "LoopX Kanban POC" python3 -m loopx.cli lark-kanban create-board --base-name "LoopX Kanban POC" --execute -python3 -m loopx.cli lark-kanban seed-task --base-token --table-id --execute -python3 -m loopx.cli lark-kanban seed-cases --base-token --table-id
--execute -python3 -m loopx.cli lark-kanban heartbeat --base-token --table-id
--execute-lark +python3 -m loopx.cli lark-kanban seed-task --base-token --table-id
--sink-visibility shared --execute +python3 -m loopx.cli lark-kanban seed-cases --base-token --table-id
--sink-visibility shared --execute +python3 -m loopx.cli lark-kanban heartbeat --base-token --table-id
--execute-lark --sink-visibility shared ``` `setup`, `create-board`, `seed-task`, `seed-cases`, `sync-loopx-todos`, and @@ -275,6 +298,11 @@ python3 -m loopx.cli lark-kanban heartbeat --base-token --table-id
]+|\\b(?:ak|sk)[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]+|(?]{12,}|\\b(?:password|secret)[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]{12,}|-{3,}\\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY|\\btoken[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]{12,})", "i"); +const SECRET_LIKE_SHAPE = new RegExp("(?:\\bbearer\\s+[a-z0-9._~+/=-]{16,}|\\b[a-z][a-z0-9+.-]*://[^/\\s:@`'\\\"<>]+:[^/\\s@`'\\\"<>]+@[^\\s`'\\\"<>]*|\\b(?:(?:aws[_-]?)?secret[_-]?access[_-]?key|(?:access|api|secret)[_-]?key)[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]+|\\b(?:ak|sk)[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]+|(?]{12,}|\\b(?:password|secret)[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]{12,}|-{3,}\\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY[\\s\\S]*?(?:-{3,}\\s*END (?:[A-Z]+ )?PRIVATE KEY-{3,}|$)|\\btoken[\\\"']?\\s*[=:]\\s*[\\\"']?[^\\s`'\\\"<>]{12,})", "i"); const LOCAL_PATH_SHAPE = new RegExp("(?]+|[A-Za-z]:[\\\\/][^\\s`'\\\"<>]+|\\\\\\\\[A-Za-z0-9_.-]+\\\\[^\\s`'\\\"<>]+)", "i"); export const CREDENTIAL_WORD_PATTERNS: RegExp[] = [/\bBearer\b/i, /\bpassword\b/i, /\bsecret\b/i]; export const PRIVATE_TEXT_PATTERNS: RegExp[] = [ diff --git a/loopx/extensions/lark/presentation/explore_results.py b/loopx/extensions/lark/presentation/explore_results.py index fe0316152d..7388378428 100644 --- a/loopx/extensions/lark/presentation/explore_results.py +++ b/loopx/extensions/lark/presentation/explore_results.py @@ -15,7 +15,9 @@ from __future__ import annotations import hashlib +import html import json +import re from dataclasses import dataclass, field from pathlib import Path from typing import Any, Mapping @@ -44,16 +46,21 @@ from ....presentation.explore_views import ( PRESENTATION_MODE_DUAL_VIEW, build_explore_presentation_bundle, + build_vertical_explore_mermaid, explore_source_digest, explore_source_revision, validate_explore_view_freshness, ) from ..private_json import write_private_json_atomic from .kanban import ( + _credential_safe_lark_values, _extract_base_token, _extract_table_id, _extract_created_record_id, + _public_safe_lark_values, _public_safe_text, + _redact_public_safe_shapes, + _redact_credential_shapes, _select_options, lark_record_rows, now_lark_datetime, @@ -337,6 +344,7 @@ def _build_upsert_command( record_id: str | None, values: Mapping[str, Any], ) -> list[str]: + safe_values = _credential_safe_lark_values(dict(values)) args = [ config.cli_bin, "base", @@ -350,7 +358,7 @@ def _build_upsert_command( ] if record_id: args.extend(["--record-id", record_id]) - args.extend(["--json", _record_json_args(values)]) + args.extend(["--json", _record_json_args(safe_values)]) return args @@ -824,6 +832,121 @@ def _role_sink( } +_EXPLORE_NODE_DISPLAY_FIELDS = ("title", "summary", "blocked_reason") +_EXPLORE_EDGE_DISPLAY_FIELDS = ("edge_type", "summary") +_SVG_TEXT_SPAN_PATTERN = re.compile( + r"(?P]*>)(?P[^<]*)(?P)" +) +_MERMAID_LABEL_PATTERN = re.compile(r'"(?P[^"\n]*)"') + + +def _redact_visual_display_text(value: Any, *, public_safe: bool) -> str: + return ( + _redact_public_safe_shapes(value) + if public_safe + else _redact_credential_shapes(value) + ) + + +def _redact_explore_display_records( + records: Any, + *, + fields: tuple[str, ...], + public_safe: bool, +) -> tuple[list[dict[str, Any]], bool]: + sanitized: list[dict[str, Any]] = [] + changed = False + for record in records or []: + if not isinstance(record, Mapping): + continue + view = dict(record) + for display_field in fields: + value = view.get(display_field) + if not isinstance(value, str) or not value: + continue + redacted = _redact_visual_display_text(value, public_safe=public_safe) + if redacted != value: + view[display_field] = redacted + changed = True + sanitized.append(view) + return sanitized, changed + + +def _redact_explore_display_values( + nodes: Any, + edges: Any, + *, + public_safe: bool, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], bool]: + """Redact display text before any SVG or Mermaid encoding. + + IDs, parent links, status, tags and timestamps stay untouched so the + topology, lane assignment and freshness keep their structural meaning. + """ + + redacted_nodes, nodes_changed = _redact_explore_display_records( + nodes, fields=_EXPLORE_NODE_DISPLAY_FIELDS, public_safe=public_safe + ) + redacted_edges, edges_changed = _redact_explore_display_records( + edges, fields=_EXPLORE_EDGE_DISPLAY_FIELDS, public_safe=public_safe + ) + return redacted_nodes, redacted_edges, nodes_changed or edges_changed + + +def _redact_explore_projection_display( + projection: Mapping[str, Any], + *, + public_safe: bool, +) -> dict[str, Any]: + nodes, edges, _ = _redact_explore_display_values( + projection.get("nodes") or [], + projection.get("edges") or [], + public_safe=public_safe, + ) + return {**projection, "nodes": nodes, "edges": edges} + + +def _redact_rendered_display_spans( + source: str, + *, + input_format: str, + public_safe: bool, +) -> str: + """Rewrite only decoded display spans of a caller-supplied rendered view. + + The semantic paths redact values before encoding. This guard keeps the + visibility boundary for pre-rendered projections while never matching + enclosing markup, so ```` and Mermaid delimiters stay intact. + """ + + if input_format == "svg": + + def redact_svg_text(match: re.Match[str]) -> str: + content = html.unescape(match.group("content")) + return ( + match.group("open") + + html.escape( + _redact_visual_display_text(content, public_safe=public_safe) + ) + + match.group("close") + ) + + return _SVG_TEXT_SPAN_PATTERN.sub(redact_svg_text, source) + if input_format == "mermaid": + + def redact_mermaid_label(match: re.Match[str]) -> str: + return ( + '"' + + _redact_visual_display_text( + match.group("content"), public_safe=public_safe + ) + + '"' + ) + + return _MERMAID_LABEL_PATTERN.sub(redact_mermaid_label, source) + return source + + def sync_explore_visual_to_lark( config: LarkExploreConfig, *, @@ -833,12 +956,16 @@ def sync_explore_visual_to_lark( semantic_digest: str, display_projection: Mapping[str, Any] | None = None, view_key: str | None = None, + sink_visibility: str = SINK_VISIBILITY_OWNER_ONLY, execute: bool = False, runner: CommandRunner = default_subprocess_runner, defer_readback: bool = False, ) -> dict[str, Any]: """Publish a configured whiteboard without conflating it with Base rows.""" + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") + public_safe = sink_visibility == SINK_VISIBILITY_SHARED if not isinstance(visual_sink, Mapping): return { "ok": True, @@ -875,18 +1002,22 @@ def sync_explore_visual_to_lark( "freshness": freshness, "error": freshness["reason"], } - graph = ( - dict(display_projection) - if isinstance(display_projection, Mapping) - else build_explore_graph_view( + if isinstance(display_projection, Mapping): + graph = dict(display_projection) + else: + render_nodes, render_edges, _ = _redact_explore_display_values( projection.get("nodes") or [], projection.get("edges") or [], + public_safe=public_safe, + ) + graph = build_explore_graph_view( + render_nodes, + render_edges, statuses=visual_sink.get("statuses") or [], tags=visual_sink.get("tags") or [], include_ancestors=bool(visual_sink.get("include_ancestors", True)), node_limit=max(1, int(visual_sink.get("mermaid_node_limit") or 100)), ) - ) sink_key = str(view_key or visual_sink.get("view_role") or "visual").strip() or "visual" try: style = resolve_explore_board_style(visual_sink) @@ -918,6 +1049,11 @@ def sync_explore_visual_to_lark( "renderer": style.renderer, "error": f"display projection does not contain {style.source_key} source", } + rendered_source = _redact_rendered_display_spans( + rendered_source, + input_format=style.input_format, + public_safe=public_safe, + ) delivery_material = json.dumps( { "source_digest": semantic_digest, @@ -1001,6 +1137,8 @@ def sync_explore_visual_to_lark( "style_fallback": style_fallback, "renderer": style.renderer, "input_format": style.input_format, + "sink_visibility": sink_visibility, + "public_safe_redaction": public_safe, "docx_token": str(visual_sink.get("docx_token") or "") or None, "graph_counts": graph.get("graph_counts"), "filter": graph.get("filter"), @@ -1036,11 +1174,15 @@ def sync_explore_visuals_to_lark( projection: Mapping[str, Any], visual_sinks: Mapping[str, Any], config_path: Path, + sink_visibility: str = SINK_VISIBILITY_OWNER_ONLY, execute: bool = False, runner: CommandRunner = default_subprocess_runner, ) -> dict[str, Any]: """Generate and publish configured same-source Explore views.""" + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") + public_safe = sink_visibility == SINK_VISIBILITY_SHARED bundle = build_explore_presentation_bundle(projection) requested_roles = [ role @@ -1081,6 +1223,25 @@ def sync_explore_visuals_to_lark( }, ) role_view = role_bundle[role] + render_nodes, render_edges, display_redacted = _redact_explore_display_values( + role_view.get("nodes") or [], + role_view.get("edges") or [], + public_safe=public_safe, + ) + if display_redacted: + render_layout = build_vertical_explore_mermaid( + render_nodes, + render_edges, + view_role=role, + group_node_limit=stage_capacity, + ) + role_view = dict( + role_view, + nodes=render_nodes, + edges=render_edges, + mermaid=render_layout["mermaid"], + stage_views=render_layout["stage_views"], + ) stage_views = [ item for item in role_view.get("stage_views") or [] @@ -1097,6 +1258,7 @@ def sync_explore_visuals_to_lark( role_sink=role_sink, stage_views=stage_views, config_path=config_path, + sink_visibility=sink_visibility, execute=execute, runner=runner, read_local_config=read_lark_explore_local_config, @@ -1165,6 +1327,7 @@ def sync_explore_visuals_to_lark( semantic_digest=role_bundle["source_digest"], display_projection=stage_projection, view_key=f"{role}_stage_{stage_index:02d}", + sink_visibility=sink_visibility, execute=bool(execute and not readback_host_wait_exhausted), runner=runner, defer_readback=bool(execute and not readback_host_wait_exhausted), @@ -1204,6 +1367,7 @@ def sync_explore_visuals_to_lark( semantic_digest=role_bundle["source_digest"], display_projection=overview_projection, view_key=f"{role}_overview", + sink_visibility=sink_visibility, execute=bool(execute and not readback_host_wait_exhausted), runner=runner, defer_readback=bool(execute and not readback_host_wait_exhausted), @@ -1548,7 +1712,7 @@ def _finding_record_values(finding: Mapping[str, Any], *, goal_id: str, source_i def _public_safe_values(values: dict[str, Any]) -> dict[str, Any]: - return {key: _public_safe_text(value) if isinstance(value, str) else value for key, value in values.items()} + return _public_safe_lark_values(values) def _with_edge_link_values( @@ -1613,6 +1777,10 @@ def sync_explore_results_to_lark( if isinstance(item, Mapping) ], } + rows_by_table = { + table_key: [_credential_safe_lark_values(values) for values in rows] + for table_key, rows in rows_by_table.items() + } if public_safe: rows_by_table = { table_key: [_public_safe_values(values) for values in rows] for table_key, rows in rows_by_table.items() @@ -1886,6 +2054,7 @@ def sync_issue_fix_explore_on_material_change( agent_id: str | None = None, project: Path | None = None, state_file: Path | None = None, + sink_visibility: str = SINK_VISIBILITY_SHARED, execute: bool = False, external_sink_delivery_authorized: bool = True, runner: CommandRunner = default_subprocess_runner, @@ -1894,6 +2063,9 @@ def sync_issue_fix_explore_on_material_change( Failed/interrupted Lark writes remain retryable; advance the stored sink digest only after successful remote sync. """ + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") + from ....capabilities.issue_fix.explore_projection import ( build_issue_fix_executive_visual_projection, project_issue_fix_explore_graph, @@ -2027,6 +2199,7 @@ def sync_issue_fix_explore_on_material_change( projection=projection_result["projection"], visual_sinks=visual_sinks, config_path=config_path, + sink_visibility=sink_visibility, execute=False, runner=runner, ) @@ -2105,6 +2278,7 @@ def sync_issue_fix_explore_on_material_change( config, projection=projection_result["projection"], config_path=config_path, + sink_visibility=sink_visibility, execute=True, runner=runner, ) @@ -2119,19 +2293,27 @@ def sync_issue_fix_explore_on_material_change( projection=projection_result["projection"], visual_sinks=visual_sinks, config_path=config_path, + sink_visibility=sink_visibility, execute=True, runner=runner, ) else: + display_projection = None + if str(visual_sink.get("projection_mode") or "") == "issue_fix_two_lane": + display_projection = build_issue_fix_executive_visual_projection( + _redact_explore_projection_display( + projection_result["projection"], + public_safe=sink_visibility == SINK_VISIBILITY_SHARED, + ) + ) visual_sync = sync_explore_visual_to_lark( config, projection=projection_result["projection"], visual_sink=visual_sink, config_path=config_path, semantic_digest=digest, - display_projection=build_issue_fix_executive_visual_projection(projection_result["projection"]) - if str(visual_sink.get("projection_mode") or "") == "issue_fix_two_lane" - else None, + display_projection=display_projection, + sink_visibility=sink_visibility, execute=True, runner=runner, ) @@ -2238,7 +2420,7 @@ def build_explore_card_markdown(projection: Mapping[str, Any]) -> str: lines.append("**Exploring now**") for node in frontier[:5]: lines.append(f"- {node.get('title')}") - return "\n".join(lines).strip() + return _redact_credential_shapes("\n".join(lines).strip()) def build_explore_result_card( @@ -2256,7 +2438,7 @@ def build_explore_result_card( markdown = build_explore_card_markdown(projection) card = build_lark_markdown_reply_card( markdown, - title=title or f"Exploration map: {goal_id}", + title=_redact_credential_shapes(title or f"Exploration map: {goal_id}"), template=template, footer=( f"LoopX explore | {projection.get('generated_at')} | {projection.get('source_event_count')} result events" diff --git a/loopx/extensions/lark/presentation/explore_stage_document.py b/loopx/extensions/lark/presentation/explore_stage_document.py index 69570e5dbc..1f566cab48 100644 --- a/loopx/extensions/lark/presentation/explore_stage_document.py +++ b/loopx/extensions/lark/presentation/explore_stage_document.py @@ -15,6 +15,8 @@ command_error, run_command, ) +from .kanban import _redact_credential_shapes, _redact_public_safe_shapes +from .sink_visibility import SINK_VISIBILITY_SHARED class StageDocumentConfig(Protocol): @@ -36,20 +38,30 @@ def _stage_section_title(*, role: str, stage_index: int) -> str: return f"{_stage_section_prefix(role=role)} {stage_index:02d}" -def _stage_section_xml(stage: Mapping[str, Any], *, role: str) -> str: +def _stage_section_xml( + stage: Mapping[str, Any], *, role: str, sink_visibility: str +) -> str: stage_index = int(stage.get("stage_index") or 0) lane_labels = { "fix_pr": "PR issue-fix", "capability": "LoopX capability", "default": "Explore work", } - lanes = ( - ", ".join( - lane_labels.get(str(item), str(item).replace("_", " ").title()) - for item in stage.get("lanes") or [] + lane_texts: list[str] = [] + for item in stage.get("lanes") or []: + raw = str(item) + safe = ( + _redact_public_safe_shapes(raw) + if sink_visibility == SINK_VISIBILITY_SHARED + else _redact_credential_shapes(raw) ) - or "Explore work" - ) + lane_texts.append( + lane_labels.get( + safe, + safe.replace("_", " ").title() if safe == raw else safe, + ) + ) + lanes = ", ".join(lane_texts) or "Explore work" return ( f"

{_stage_section_title(role=role, stage_index=stage_index)}

" "

" @@ -214,6 +226,7 @@ def ensure_stage_whiteboards( role_sink: Mapping[str, Any], stage_views: list[Mapping[str, Any]], config_path: Path, + sink_visibility: str, execute: bool, runner: CommandRunner, read_local_config: LocalConfigReader, @@ -598,7 +611,9 @@ def observe(command: Mapping[str, Any], *, action: str) -> None: "--command", "append", "--content", - _stage_section_xml(stage, role=role), + _stage_section_xml( + stage, role=role, sink_visibility=sink_visibility + ), "--format", "json", ], diff --git a/loopx/extensions/lark/presentation/kanban.py b/loopx/extensions/lark/presentation/kanban.py index b359bcb6aa..f54e7bc715 100644 --- a/loopx/extensions/lark/presentation/kanban.py +++ b/loopx/extensions/lark/presentation/kanban.py @@ -24,6 +24,12 @@ plan_projection_source_reconcile, validate_projection_source_reconcile_request, ) +from ....public_safe_text import ( + HOME_RELATIVE_PATH_PATTERN, + PRESENTATION_LOCAL_PATH_PATTERNS, + SECRET_LIKE_SURFACE_PATTERN, + SINK_COMPOUND_CREDENTIAL_ASSIGNMENT_PATTERN, +) from .. import cli_transport from ..private_json import write_private_json_atomic from . import issue_fix_surface @@ -440,40 +446,117 @@ def _compact_text(value: Any, *, limit: int = TEXT_LIMIT) -> str: return text[: max(0, limit - 1)].rstrip() + "..." -def _public_safe_text(value: Any, *, limit: int = TEXT_LIMIT) -> str: - text = _compact_text(value, limit=limit) - patterns = ( - (re.compile(r"file://[^\s,;)]+", re.IGNORECASE), "[local-path-redacted]"), - ( - re.compile(r"(? str: + text = _ANSI_CONTROL_SEQUENCE_PATTERN.sub("", str(value or "")) + text = SECRET_LIKE_SURFACE_PATTERN.sub("[credential-redacted]", text) + return SINK_COMPOUND_CREDENTIAL_ASSIGNMENT_PATTERN.sub( + "[credential-redacted]", text + ) + + +_SHARED_REDACTION_PATTERNS = ( + (re.compile(r"file://[^\s,;)]+", re.IGNORECASE), "[local-path-redacted]"), + *((pattern, "[local-path-redacted]") for pattern in PRESENTATION_LOCAL_PATH_PATTERNS), + (HOME_RELATIVE_PATH_PATTERN, "[local-path-redacted]"), + ( + re.compile( + r"https?://[^\s,;)]*(?:feishu|larksuite|internal|corp|localhost|127\.0\.0\.1)[^\s,;)]*", + re.IGNORECASE, ), - (re.compile(r"\b(?:base|tbl|vew|rec)[A-Za-z0-9_-]{6,}\b"), "[external-id-redacted]"), - ( - re.compile( - r"\b(?:PRIVATE_MATERIAL|PRIVATE_REF|SECRET_REF|CREDENTIAL_REF)[A-Za-z0-9_.:-]*\b", - re.IGNORECASE, - ), - "[private-ref-redacted]", + "[private-link-redacted]", + ), + (re.compile(r"\b(?:base|tbl|vew|rec)[A-Za-z0-9_-]{6,}\b"), "[external-id-redacted]"), + ( + re.compile( + r"\b(?:PRIVATE_MATERIAL|PRIVATE_REF|SECRET_REF|CREDENTIAL_REF)[A-Za-z0-9_.:-]*\b", + re.IGNORECASE, ), - ) - for pattern, replacement in patterns: + "[private-ref-redacted]", + ), +) + + +def _redact_non_credential_shapes(text: str) -> str: + for pattern, replacement in _SHARED_REDACTION_PATTERNS: text = pattern.sub(replacement, text) - return _compact_text(text, limit=limit) + return text + + +def _redact_public_safe_shapes(value: Any) -> str: + """Apply the shared boundary without collapsing whitespace or truncating.""" + + return _redact_non_credential_shapes(_redact_credential_shapes(value)) + + +def _credential_safe_text(value: Any, *, limit: int = TEXT_LIMIT) -> str: + return _compact_text(_redact_credential_shapes(value), limit=limit) + + +def _public_safe_text(value: Any, *, limit: int = TEXT_LIMIT) -> str: + text = _compact_text(_redact_credential_shapes(value), limit=limit) + return _compact_text(_redact_non_credential_shapes(text), limit=limit) + + +_STRUCTURAL_ID_KEYS = frozenset({"id", "record_id", "record_ids"}) +_STRUCTURAL_RECORD_ID_PATTERN = re.compile(r"^rec[A-Za-z0-9_-]{6,}$") + + +def _public_safe_value(value: Any, *, preserve_external_id: bool = False) -> Any: + if isinstance(value, str): + if preserve_external_id and _STRUCTURAL_RECORD_ID_PATTERN.fullmatch(value): + return value + return _public_safe_text(value) + if isinstance(value, dict): + return { + _public_safe_text(str(key)): _public_safe_value( + item, + preserve_external_id=str(key).lower() in _STRUCTURAL_ID_KEYS, + ) + for key, item in value.items() + } + if isinstance(value, list): + return [ + _public_safe_value(item, preserve_external_id=preserve_external_id) + for item in value + ] + if isinstance(value, tuple): + return tuple( + _public_safe_value(item, preserve_external_id=preserve_external_id) + for item in value + ) + return value def _public_safe_lark_values(values: dict[str, Any]) -> dict[str, Any]: - return { - key: _public_safe_text(value) if isinstance(value, str) else value - for key, value in values.items() - } + credential_safe = _credential_safe_lark_values(values) + return {key: _public_safe_value(value) for key, value in credential_safe.items()} + + +def _credential_safe_value(value: Any) -> Any: + if isinstance(value, str): + return _redact_credential_shapes(value) + if isinstance(value, dict): + return { + _redact_credential_shapes(key): _credential_safe_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_credential_safe_value(item) for item in value] + if isinstance(value, tuple): + return tuple(_credential_safe_value(item) for item in value) + return value + + +def _credential_safe_lark_values(values: dict[str, Any]) -> dict[str, Any]: + return {key: _credential_safe_value(value) for key, value in values.items()} + + +def _sink_safe_lark_values(values: dict[str, Any], *, public_safe: bool) -> dict[str, Any]: + values = _credential_safe_lark_values(values) + return _public_safe_lark_values(values) if public_safe else values def now_lark_datetime(now: datetime | None = None) -> str: @@ -706,6 +789,7 @@ def build_record_upsert_command( record_id: str | None, values: dict[str, Any], ) -> list[str]: + safe_values = _credential_safe_lark_values(values) args = [ config.cli_bin, "base", @@ -719,7 +803,7 @@ def build_record_upsert_command( ] if record_id: args.extend(["--record-id", record_id]) - args.extend(["--json", _record_json_args(values)]) + args.extend(["--json", _record_json_args(safe_values)]) return args @@ -1819,6 +1903,7 @@ def _lark_record_from_projection_block( } ) values.update(issue_fix_surface.issue_fix_record_values(block, compact_text=_compact_text)) + values = _credential_safe_lark_values(values) return _public_safe_lark_values(values) if public_safe else values @@ -2039,6 +2124,7 @@ def sync_loopx_todos_to_lark_kanban( state_file: Path | None = None, include_done: bool = False, limit: int = 50, + sink_visibility: str = SINK_VISIBILITY_SHARED, execute: bool = False, runner: cli_transport.CommandRunner = cli_transport.default_subprocess_runner, ) -> dict[str, Any]: @@ -2049,6 +2135,10 @@ def sync_loopx_todos_to_lark_kanban( from ....control_plane.todos.path_resolution import resolve_todo_state_path from ....control_plane.todos.active_state_editing import section_bounds, todo_blocks + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") + public_safe = sink_visibility == SINK_VISIBILITY_SHARED + resolved_project, resolved_state_file = resolve_todo_state_path( registry_path=registry_path, goal_id=goal_id, @@ -2111,6 +2201,8 @@ def sync_loopx_todos_to_lark_kanban( "execute": execute, "goal_id": goal_id, "agent_id": agent_id, + "sink_visibility": sink_visibility, + "public_safe_redaction": public_safe, "base_token": config.base_token, "table_id": config.table_id, "view_id": config.view_id, @@ -2153,6 +2245,8 @@ def sync_loopx_todos_to_lark_kanban( state_file=resolved_state_file, priority=todo_priority_label(block, text_mode="prefix") or "P2", ) + if public_safe: + values = _public_safe_lark_values(values) result = cli_transport.run_command( build_record_upsert_command(config, record_id=record_map.get(key), values=values), execute=execute, @@ -2188,6 +2282,7 @@ def sync_loopx_todos_to_lark_kanban( row, goal_id=row_goal_id, source_id=outcome_source_id, + public_safe=public_safe, ) result = cli_transport.run_command( build_record_upsert_command( @@ -2230,6 +2325,8 @@ def sync_loopx_todos_to_lark_kanban( "execute": execute, "goal_id": goal_id, "agent_id": agent_id, + "sink_visibility": sink_visibility, + "public_safe_redaction": public_safe, "base_token": config.base_token, "table_id": config.table_id, "view_id": config.view_id, @@ -2356,7 +2453,7 @@ def _lark_record_from_todo_block( scope_text = str(scope or "") evidence = str(block.get("evidence") or block.get("reason") or block.get("note") or "") text = str(block.get("text") or "") - return { + values = { "Task": text, "Status": lark_status, "Claim": claim, @@ -2377,17 +2474,23 @@ def _lark_record_from_todo_block( "Last Error": "", "Last Result Code": None, } + return _credential_safe_lark_values(values) def seed_lark_kanban_task( config: LarkKanbanConfig, *, task: dict[str, Any], + sink_visibility: str = SINK_VISIBILITY_SHARED, execute: bool = False, runner: cli_transport.CommandRunner = cli_transport.default_subprocess_runner, ) -> dict[str, Any]: + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") + public_safe = sink_visibility == SINK_VISIBILITY_SHARED + safe_task = _sink_safe_lark_values(task, public_safe=public_safe) result = cli_transport.run_command( - build_record_upsert_command(config, record_id=None, values=task), + build_record_upsert_command(config, record_id=None, values=safe_task), execute=execute, runner=runner, ) @@ -2397,7 +2500,9 @@ def seed_lark_kanban_task( "schema_version": LARK_KANBAN_SCHEMA_VERSION, "record_id": record_id, "command": result, - "task": task, + "sink_visibility": sink_visibility, + "public_safe_redaction": public_safe, + "task": safe_task, } @@ -2405,15 +2510,19 @@ def seed_lark_kanban_records( config: LarkKanbanConfig, *, records: list[dict[str, Any]], + sink_visibility: str = SINK_VISIBILITY_SHARED, execute: bool = False, runner: cli_transport.CommandRunner = cli_transport.default_subprocess_runner, ) -> dict[str, Any]: + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") results: list[dict[str, Any]] = [] ok = True for record in records: result = seed_lark_kanban_task( config, task=record, + sink_visibility=sink_visibility, execute=execute, runner=runner, ) @@ -2425,16 +2534,20 @@ def seed_lark_kanban_records( "ok": ok, "schema_version": LARK_KANBAN_SCHEMA_VERSION, "record_count": len(records), + "sink_visibility": sink_visibility, + "public_safe_redaction": sink_visibility == SINK_VISIBILITY_SHARED, "created_record_ids": [item.get("record_id") for item in results], "records": results, } + def lark_kanban_heartbeat( config: LarkKanbanConfig, *, agent_id: str = DEFAULT_AGENT_ID, fixture: dict[str, Any] | None = None, worker_command: str | None = None, + sink_visibility: str = SINK_VISIBILITY_SHARED, execute_lark: bool = False, execute_worker: bool = False, complete_on_success: bool = False, @@ -2443,6 +2556,9 @@ def lark_kanban_heartbeat( now: datetime | None = None, worker_timeout_seconds: float = 600.0, ) -> dict[str, Any]: + if sink_visibility not in SINK_VISIBILITIES: + raise ValueError(f"sink_visibility must be one of {sorted(SINK_VISIBILITIES)}") + public_safe = sink_visibility == SINK_VISIBILITY_SHARED commands: list[dict[str, Any]] = [] if fixture is None: list_result = cli_transport.run_command( @@ -2490,6 +2606,7 @@ def lark_kanban_heartbeat( "Last Heartbeat": timestamp, "Run History": append_history(task.get("Run History"), f"{timestamp} {agent_id}: claimed"), } + claim_values = _sink_safe_lark_values(claim_values, public_safe=public_safe) claim_result = cli_transport.run_command( build_record_upsert_command(config, record_id=record_id, values=claim_values), execute=execute_lark, @@ -2506,6 +2623,7 @@ def lark_kanban_heartbeat( selected_task=task, ) command = (worker_command if worker_command is not None else str(task.get("Worker Command") or "")).strip() + safe_command = _redact_credential_shapes(command) if not command: worker = { "attempted": False, @@ -2523,10 +2641,10 @@ def lark_kanban_heartbeat( "executed": False, "ok": True, "reason": "execute_worker_not_set", - "command": command, + "command": safe_command, } final_status = STATUS_CLAIMED - evidence = f"dry-run worker command: {command}" + evidence = f"dry-run worker command: {safe_command}" result_code = None elif not allowed_worker_command(command, allowed_command_prefixes or []): worker = { @@ -2534,7 +2652,7 @@ def lark_kanban_heartbeat( "executed": False, "ok": False, "reason": "worker_command_prefix_not_allowed", - "command": command, + "command": safe_command, "allowed_prefixes": allowed_command_prefixes or [], } final_status = STATUS_BLOCKED @@ -2555,15 +2673,15 @@ def lark_kanban_heartbeat( "executed": True, "ok": bool(worker_result.get("ok")), "reason": "completed" if worker_result.get("ok") else "failed", - "command": command, + "command": safe_command, "returncode": worker_result.get("returncode"), - "stdout": _compact_text(worker_result.get("stdout"), limit=cli_transport.OUTPUT_LIMIT), - "stderr": _compact_text(worker_result.get("stderr"), limit=cli_transport.OUTPUT_LIMIT), + "stdout": _credential_safe_text(worker_result.get("stdout"), limit=cli_transport.OUTPUT_LIMIT), + "stderr": _credential_safe_text(worker_result.get("stderr"), limit=cli_transport.OUTPUT_LIMIT), } final_status = STATUS_DONE if worker_result.get("ok") and complete_on_success else STATUS_REVIEW if not worker_result.get("ok"): final_status = STATUS_BLOCKED - evidence = _compact_text( + evidence = _credential_safe_text( worker.get("stdout") or worker.get("stderr") or worker.get("reason"), limit=cli_transport.OUTPUT_LIMIT, ) @@ -2575,13 +2693,14 @@ def lark_kanban_heartbeat( "Last Heartbeat": timestamp, "Evidence": evidence, "Last Result Code": result_code, - "Last Error": "" if worker.get("ok") else _compact_text(worker.get("stderr") or worker.get("reason")), + "Last Error": "" if worker.get("ok") else _credential_safe_text(worker.get("stderr") or worker.get("reason")), "Run History": append_history( claim_values["Run History"], f"{timestamp} {agent_id}: worker {worker.get('reason')} -> {final_status}", ), "Handoff": _handoff_for_status(final_status, agent_id=agent_id), } + finish_values = _sink_safe_lark_values(finish_values, public_safe=public_safe) finish_result = cli_transport.run_command( build_record_upsert_command(config, record_id=record_id, values=finish_values), execute=execute_lark, diff --git a/loopx/public_safe_text.py b/loopx/public_safe_text.py index 9ec8cecc0b..e2c5c6c928 100644 --- a/loopx/public_safe_text.py +++ b/loopx/public_safe_text.py @@ -200,6 +200,17 @@ r"[A-Za-z0-9_]*(?:" + _COMPOUND_LABEL_SOURCE + r")[A-Za-z0-9_]*[\"']?\s*[:=]", re.IGNORECASE, ) +# Sink redaction needs the value in the same match so a label and its value are +# replaced together. Keep this value-inclusive form in the canonical owner rather +# than rebuilding it from the base label pattern in each presentation module. +SINK_COMPOUND_CREDENTIAL_ASSIGNMENT_PATTERN = re.compile( + r"(?:" + + COMPOUND_CREDENTIAL_FIELD_ASSIGNMENT_PATTERN.pattern + + r"|\b[A-Za-z0-9_]*(?:token|access[_-]?key|private[_-]?key)" + r"[A-Za-z0-9_]*[\"']?\s*[:=])" + r"\s*(?:\"[^\"\r\n]*\"|'[^'\r\n]*'|[^\s`'\"<>]+)", + re.IGNORECASE, +) # The credential half of the rule, on its own, named so the two capability faces # that ask it share one definition instead of each restating a category pair. Both # categories are listed rather than subtracted from `ALL_CATEGORIES`: @@ -348,14 +359,16 @@ REMOTE_LOCATION_SURFACE_PATTERN = re.compile(r"(?i)\b(?:https?|file|s3|gs|tos|hdfs)://") SECRET_LIKE_SURFACE_PATTERN = re.compile( r"(?i)(?:\bbearer\s+[a-z0-9._~+/=-]{16,}|" - r"\b(?:access|api|secret)[_-]?key[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" + r"\b[a-z][a-z0-9+.-]*://[^/\s:@`'\"<>]+:[^/\s@`'\"<>]+@[^\s`'\"<>]*|" + r"\b(?:(?:aws[_-]?)?secret[_-]?access[_-]?key|(?:access|api|secret)[_-]?key)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" r"\b(?:ak|sk)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" r"(?]{12,}|" r"\b(?:password|secret)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|" - r"-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY|" + r"-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY[\s\S]*?" + r"(?:-{3,}\s*END (?:[A-Z]+ )?PRIVATE KEY-{3,}|$)|" r"\btoken[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,})" ) diff --git a/packages/dsh-loopx-plugin/src/observer.ts b/packages/dsh-loopx-plugin/src/observer.ts index 078ec632f6..c6a1a00c40 100644 --- a/packages/dsh-loopx-plugin/src/observer.ts +++ b/packages/dsh-loopx-plugin/src/observer.ts @@ -94,7 +94,8 @@ const LOCAL_PATH_SURFACES = [ */ const SECRET_LIKE_SURFACES = [ /\bbearer\s+[a-z0-9._~+/=-]{16,}/iu, - /\b(?:access|api|secret)[_-]?key["']?\s*[=:]\s*["']?[^\s`'"<>]+/iu, + /\b[a-z][a-z0-9+.-]*:\/\/[^\/\s:@`'"<>]+:[^\/\s@`'"<>]+@[^\s`'"<>]*/iu, + /\b(?:(?:aws[_-]?)?secret[_-]?access[_-]?key|(?:access|api|secret)[_-]?key)["']?\s*[=:]\s*["']?[^\s`'"<>]+/iu, /\b(?:ak|sk)["']?\s*[=:]\s*["']?[^\s`'"<>]+/iu, /(?]{12,}/iu, /\b(?:password|secret)["']?\s*[=:]\s*["']?[^\s`'"<>]{12,}/iu, - /-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY/iu, + /-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY[\s\S]*?(?:-{3,}\s*END (?:[A-Z]+ )?PRIVATE KEY-{3,}|$)/iu, /\btoken["']?\s*[=:]\s*["']?[^\s`'"<>]{12,}/iu, ] as const const CREDENTIAL_FIELD_FAMILIES = new Set([ diff --git a/tests/control_plane/test_public_safety_credential_shape_owner.py b/tests/control_plane/test_public_safety_credential_shape_owner.py index 4551af7672..963803a6d3 100644 --- a/tests/control_plane/test_public_safety_credential_shape_owner.py +++ b/tests/control_plane/test_public_safety_credential_shape_owner.py @@ -55,11 +55,18 @@ "password assignment": "pass" + "word=hunter2hunter2", "api key colon": "api_key: qwertyuiopasdfghjkl", "secret assignment": "secret=abcdefghijklmn", + "aws secret access key assignment": "AWS_SECRET_ACCESS_KEY=" + "a1B2" * 10, + "database URI userinfo": "postgres://operator:" + "a1B2" * 10 + "@db.example/app", "token assignment": "tok" + "en=abcdefghijklmn", "access token assignment": "access_token=abcdefghijklmn", "bearer value": "Bearer " + "z" * 20, "signed key pair": "sk-" + "a" * 30, "private key material": "-----BEGIN RSA PRIVATE KEY-----", + "complete private key block": ( + "-----BEGIN RSA PRIVATE KEY-----\n" + + "MIIE" + "syntheticPrivateKeyBody" * 4 + "\n" + "-----END RSA PRIVATE KEY-----" + ), "openssh key material": "---- BEGIN OPENSSH PRIVATE KEY ----", "fine-grained pat": "github_pat_" + "a1B2" * 8, "aws access id": "AKIA" + "A1B2C3D4E5F6G7H8", @@ -121,6 +128,7 @@ def test_benign_text_still_passes_the_owner() -> None: "weekly cadence digest rendered for goal_42", "token budget left for this stage: 1200", "the operator rotated the deploy credentials yesterday", + "read GITHUB_TOKEN from the environment before starting the worker", ): assert not SECRET_LIKE_SURFACE_PATTERN.search(text), text diff --git a/tests/extensions/test_lark_shared_sink_redaction.py b/tests/extensions/test_lark_shared_sink_redaction.py new file mode 100644 index 0000000000..f607864fbe --- /dev/null +++ b/tests/extensions/test_lark_shared_sink_redaction.py @@ -0,0 +1,809 @@ +from __future__ import annotations + +import json +import re +import xml.etree.ElementTree as ET +from inspect import signature +from pathlib import Path + +import pytest + +from loopx.extensions.lark.presentation.explore_results import ( + LarkExploreConfig, + sync_issue_fix_explore_on_material_change, + sync_explore_results_to_lark, + sync_explore_visual_to_lark, + sync_explore_visuals_to_lark, +) +from loopx.extensions.lark.presentation.explore_stage_document import ( + ensure_stage_whiteboards, +) +from loopx.extensions.lark.presentation.kanban import ( + LarkKanbanConfig, + _public_safe_lark_values, + _public_safe_text, + build_record_upsert_command, + lark_kanban_heartbeat, + seed_lark_kanban_records, + seed_lark_kanban_task, + sync_loopx_projection_to_lark_kanban, + sync_loopx_todos_to_lark_kanban, +) +from loopx.extensions.lark.presentation.sink_visibility import ( + SINK_VISIBILITY_OWNER_ONLY, + SINK_VISIBILITY_SHARED, +) + + +PROVIDER_TOKENS = ( + pytest.param("gh" "p_" + "A" * 36, id="github"), + pytest.param("AK" "IA" + "A" * 16, id="aws"), + pytest.param("xox" "b-" + "a" * 24, id="slack"), + pytest.param("AI" "za" + "a" * 24, id="google"), + pytest.param("sk_" "live_" + "a" * 24, id="stripe"), + pytest.param("np" "m_" + "a" * 24, id="npm"), + pytest.param("py" "pi-" + "a" * 24, id="pypi"), + pytest.param("ey" "J" + "a" * 12 + "." + "b" * 12 + "." + "c" * 12, id="jwt"), + pytest.param("Bear" "er " + "a" * 24, id="bearer"), + pytest.param("AWS_SECRET_ACCESS_KEY=" + "a1B2" * 10, id="aws-secret-access-key"), +) + + +def _kanban_record_text(title: str, *, sink_visibility: str, tmp_path: Path) -> str: + packet = sync_loopx_projection_to_lark_kanban( + LarkKanbanConfig( + base_token="base_public_fixture", + table_id="tbl_public_fixture", + cli_bin="lark-cli", + identity="user", + ), + projection={ + "schema_version": "goal_channel_projection_v0", + "goal_id": "goal_public_fixture", + "source_id": "source-public-fixture", + "agent_todos": [ + { + "todo_id": "todo_public_fixture", + "title": title, + "status": "open", + "task_class": "advancement_task", + } + ], + }, + config_path=tmp_path / "lark-kanban.json", + sink_visibility=sink_visibility, + ) + return repr(packet["records"]) + + +def _explore_record_text(title: str, *, sink_visibility: str, tmp_path: Path) -> str: + packet = sync_explore_results_to_lark( + LarkExploreConfig( + base_token="base_public_fixture", + table_ids={"nodes": "tblN", "edges": "tblE", "findings": "tblF"}, + ), + projection={ + "schema_version": "loopx_explore_result_projection_v0", + "goal_id": "goal_public_fixture", + "nodes": [ + { + "node_id": "candidate", + "title": title, + "node_kind": "experiment", + "status": "resolved", + "parent_id": "", + "tags": [], + "summary": "fixture", + } + ], + "edges": [], + "findings": [], + }, + config_path=tmp_path / "lark-explore.json", + sink_visibility=sink_visibility, + ) + return repr(packet["records"]) + + +def _sync_todos_record_text(title: str, *, tmp_path: Path) -> str: + project = tmp_path / "project" + registry_path = project / ".loopx" / "registry.json" + state_file = project / "ACTIVE_GOAL_STATE.md" + registry_path.parent.mkdir(parents=True) + registry_path.write_text( + json.dumps( + { + "schema_version": "0.1", + "common_runtime_root": str(tmp_path / "runtime"), + "goals": [ + { + "id": "goal_public_fixture", + "repo": str(project), + "state_file": str(state_file), + } + ], + } + ), + encoding="utf-8", + ) + state_file.write_text( + f"""# Active Goal State + +## User Todo / Owner Review Reading Queue + +## Agent Todo + +- [ ] [P1] {title} + +""", + encoding="utf-8", + ) + packet = sync_loopx_todos_to_lark_kanban( + LarkKanbanConfig( + base_token="base_public_fixture", + table_id="tbl_public_fixture", + cli_bin="lark-cli", + identity="user", + ), + registry_path=registry_path, + goal_id="goal_public_fixture", + ) + return repr(packet["records"]) + + +def _heartbeat_fixture() -> dict[str, object]: + return { + "data": { + "fields": [ + "Status", + "Claim", + "Claimed By", + "Worker Command", + "Workdir", + "Run History", + "Evidence", + ], + "data": [["Todo", "Unclaimed", "", "echo fixture", ".", "", ""]], + "record_id_list": ["rec_public_fixture"], + } + } + + +def _private_key_block() -> tuple[str, str]: + body = "MIIE" + "syntheticPrivateKeyBody" * 4 + return ( + "-----BEGIN RSA PRIVATE KEY-----\n" + + body + + "\n-----END RSA PRIVATE KEY-----", + body, + ) + + +@pytest.mark.parametrize("sink", [pytest.param("kanban", id="kanban"), pytest.param("explore", id="explore")]) +@pytest.mark.parametrize("credential", PROVIDER_TOKENS) +def test_shared_sink_redacts_canonical_provider_credentials( + sink: str, + credential: str, + tmp_path: Path, +) -> None: + title = f"Rotate leaked token {credential}" + record_text = ( + _kanban_record_text(title, sink_visibility="shared", tmp_path=tmp_path) + if sink == "kanban" + else _explore_record_text(title, sink_visibility="shared", tmp_path=tmp_path) + ) + + assert credential not in record_text + assert "[credential-redacted]" in record_text + + +def test_shared_sink_keeps_existing_redactions() -> None: + text = " ".join( + ( + "file://" + "/Us" "ers/operator/run.log", + "https://" + "example.feishu.cn/base/private", + "base" + "abcdef123456", + "PRIVATE_" + "REF_FIXTURE", + ) + ) + + redacted = _public_safe_text(text) + + assert "[local-path-redacted]" in redacted + assert "[private-link-redacted]" in redacted + assert "[external-id-redacted]" in redacted + assert "[private-ref-redacted]" in redacted + + +@pytest.mark.parametrize( + "local_path", + [ + pytest.param(r"C:\Users\operator\secret.txt", id="windows-backslash"), + pytest.param("C:/Users/operator/secret.txt", id="windows-slash"), + pytest.param(r"\\server\share\secret.txt", id="unc"), + ], +) +def test_shared_sink_redacts_windows_and_unc_paths(local_path: str) -> None: + redacted = _public_safe_text(f"Read {local_path} before continuing") + + assert local_path not in redacted + assert "[local-path-redacted]" in redacted + + +@pytest.mark.parametrize( + "credential", + [ + pytest.param("GITHUB_TOKEN=" + "a1B2" * 10, id="github-env"), + pytest.param("CLIENT_SECRET=" + "a1B2" * 10, id="client-secret-env"), + pytest.param("AWS_SESSION_TOKEN=" + "a1B2" * 10, id="aws-session-env"), + pytest.param("DB_PASSWORD=" + "a1B2" * 10, id="database-password-env"), + pytest.param( + "postgres://operator:" + "a1B2" * 10 + "@db.example/app", + id="database-uri-userinfo", + ), + ], +) +def test_shared_sink_redacts_compound_credential_forms(credential: str) -> None: + redacted = _public_safe_text(f"Rotate {credential} before retrying") + + assert credential not in redacted + assert "[credential-redacted]" in redacted + + +def test_shared_sink_normalizes_terminal_controls_before_detection() -> None: + credential = "gh" "p_" + "A" * 36 + obfuscated = "gh" "p_" + "\x1b[31m" + "A" * 36 + "\x1b[0m" + + redacted = _public_safe_text(f"Rotate {obfuscated}") + + assert credential not in redacted + assert "\x1b" not in redacted + assert "[credential-redacted]" in redacted + + +def test_shared_sink_redacts_credentials_before_truncation() -> None: + credential = "gh" "p_" + "A" * 36 + text = "x" * 69 + " " + credential + + redacted = _public_safe_text(text, limit=80) + + assert "gh" "p_" not in redacted + + +def test_shared_sink_redacts_nested_text_without_breaking_record_links() -> None: + credential = "gh" "p_" + "A" * 36 + local_path = "file://" + "/Us" "ers/operator/run.log" + + values = _public_safe_lark_values( + { + "Evidence": [{"text": f"token {credential} in {local_path}"}], + "From Node Link": [{"id": "rec_public_fixture"}], + } + ) + serialized = json.dumps(values) + + assert credential not in serialized + assert local_path not in serialized + assert "[credential-redacted]" in serialized + assert "[local-path-redacted]" in serialized + assert values["From Node Link"] == [{"id": "rec_public_fixture"}] + + +def test_shared_sink_redacts_private_content_in_nested_id_fields() -> None: + local_path = "file://" + "/Us" "ers/operator/run.log" + private_link = "https://" + "example.feishu.cn/base/private" + private_ref = "PRIVATE_" + "REF_FIXTURE" + + values = _public_safe_lark_values( + { + "Evidence": [ + {"id": local_path}, + {"record_id": private_link}, + {"record_ids": [private_ref]}, + ] + } + ) + serialized = json.dumps(values) + + assert local_path not in serialized + assert private_link not in serialized + assert private_ref not in serialized + assert "[local-path-redacted]" in serialized + assert "[private-link-redacted]" in serialized + assert "[private-ref-redacted]" in serialized + + +def test_shared_sink_redacts_private_content_in_nested_dict_keys() -> None: + credential = "gh" "p_" + "A" * 36 + local_path = r"C:\Users\operator\secret.txt" + + values = _public_safe_lark_values( + { + "Evidence": [ + {f"token {credential}": "value"}, + {local_path: "value"}, + ] + } + ) + serialized = json.dumps(values) + + assert credential not in serialized + assert local_path not in serialized + assert "[credential-redacted]" in serialized + assert "[local-path-redacted]" in serialized + + +@pytest.mark.parametrize("sink_visibility", ["shared", "owner-only"]) +def test_stage_document_redacts_lane_labels_before_docs_update( + sink_visibility: str, + tmp_path: Path, +) -> None: + credential = "gh" "p_" + "A" * 36 + local_path = r"C:\Users\operator\secret.txt" + contents: list[str] = [] + created = False + + def runner( + args: list[str], cwd: Path | None, _timeout: float | None + ) -> dict[str, object]: + nonlocal created + if "docs" in args and "+update" in args: + contents.append(args[args.index("--content") + 1]) + created = True + return { + "returncode": 0, + "stdout": json.dumps( + { + "ok": True, + "data": { + "document": { + "new_blocks": [ + { + "block_id": "block_stage_02", + "block_type": "whiteboard", + "block_token": "wb_stage_02", + } + ] + } + }, + } + ), + "stderr": "", + } + assert "docs" in args and "+fetch" in args + heading = ( + '

Executive Evidence Stage 02

' + if created + else "" + ) + return { + "returncode": 0, + "stdout": json.dumps( + { + "ok": True, + "data": { + "document": { + "revision_id": 1, + "content": ( + '' + + heading + + "" + ), + } + }, + } + ), + "stderr": "", + } + + config_path = tmp_path / "lark-explore.json" + _, _, _, _, error = ensure_stage_whiteboards( + LarkExploreConfig(base_token="PUBLIC_FIXTURE_BASE"), + role="executive", + role_sink={ + "docx_token": "doc_public_fixture", + "stage_whiteboards": [ + {"stage_index": 1, "whiteboard_token": "wb_stage_01"} + ], + }, + stage_views=[ + { + "stage_index": 1, + "lanes": ["capability"], + "primary_node_count": 1, + "context_node_count": 0, + "cross_lane_edge_count": 0, + }, + { + "stage_index": 2, + "lanes": [credential, local_path, "R&D"], + "primary_node_count": 1, + "context_node_count": 0, + "cross_lane_edge_count": 0, + }, + ], + config_path=config_path, + sink_visibility=sink_visibility, + execute=True, + runner=runner, + read_local_config=lambda _path: { + "visual_sinks": {"executive": {}}, + }, + write_local_config=lambda _path, _payload: None, + ) + + assert error is None + assert len(contents) == 1 + content = contents[0] + assert credential not in content + assert "ghp" not in content.casefold() + assert "R&D" in content + if sink_visibility == "shared": + assert local_path not in content + assert "[local-path-redacted]" in content + else: + assert local_path.casefold() in content.casefold() + + +@pytest.mark.parametrize("sink", [pytest.param("kanban", id="kanban"), pytest.param("explore", id="explore")]) +@pytest.mark.parametrize("sink_visibility", ["owner-only", "shared"]) +def test_private_key_body_is_removed_from_record_sinks( + sink: str, + sink_visibility: str, + tmp_path: Path, +) -> None: + private_key, body = _private_key_block() + record_text = ( + _kanban_record_text( + f"Rotate key {private_key}", + sink_visibility=sink_visibility, + tmp_path=tmp_path, + ) + if sink == "kanban" + else _explore_record_text( + f"Rotate key {private_key}", + sink_visibility=sink_visibility, + tmp_path=tmp_path, + ) + ) + + assert body not in record_text + assert "[credential-redacted]" in record_text + + +@pytest.mark.parametrize("sink", [pytest.param("kanban", id="kanban"), pytest.param("explore", id="explore")]) +def test_owner_only_sink_still_redacts_credentials(sink: str, tmp_path: Path) -> None: + credential = "gh" "p_" + "A" * 36 + local_path = "file://" + "/Us" "ers/operator/run.log" + title = f"Owner-only {credential} at {local_path}" + record_text = ( + _kanban_record_text(title, sink_visibility="owner-only", tmp_path=tmp_path) + if sink == "kanban" + else _explore_record_text(title, sink_visibility="owner-only", tmp_path=tmp_path) + ) + + assert credential not in record_text + assert "[credential-redacted]" in record_text + assert local_path in record_text + + +def test_sync_loopx_todos_redacts_credentials_and_private_paths(tmp_path: Path) -> None: + credential = "gh" "p_" + "A" * 36 + local_path = "file://" + "/Us" "ers/operator/run.log" + + record_text = _sync_todos_record_text( + f"Rotate {credential} from {local_path}", + tmp_path=tmp_path, + ) + + assert credential not in record_text + assert local_path not in record_text + assert "[credential-redacted]" in record_text + assert "[local-path-redacted]" in record_text + + +def test_heartbeat_writeback_redacts_worker_output(tmp_path: Path) -> None: + credential = "gh" "p_" + "A" * 36 + + def runner( + args: list[str], + cwd: Path | None, + timeout: float | None, + ) -> dict[str, object]: + assert args == ["echo", "fixture"] + return { + "returncode": 1, + "stdout": f"worker token {credential}", + "stderr": f"AWS_SECRET_ACCESS_KEY={credential}", + "timed_out": False, + } + + packet = lark_kanban_heartbeat( + LarkKanbanConfig( + base_token="base_public_fixture", + table_id="tbl_public_fixture", + cli_bin="lark-cli", + identity="user", + ), + fixture=_heartbeat_fixture(), + worker_command="echo fixture", + execute_worker=True, + allowed_command_prefixes=["echo"], + runner=runner, + ) + + assert credential not in repr(packet["writeback"]) + assert credential not in repr(packet["worker"]) + assert credential not in repr(packet["commands"]) + assert "[credential-redacted]" in packet["writeback"]["Evidence"] + assert "[credential-redacted]" in packet["writeback"]["Last Error"] + + +def test_heartbeat_redacts_private_key_body_from_worker_outputs(tmp_path: Path) -> None: + private_key, body = _private_key_block() + + def runner( + args: list[str], + cwd: Path | None, + timeout: float | None, + ) -> dict[str, object]: + assert args == ["echo", "fixture"] + return { + "returncode": 1, + "stdout": f"worker key {private_key}", + "stderr": f"failed with {private_key}", + "timed_out": False, + } + + packet = lark_kanban_heartbeat( + LarkKanbanConfig( + base_token="base_public_fixture", + table_id="tbl_public_fixture", + cli_bin="lark-cli", + identity="user", + ), + fixture=_heartbeat_fixture(), + worker_command="echo fixture", + execute_worker=True, + allowed_command_prefixes=["echo"], + runner=runner, + ) + + assert body not in repr(packet["writeback"]) + assert body not in repr(packet["worker"]) + assert body not in repr(packet["commands"]) + assert "[credential-redacted]" in packet["writeback"]["Evidence"] + assert "[credential-redacted]" in packet["writeback"]["Last Error"] + + +def test_shared_visual_sink_redacts_the_rendered_source(tmp_path: Path) -> None: + credential = "gh" "p_" + "A" * 36 + local_path = "file://" + "/Us" "ers/operator/run.log" + private_link = "https://" + "example.feishu.cn/base/private" + published_source: list[str] = [] + published_marker = "" + + def runner(args: list[str], cwd: Path | None, _timeout: float | None) -> dict[str, object]: + nonlocal published_marker + if "+update" in args: + source_arg = args[args.index("--source") + 1] + assert cwd is not None + source = (cwd / source_arg.removeprefix("@")).read_text(encoding="utf-8") + published_source.append(source) + marker = re.search(r"LoopX delivery [0-9a-f]{20}", source) + assert marker is not None + published_marker = marker.group(0) + return {"returncode": 0, "stdout": json.dumps({"ok": True}), "stderr": ""} + assert "+query" in args + return { + "returncode": 0, + "stdout": json.dumps( + { + "ok": True, + "data": {"nodes": [{"text": {"text": published_marker}}]}, + } + ), + "stderr": "", + } + + result = sync_explore_visual_to_lark( + LarkExploreConfig(base_token="PUBLIC_FIXTURE_BASE"), + projection={ + "goal_id": "goal_visual_fixture", + "nodes": [], + "edges": [], + "findings": [], + }, + visual_sink={ + "whiteboard_token": "wb_visual_fixture", + "view_role": "canonical", + "renderer": "mermaid", + }, + config_path=tmp_path / "lark-explore.json", + semantic_digest="synthetic-visual-digest", + display_projection={ + "view_role": "canonical", + "mermaid": ( + 'flowchart TD\n node_fixture["Credential ' + f"{credential} at {local_path} via {private_link}" + '"]\n' + ), + }, + sink_visibility="shared", + execute=True, + runner=runner, + ) + + assert result["ok"] is True + assert result["sink_visibility"] == "shared" + assert len(published_source) == 1 + assert credential not in published_source[0] + assert local_path not in published_source[0] + assert private_link not in published_source[0] + assert "[credential-redacted]" in published_source[0] + assert "[local-path-redacted]" in published_source[0] + assert "[private-link-redacted]" in published_source[0] + + +def test_shared_stage_svg_redacts_display_values_before_encoding(tmp_path: Path) -> None: + credential = "gh" "p_" + "A" * 36 + private_link = "https://" + "example.feishu.cn/base/private" + published_sources: list[str] = [] + published_marker = "" + + def runner(args: list[str], cwd: Path | None, _timeout: float | None) -> dict[str, object]: + nonlocal published_marker + if "+update" in args: + source_arg = args[args.index("--source") + 1] + assert cwd is not None + source = (cwd / source_arg.removeprefix("@")).read_text(encoding="utf-8") + published_sources.append(source) + marker = re.search(r"LoopX delivery [0-9a-f]{20}", source) + assert marker is not None + published_marker = marker.group(0) + return {"returncode": 0, "stdout": json.dumps({"ok": True}), "stderr": ""} + assert "+query" in args + return { + "returncode": 0, + "stdout": json.dumps( + { + "ok": True, + "data": {"nodes": [{"text": {"text": published_marker}}]}, + } + ), + "stderr": "", + } + + projection = { + "goal_id": "goal_public_fixture", + "nodes": [ + { + "node_id": "stage_node_one", + "title": "Confirm the failing stage", + "node_kind": "decision", + "status": "exploring", + "summary": ( + "Measured 12ms improvement from the shared board. " + f"See {private_link}" + ), + "parent_id": "", + "tags": ["lane-fix_pr"], + }, + { + "node_id": "stage_node_two", + "title": "Apply the fix", + "node_kind": "decision", + "status": "resolved", + "summary": f"Patch verified with {credential}.", + "parent_id": "stage_node_one", + "tags": ["lane-fix_pr"], + }, + ], + "edges": [], + "findings": [], + } + + result = sync_explore_visuals_to_lark( + LarkExploreConfig(base_token="PUBLIC_FIXTURE_BASE"), + projection=projection, + visual_sinks={ + "canonical": { + "board_style": "semantic_lane_columns", + "stage_capacity": 14, + "stage_whiteboards": [ + {"stage_index": 1, "whiteboard_token": "wb_public_stage_fixture"} + ], + } + }, + config_path=tmp_path / "lark-explore.json", + sink_visibility=SINK_VISIBILITY_SHARED, + execute=True, + runner=runner, + ) + + assert result["ok"] is True + assert len(published_sources) == 1 + source = published_sources[0] + parsed = ET.fromstring(source) + assert parsed.tag.endswith("svg") + assert credential not in source + assert private_link not in source + assert "[credential-redacted]" in source + assert "[private-link-redacted]" in source + + +def test_record_upsert_command_redacts_nested_credentials() -> None: + credential = "gh" "p_" + "A" * 36 + + command = build_record_upsert_command( + LarkKanbanConfig( + base_token="base_public_fixture", + table_id="tbl_public_fixture", + cli_bin="lark-cli", + identity="user", + ), + record_id=None, + values={"Evidence": [{"text": f"token {credential}"}]}, + ) + + assert credential not in " ".join(command) + assert "[credential-redacted]" in " ".join(command) + + +def test_seed_writers_default_to_shared_and_redact_nested_values() -> None: + credential = "gh" "p_" + "A" * 36 + local_path = r"C:\Users\operator\private" + config = LarkKanbanConfig( + base_token="base_public_fixture", + table_id="tbl_public_fixture", + cli_bin="lark-cli", + identity="user", + ) + task = { + "Task": f"Rotate {credential}", + "Workdir": local_path, + "Worker Command": f"echo {credential}", + } + + for sink in (seed_lark_kanban_task, seed_lark_kanban_records): + assert ( + signature(sink).parameters["sink_visibility"].default + == SINK_VISIBILITY_SHARED + ) + + task_packet = seed_lark_kanban_task(config, task=task) + records_packet = seed_lark_kanban_records(config, records=[task]) + for packet in (task_packet, records_packet): + serialized = json.dumps(packet) + assert credential not in serialized + assert local_path not in serialized + assert "[credential-redacted]" in serialized + assert "[local-path-redacted]" in serialized + + owner_packet = seed_lark_kanban_task( + config, + task=task, + sink_visibility=SINK_VISIBILITY_OWNER_ONLY, + ) + assert owner_packet["task"]["Workdir"] == local_path + assert credential not in json.dumps(owner_packet) + + +def test_operator_and_automatic_sink_paths_default_to_shared() -> None: + for sink in ( + sync_loopx_todos_to_lark_kanban, + lark_kanban_heartbeat, + sync_issue_fix_explore_on_material_change, + ): + assert ( + signature(sink).parameters["sink_visibility"].default + == SINK_VISIBILITY_SHARED + ) + + +def test_explicit_visual_sink_paths_stay_owner_only_by_default() -> None: + for sink in (sync_explore_visual_to_lark, sync_explore_visuals_to_lark): + assert ( + signature(sink).parameters["sink_visibility"].default + == SINK_VISIBILITY_OWNER_ONLY + )