diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..8797c2a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,65 @@ +name: Publish to PyPI + +# Triggers on a GitHub Release creation (v* tag). To publish: +# 1. Create a release on GitHub with a tag like "v0.1.2". +# 2. This workflow builds the sdist + wheel and uploads them to PyPI using +# the trusted-publisher (OIDC) mechanism — no API token stored in secrets. +# +# Prerequisites (one-time setup): +# - Add this repository as a trusted publisher on https://pypi.org/manage/ +# project/kairos/settings/publishing/ with: +# Publisher: GitHub Actions +# Owner: +# Repository: +# Workflow name: publish.yml +# Environment name: pypi + +on: + release: + types: [published] + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build tools + run: pip install build + + - name: Build sdist and wheel + run: python -m build + + - name: Upload distribution artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/kairos/ + permissions: + contents: read + id-token: write # required for OIDC trusted-publisher upload + + steps: + - name: Download distribution artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.md b/README.md index 74ccd4d..560f1aa 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,57 @@ Creates a temporary workspace, ingests all parser fixture types (Markdown, JSON, --- +## A real-world aha moment + +> *"Our firmware review process involves a Kconfig tree, a changelog, release notes PDF, and Python tooling — all cross-referencing each other. I needed to answer: what docs and code are affected by `CONFIG_WIFI_POWER_SAVE`?"* + +With KAIROS, that's five commands: + +```bash +# 1. Create a workspace inside the firmware repo +kairos init . + +# 2. Ingest everything at once — Kconfig JSON, Markdown docs, PDF release notes, Python tooling +kairos ingest --recursive . + +# 3. Search confirms the symbol is indexed with its exact location +$ kairos search CONFIG_WIFI_POWER_SAVE + + [1] kconfig:Main/Networking/CONFIG_WIFI_POWER_SAVE + firmware/menuconfig.json (parser: kairos.kconfig v1.0.0 · extracted) + "Enable power-save mode for the Wi-Fi driver" + + [2] lines:42-42 + docs/changelog.md (parser: kairos.markdown v1.0.0 · extracted) + "- Disabled CONFIG_WIFI_POWER_SAVE by default (see PR #881)" + + [3] page:7 + release_notes.pdf (parser: kairos.pdf v1.0.0 · extracted) + "Power-save mode is off by default in v2.4.1" + +# 4. Trace the symbol across ALL documents — two hops, deterministically +$ kairos trace CONFIG_WIFI_POWER_SAVE --depth 3 + + entity CONFIG_WIFI_POWER_SAVE (kconfig_symbol) + │ + ├─ mentioned in ──► [span] kconfig:Main/Networking/CONFIG_WIFI_POWER_SAVE + │ firmware/menuconfig.json + │ + ├─ depends_on ◄──── [entity] CONFIG_WIFI (kconfig_symbol) + │ firmware/menuconfig.json + │ + └─ heading_contains ──► [span] lines:40-50 + docs/changelog.md ← crossed document boundary + "## v2.4.1 Changes" + +# 5. Add a note so the finding is preserved in the workspace +kairos note "Confirmed: power-save off by default since v2.4.1 (PR #881)" +``` + +**What just happened:** KAIROS walked from a Kconfig symbol entity → to its mention in the firmware JSON → through a `heading_contains` relation → into a completely different Markdown file, in three hops, with zero embeddings and zero guessing. Every step shows you the exact artifact, locator, and relation rule that got it there. You can re-run it six months later on a new checkout and get the same answer, or a provably different one. + +--- + ## How it's built - **Storage**: SQLite as the canonical store (9 tables), plus an FTS5 virtual table with sync triggers — no separate search service, no vector database. diff --git a/src/kairos/infrastructure/parsers/json_parser.py b/src/kairos/infrastructure/parsers/json_parser.py index 3a83b42..4079ba5 100644 --- a/src/kairos/infrastructure/parsers/json_parser.py +++ b/src/kairos/infrastructure/parsers/json_parser.py @@ -64,7 +64,15 @@ def parse(self, path: Path, artifact_id: str) -> ParseResult: ordinal_counter = [0] - def visit(value: JsonValue, json_path: str, parent_span_id: str | None) -> str: + # Iterative DFS traversal — avoids Python's recursion limit on + # deeply nested JSON documents (e.g. 1 000+ levels). Each stack + # frame is (value, json_path, parent_span_id); the span_id for the + # current node is allocated before its children so containment + # relations can reference it immediately. + stack: list[tuple[JsonValue, str, str | None]] = [(document, "$", None)] + while stack: + value, json_path, parent_span_id = stack.pop() + span_id = new_id() ordinal = ordinal_counter[0] ordinal_counter[0] += 1 @@ -89,45 +97,29 @@ def visit(value: JsonValue, json_path: str, parent_span_id: str | None) -> str: ) ) - if isinstance(value, dict): - for key, child in value.items(): - child_path = f"{json_path}.{key}" - child_span_id = visit(child, child_path, span_id) - result.relations.append( - Relation( - id=new_id(), - subject_id=span_id, - subject_kind="span", - predicate=RelationPredicate.JSON_CONTAINS.value, - object_id=child_span_id, - object_kind="span", - evidence_span_id=span_id, - origin=Origin.DERIVED, - derivation_rule="json.tree_containment.v1", - confidence=1.0, - ) - ) - elif isinstance(value, list): - for i, child in enumerate(value): - child_path = f"{json_path}[{i}]" - child_span_id = visit(child, child_path, span_id) - result.relations.append( - Relation( - id=new_id(), - subject_id=span_id, - subject_kind="span", - predicate=RelationPredicate.JSON_CONTAINS.value, - object_id=child_span_id, - object_kind="span", - evidence_span_id=span_id, - origin=Origin.DERIVED, - derivation_rule="json.tree_containment.v1", - confidence=1.0, - ) + if parent_span_id is not None: + result.relations.append( + Relation( + id=new_id(), + subject_id=parent_span_id, + subject_kind="span", + predicate=RelationPredicate.JSON_CONTAINS.value, + object_id=span_id, + object_kind="span", + evidence_span_id=parent_span_id, + origin=Origin.DERIVED, + derivation_rule="json.tree_containment.v1", + confidence=1.0, ) + ) - return span_id - - visit(document, "$", None) + if isinstance(value, dict): + # Push children in reverse order so left-to-right ordinals come + # out naturally when the stack unwinds. + for key in reversed(value): + stack.append((value[key], f"{json_path}.{key}", span_id)) + elif isinstance(value, list): + for i in reversed(range(len(value))): + stack.append((value[i], f"{json_path}[{i}]", span_id)) result.parse_status = ParseStatus.OK return result diff --git a/src/kairos/infrastructure/parsers/kconfig.py b/src/kairos/infrastructure/parsers/kconfig.py index 808bfbe..bac8c71 100644 --- a/src/kairos/infrastructure/parsers/kconfig.py +++ b/src/kairos/infrastructure/parsers/kconfig.py @@ -85,7 +85,13 @@ def parse(self, path: Path, artifact_id: str) -> ParseResult: symbol_entity_ids: dict[str, str] = {} pending_depends: list[tuple[str, str]] = [] # (subject_entity_id, depends_on_text) - def visit(node: dict[str, JsonValue], menu_path: str, parent_span_id: str | None) -> None: + # Iterative traversal — avoids Python's recursion limit on deeply + # nested Kconfig trees (e.g. 1 000+ levels). Each stack frame is + # (node, menu_path, parent_span_id). + stack: list[tuple[dict[str, JsonValue], str, str | None]] = [(document, "", None)] + while stack: + node, menu_path, parent_span_id = stack.pop() + node_type = node.get("node_type", "menu") name = str(node.get("name", "?")) node_path = f"{menu_path}/{name}" if menu_path else name @@ -162,9 +168,11 @@ def visit(node: dict[str, JsonValue], menu_path: str, parent_span_id: str | None children = node.get("children", []) if isinstance(children, list): - for child in children: + # Push children in reverse order so they are processed + # left-to-right (stack pops from the right). + for child in reversed(children): if isinstance(child, dict): - visit(child, node_path, span_id) + stack.append((child, node_path, span_id)) else: # A non-object entry in a children array — never # silently skipped, per the project's founding @@ -179,8 +187,6 @@ def visit(node: dict[str, JsonValue], menu_path: str, parent_span_id: str | None ) ) - visit(document, "", None) - for subject_entity_id, depends_expr in pending_depends: tokens = [t.strip() for t in depends_expr.split("&&")] if all(_SIMPLE_IDENTIFIER_RE.match(t) for t in tokens): diff --git a/src/kairos/services/events.py b/src/kairos/services/events.py index 4651f26..f08b510 100644 --- a/src/kairos/services/events.py +++ b/src/kairos/services/events.py @@ -8,6 +8,7 @@ import json from datetime import UTC, datetime +from sqlalchemy import event from sqlalchemy.orm import Session from kairos.domain.ids import new_id @@ -35,13 +36,30 @@ def append_event( ) session.flush() - line = { - "id": event_id, - "occurred_at": occurred_at.isoformat(), - "event_type": event_type, - "payload": payload, - } - with workspace.events_path.open("a", encoding="utf-8") as f: - f.write(json.dumps(line, default=str) + "\n") + # The JSONL line is written *only after* the DB commit has succeeded. + # Callers that use session_scope must therefore call append_event and then + # let session_scope commit; we register a post-commit hook via SQLAlchemy's + # after_commit event so that the file write is never attempted if the + # transaction is rolled back. + line = json.dumps( + { + "id": event_id, + "occurred_at": occurred_at.isoformat(), + "event_type": event_type, + "payload": payload, + }, + default=str, + ) + events_path = workspace.events_path + + @event.listens_for(session, "after_commit", once=True) + def _write_jsonl(_session: Session) -> None: # pyright: ignore[reportUnusedFunction] + try: + with events_path.open("a", encoding="utf-8") as f: + f.write(line + "\n") + except OSError: + # The DB commit already succeeded — the event is durable in SQLite. + # A best-effort JSONL mirror failure should not crash the caller. + pass return event_id diff --git a/tests/unit/test_parsers_json.py b/tests/unit/test_parsers_json.py index d71c51d..af9016c 100644 --- a/tests/unit/test_parsers_json.py +++ b/tests/unit/test_parsers_json.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from kairos.domain.enums import ParseStatus, SpanKind @@ -43,3 +44,25 @@ def test_malformed_json_is_not_silently_dropped(tmp_path: Path) -> None: assert result.diagnostics assert len(result.spans) == 1 assert '"a": 1' in result.spans[0].text_content + + +def test_json_deeply_nested_does_not_raise(tmp_path: Path) -> None: + """A JSON document with >1000 levels of nesting must parse without + RecursionError — the iterative traversal replaces the former recursive + visit() closure that hit Python's default stack limit at ~950 levels.""" + depth = 1100 + # Build {"a": {"a": {"a": ... }}} depth levels deep + doc: object = "leaf" + for _ in range(depth): + doc = {"a": doc} + path = tmp_path / "deep.json" + path.write_text(json.dumps(doc), encoding="utf-8") + + parser = JsonParser() + result = parser.parse(path, "artifact-deep") + + assert result.parse_status == ParseStatus.OK + # 1100 nested dict containers + 1 scalar leaf = depth + 1 spans + # 1100 json_contains relations (one per container-to-child edge) + assert len(result.spans) == depth + 1 + assert len(result.relations) == depth diff --git a/tests/unit/test_parsers_kconfig.py b/tests/unit/test_parsers_kconfig.py index a748887..bb8da5c 100644 --- a/tests/unit/test_parsers_kconfig.py +++ b/tests/unit/test_parsers_kconfig.py @@ -116,3 +116,26 @@ def test_non_dict_child_is_diagnosed_not_dropped(tmp_path: Path) -> None: assert any("not-a-node" in d.message for d in result.diagnostics) # the well-formed sibling is still parsed, not dropped along with its bad sibling assert any(e.canonical_name == "CONFIG_A" for e in result.entities) + + +def test_kconfig_deeply_nested_does_not_raise(tmp_path: Path) -> None: + """A Kconfig tree with >1000 levels of nesting must parse without + RecursionError — the iterative traversal replaces the former recursive + visit() closure that hit Python's default stack limit at ~950 levels.""" + depth = 1100 + # Build a chain of menus: {"kairos_kind": ..., "name": "L0", + # "children": [{"name": "L1", "children": [...]}]} + node: dict[str, object] = {"name": f"L{depth}", "children": []} + for i in range(depth - 1, -1, -1): + node = {"name": f"L{i}", "children": [node]} + doc: dict[str, object] = { + "kairos_kind": "kconfig_menu", + **node, + } + path = tmp_path / "deep_kconfig.json" + path.write_text(json.dumps(doc), encoding="utf-8") + + result = KconfigParser().parse(path, "artifact-deep-kconfig") + + assert result.parse_status == ParseStatus.OK + assert len(result.spans) == depth + 1 # root + depth children