From 8970e0e7a090b39381bf9ed621dcb059897abb8f Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 12:57:10 +0200 Subject: [PATCH 01/25] feat: extensible format and provider system with YAML and local ref support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces two independent plugin-style registries: - formats/ — Format + LineScanner protocols, with MarkdownFormat (code-fence aware), YamlFormat (# comments, block-scalar aware), and TextFormat fallback - providers/ — Provider + ProviderRegistry with GitHubProvider (was GitHubClient) and LocalProvider for local:path refs with no release dependency Adds a self-hosted update-fragments.yml workflow that installs embedder from the local repo (pip install -e .) and runs at 10:00 Berlin time (CEST). Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/update-fragments-shared.yml | 2 +- .github/workflows/update-fragments.yml | 55 ++++++++++++ src/embedder/blocks.py | 46 ++++------ src/embedder/cli.py | 34 +++---- src/embedder/formats/__init__.py | 21 +++++ src/embedder/formats/base.py | 29 ++++++ src/embedder/formats/markdown.py | 33 +++++++ src/embedder/formats/text.py | 35 ++++++++ src/embedder/formats/yaml.py | 59 +++++++++++++ src/embedder/providers/__init__.py | 29 ++++++ src/embedder/providers/base.py | 26 ++++++ src/embedder/{ => providers}/github.py | 23 ++++- src/embedder/providers/local.py | 38 ++++++++ src/embedder/refs.py | 2 +- src/embedder/updater.py | 44 ++++++---- tests/helpers.py | 8 ++ tests/test_blocks.py | 53 ++++++++++- tests/test_cli.py | 2 +- tests/test_refs.py | 37 ++++++-- tests/test_updater.py | 88 +++++++++++++++---- 20 files changed, 566 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/update-fragments.yml create mode 100644 src/embedder/formats/__init__.py create mode 100644 src/embedder/formats/base.py create mode 100644 src/embedder/formats/markdown.py create mode 100644 src/embedder/formats/text.py create mode 100644 src/embedder/formats/yaml.py create mode 100644 src/embedder/providers/__init__.py create mode 100644 src/embedder/providers/base.py rename src/embedder/{ => providers}/github.py (77%) create mode 100644 src/embedder/providers/local.py diff --git a/.github/workflows/update-fragments-shared.yml b/.github/workflows/update-fragments-shared.yml index 145d537..7a133bd 100644 --- a/.github/workflows/update-fragments-shared.yml +++ b/.github/workflows/update-fragments-shared.yml @@ -54,7 +54,7 @@ jobs: body=$(jq -r ' "## Embedded fragment updates\n\n| File | Source | Asset | Old | New |\n|---|---|---|---:|---:|\n" + ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(.block.ref.owner)/\(.block.ref.repo) | \(.block.ref.asset) | \(.block.ref.tag) | \(.latest_tag) |" + "| \(.block.path) | \(.block.ref.owner)/\(.block.ref.repo) | \(.block.ref.asset) | \(.block.ref.tag) | \(.latest_ref.tag) |" ] | join("\n")) ' /tmp/embedder-result.json) existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') diff --git a/.github/workflows/update-fragments.yml b/.github/workflows/update-fragments.yml new file mode 100644 index 0000000..ff4d3ac --- /dev/null +++ b/.github/workflows/update-fragments.yml @@ -0,0 +1,55 @@ +name: Update embedded fragments +on: + schedule: + - cron: '0 8 * * *' # 10:00 Berlin (CEST, UTC+2) + workflow_dispatch: +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install embedder + run: pip install -e . + - name: Run embedder update + env: + GH_TOKEN: ${{ github.token }} + run: embedder update . --json > /tmp/embedder-result.json + - name: Detect changes + id: diff + run: | + if git diff --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Open pull request + if: steps.diff.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + branch="embedder/update-fragments" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$branch" + git add -A + git commit -m "chore: update embedded fragments" + git push origin "$branch" --force + body=$(jq -r ' + "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + + ([.changed[] | .changed_blocks[] | + "| \(.block.path) | \(.block.ref.render // .block.ref.path // (.block.ref.owner + "/" + .block.ref.repo + ":" + .block.ref.asset)) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" + ] | join("\n")) + ' /tmp/embedder-result.json) + existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') + if [ -z "$existing" ]; then + gh pr create \ + --title "chore: update embedded fragments" \ + --body "$body" \ + --head "$branch" + fi diff --git a/src/embedder/blocks.py b/src/embedder/blocks.py index 943d441..48d88b5 100644 --- a/src/embedder/blocks.py +++ b/src/embedder/blocks.py @@ -1,15 +1,11 @@ from __future__ import annotations import os -import re from dataclasses import dataclass from pathlib import Path -from embedder.refs import GitHubAssetRef, RefError, parse_ref +from embedder.refs import RefError -OPEN_MARKER_RE = re.compile(r"^\s*\s*$") -CLOSE_MARKER_RE = re.compile(r"^\s*\s*$") -FENCE_RE = re.compile(r"^\s*(?P`{3,}|~{3,})") SKIP_DIRS = { ".git", ".hg", @@ -37,41 +33,34 @@ class EmbedderBlock: path: str start_line: int end_line: int - ref: GitHubAssetRef + ref: object # AnyRef — typed as object to avoid circular import body: str @dataclass(frozen=True) class BlockUpdate: block: EmbedderBlock - new_ref: GitHubAssetRef + new_ref: object # AnyRef new_body: str def parse_blocks(path: Path, text: str) -> list[EmbedderBlock]: + from embedder.formats import get_format + from embedder.providers import parse_ref + + fmt = get_format(path) + scanner = fmt.make_scanner() lines = text.splitlines(keepends=True) blocks: list[EmbedderBlock] = [] active_start: int | None = None - active_ref: GitHubAssetRef | None = None + active_ref = None body_start = 0 - active_fence: str | None = None - markdown = path.suffix.lower() in {".md", ".markdown"} for index, line in enumerate(lines): - if markdown and active_start is None: - fence_match = FENCE_RE.match(line) - if fence_match: - fence = fence_match["fence"] - marker = fence[0] - if active_fence is None: - active_fence = marker - elif active_fence == marker: - active_fence = None - continue - if active_fence is not None: - continue - - open_match = OPEN_MARKER_RE.match(line) + if active_start is None and scanner.advance(line): + continue + + open_match = fmt.open_re.match(line) if open_match: if active_start is not None: raise EmbedderError( @@ -85,7 +74,7 @@ def parse_blocks(path: Path, text: str) -> list[EmbedderBlock]: body_start = index + 1 continue - if CLOSE_MARKER_RE.match(line): + if fmt.close_re.match(line): if active_start is None or active_ref is None: raise EmbedderError( f"{path}:{index + 1}: closing embedder marker without opening marker" @@ -147,10 +136,13 @@ def scan_paths(paths: list[Path]) -> list[EmbedderBlock]: return blocks -def apply_updates(text: str, updates: list[BlockUpdate]) -> str: +def apply_updates(path: Path, text: str, updates: list[BlockUpdate]) -> str: + from embedder.formats import get_format + if not updates: return text + fmt = get_format(path) lines = text.splitlines(keepends=True) output: list[str] = [] cursor = 0 @@ -169,7 +161,7 @@ def apply_updates(text: str, updates: list[BlockUpdate]) -> str: replacement_body += "\n" output.extend(lines[cursor:start]) - output.append(f"{indent}{newline}") + output.append(f"{indent}{fmt.render_open(update.new_ref.render())}{newline}") output.append(replacement_body) output.append(lines[end]) cursor = end + 1 diff --git a/src/embedder/cli.py b/src/embedder/cli.py index 5e26dc2..6fc4342 100644 --- a/src/embedder/cli.py +++ b/src/embedder/cli.py @@ -3,7 +3,7 @@ import argparse import json import sys -from dataclasses import asdict +from dataclasses import asdict, fields, is_dataclass from enum import IntEnum from pathlib import Path from typing import Any @@ -15,7 +15,7 @@ EmbedderError, scan_paths, ) -from embedder.github import GitHubClient +from embedder.providers.github import GitHubProvider from embedder.updater import check_blocks, update_files @@ -28,8 +28,8 @@ class ExitCode(IntEnum): def to_json(data: Any) -> None: def default(value: Any) -> Any: - if hasattr(value, "__dataclass_fields__"): - return asdict(value) + if is_dataclass(value) and not isinstance(value, type): + return {f.name: getattr(value, f.name) for f in fields(value)} if isinstance(value, Path): return str(value) raise TypeError(f"Object is not JSON serializable: {value!r}") @@ -48,26 +48,18 @@ def command_scan(args: argparse.Namespace) -> int: return int(ExitCode.OK) for block in blocks: - ref = block.ref - print( - f"{block.path}:{block.start_line} " - f"{ref.owner}/{ref.repo}@{ref.tag}:{ref.asset}" - ) + print(f"{block.path}:{block.start_line} {block.ref.render()}") return int(ExitCode.OK) -def print_block(block: EmbedderBlock, *, latest_tag: str | None = None) -> None: - ref = block.ref - suffix = "" if latest_tag is None else f" -> {latest_tag}" - print( - f"{block.path}:{block.start_line} " - f"{ref.owner}/{ref.repo}@{ref.tag}:{ref.asset}{suffix}" - ) +def _print_block(block: EmbedderBlock, *, latest_ref: Any = None) -> None: + suffix = "" if latest_ref is None else f" -> {latest_ref.render()}" + print(f"{block.path}:{block.start_line} {block.ref.render()}{suffix}") def command_check(args: argparse.Namespace) -> int: blocks = scan_paths([Path(path) for path in args.paths]) - results = check_blocks(blocks, GitHubClient()) + results = check_blocks(blocks) updates = [result for result in results if result.update_available] if args.json: @@ -82,12 +74,12 @@ def command_check(args: argparse.Namespace) -> int: return int(ExitCode.OK) for result in updates: - print_block(result.block, latest_tag=result.latest_tag) + _print_block(result.block, latest_ref=result.latest_ref) return int(ExitCode.UPDATES_AVAILABLE) def command_update(args: argparse.Namespace) -> int: - changed = update_files([Path(path) for path in args.paths], GitHubClient()) + changed = update_files([Path(path) for path in args.paths]) if args.json: to_json({"changed": changed}) return int(ExitCode.OK) @@ -96,12 +88,12 @@ def command_update(args: argparse.Namespace) -> int: return int(ExitCode.OK) for file_update in changed: for result in file_update.changed_blocks: - print_block(result.block, latest_tag=result.latest_tag) + _print_block(result.block, latest_ref=result.latest_ref) return int(ExitCode.OK) def command_doctor(args: argparse.Namespace) -> int: - github = GitHubClient() + github = GitHubProvider() gh_available = github.available() checks = { "gh": "ok" if gh_available else "missing", diff --git a/src/embedder/formats/__init__.py b/src/embedder/formats/__init__.py new file mode 100644 index 0000000..0011774 --- /dev/null +++ b/src/embedder/formats/__init__.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from pathlib import Path + +from embedder.formats.base import Format +from embedder.formats.markdown import MarkdownFormat +from embedder.formats.text import TextFormat +from embedder.formats.yaml import YamlFormat + +_FORMATS: list[Format] = [ + MarkdownFormat(), + YamlFormat(), + TextFormat(), +] + + +def get_format(path: Path) -> Format: + for fmt in _FORMATS: + if fmt.matches(path): + return fmt + return TextFormat() diff --git a/src/embedder/formats/base.py b/src/embedder/formats/base.py new file mode 100644 index 0000000..6941806 --- /dev/null +++ b/src/embedder/formats/base.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class LineScanner(Protocol): + def advance(self, line: str) -> bool: + """Advance by one line. Returns True if the line should be skipped.""" + ... + + +@runtime_checkable +class Format(Protocol): + @property + def open_re(self) -> re.Pattern[str]: ... + + @property + def close_re(self) -> re.Pattern[str]: ... + + def matches(self, path: Path) -> bool: ... + + def make_scanner(self) -> LineScanner: ... + + def render_open(self, ref_str: str) -> str: ... + + def render_close(self) -> str: ... diff --git a/src/embedder/formats/markdown.py b/src/embedder/formats/markdown.py new file mode 100644 index 0000000..aa056eb --- /dev/null +++ b/src/embedder/formats/markdown.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import re +from pathlib import Path + +from embedder.formats.text import TextFormat + + +FENCE_RE = re.compile(r"^\s*(?P`{3,}|~{3,})") + + +class _MarkdownScanner: + def __init__(self) -> None: + self._fence: str | None = None + + def advance(self, line: str) -> bool: + m = FENCE_RE.match(line) + if m: + marker = m["fence"][0] + if self._fence is None: + self._fence = marker + elif self._fence == marker: + self._fence = None + return True + return self._fence is not None + + +class MarkdownFormat(TextFormat): + def matches(self, path: Path) -> bool: + return path.suffix.lower() in {".md", ".markdown"} + + def make_scanner(self) -> _MarkdownScanner: + return _MarkdownScanner() diff --git a/src/embedder/formats/text.py b/src/embedder/formats/text.py new file mode 100644 index 0000000..c8b511e --- /dev/null +++ b/src/embedder/formats/text.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import re +from pathlib import Path + + +OPEN_RE = re.compile(r"^\s*\s*$") +CLOSE_RE = re.compile(r"^\s*\s*$") + + +class _NoopScanner: + def advance(self, line: str) -> bool: + return False + + +class TextFormat: + @property + def open_re(self) -> re.Pattern[str]: + return OPEN_RE + + @property + def close_re(self) -> re.Pattern[str]: + return CLOSE_RE + + def matches(self, path: Path) -> bool: + return True + + def make_scanner(self) -> _NoopScanner: + return _NoopScanner() + + def render_open(self, ref_str: str) -> str: + return f"" + + def render_close(self) -> str: + return "" diff --git a/src/embedder/formats/yaml.py b/src/embedder/formats/yaml.py new file mode 100644 index 0000000..fd15298 --- /dev/null +++ b/src/embedder/formats/yaml.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import re +from pathlib import Path + +from embedder.formats.base import LineScanner + + +OPEN_RE = re.compile(r"^\s*#\s+embedder\s+(?P\S+)\s*$") +CLOSE_RE = re.compile(r"^\s*#\s+/embedder\s*$") + +# Matches the start of a block scalar (key: | or key: >), capturing indentation level. +_BLOCK_SCALAR_RE = re.compile(r"^(?P\s*)\S.*:\s*[|>][+-]?\s*$") + + +class _YamlScanner: + """Skips lines inside YAML block scalars to prevent false-positive marker detection.""" + + def __init__(self) -> None: + self._block_indent: int | None = None + + def advance(self, line: str) -> bool: + stripped = line.rstrip("\n") + + if self._block_indent is not None: + if stripped and not stripped[0].isspace(): + self._block_indent = None + elif len(stripped) - len(stripped.lstrip()) > self._block_indent: + return True + else: + self._block_indent = None + + m = _BLOCK_SCALAR_RE.match(stripped) + if m: + self._block_indent = len(m["indent"]) + + return False + + +class YamlFormat: + @property + def open_re(self) -> re.Pattern[str]: + return OPEN_RE + + @property + def close_re(self) -> re.Pattern[str]: + return CLOSE_RE + + def matches(self, path: Path) -> bool: + return path.suffix.lower() in {".yml", ".yaml"} + + def make_scanner(self) -> _YamlScanner: + return _YamlScanner() + + def render_open(self, ref_str: str) -> str: + return f"# embedder {ref_str}" + + def render_close(self) -> str: + return "# /embedder" diff --git a/src/embedder/providers/__init__.py b/src/embedder/providers/__init__.py new file mode 100644 index 0000000..0819968 --- /dev/null +++ b/src/embedder/providers/__init__.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from embedder.providers.base import Provider +from embedder.providers.github import GitHubProvider +from embedder.providers.local import LocalRef, LocalProvider +from embedder.refs import GitHubAssetRef, RefError + +AnyRef = GitHubAssetRef | LocalRef + + +class ProviderRegistry: + def __init__(self, providers: list[Provider]) -> None: + self._providers = providers + + def get(self, raw: str) -> Provider: + for provider in self._providers: + if provider.matches(raw): + return provider + raise RefError(f"Unknown ref scheme: {raw!r}") + + def parse_ref(self, raw: str) -> AnyRef: + return self.get(raw).parse_ref(raw) + + +DEFAULT_REGISTRY = ProviderRegistry([GitHubProvider(), LocalProvider()]) + + +def parse_ref(raw: str) -> AnyRef: + return DEFAULT_REGISTRY.parse_ref(raw) diff --git a/src/embedder/providers/base.py b/src/embedder/providers/base.py new file mode 100644 index 0000000..e7eeb0d --- /dev/null +++ b/src/embedder/providers/base.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from embedder.providers import AnyRef + + +@runtime_checkable +class Provider(Protocol): + def matches(self, raw: str) -> bool: ... + + def parse_ref(self, raw: str) -> "AnyRef": ... + + def resolve(self, ref: "AnyRef") -> "AnyRef": + """Return the ref updated to the latest available version.""" + ... + + def fetch(self, ref: "AnyRef", base_dir: Path) -> str: + """Return the fragment content.""" + ... + + def cache_key(self, ref: "AnyRef") -> str | None: + """Deduplication key for resolve() calls. None means no caching.""" + ... diff --git a/src/embedder/github.py b/src/embedder/providers/github.py similarity index 77% rename from src/embedder/github.py rename to src/embedder/providers/github.py index debc26a..c2798ca 100644 --- a/src/embedder/github.py +++ b/src/embedder/providers/github.py @@ -7,7 +7,7 @@ from pathlib import Path from embedder.blocks import EmbedderEnvironmentError, EmbedderError -from embedder.refs import GitHubAssetRef +from embedder.refs import GitHubAssetRef, parse_github_ref @dataclass(frozen=True) @@ -17,7 +17,22 @@ class CommandResult: stderr: str -class GitHubClient: +class GitHubProvider: + def matches(self, raw: str) -> bool: + return raw.startswith("github.com/") + + def parse_ref(self, raw: str) -> GitHubAssetRef: + return parse_github_ref(raw) + + def resolve(self, ref: GitHubAssetRef) -> GitHubAssetRef: + return ref.with_tag(self._latest_tag(ref)) + + def fetch(self, ref: GitHubAssetRef, base_dir: Path) -> str: + return self._download_asset(ref) + + def cache_key(self, ref: GitHubAssetRef) -> str: + return ref.repository + def available(self) -> bool: return shutil.which("gh") is not None @@ -45,7 +60,7 @@ def run(self, args: list[str], *, check: bool = True) -> CommandResult: def auth_ok(self) -> bool: return self.run(["auth", "status"], check=False).returncode == 0 - def latest_tag(self, ref: GitHubAssetRef) -> str: + def _latest_tag(self, ref: GitHubAssetRef) -> str: result = self.run( [ "release", @@ -62,7 +77,7 @@ def latest_tag(self, ref: GitHubAssetRef) -> str: raise EmbedderError(f"Could not resolve latest release for {ref.repository}") return result.stdout - def download_asset(self, ref: GitHubAssetRef) -> str: + def _download_asset(self, ref: GitHubAssetRef) -> str: with tempfile.TemporaryDirectory(prefix="embedder-") as tmpdir: self.run( [ diff --git a/src/embedder/providers/local.py b/src/embedder/providers/local.py new file mode 100644 index 0000000..6f9dd04 --- /dev/null +++ b/src/embedder/providers/local.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from embedder.blocks import EmbedderError +from embedder.refs import RefError + + +@dataclass(frozen=True) +class LocalRef: + path: str + + def render(self) -> str: + return f"local:{self.path}" + + +class LocalProvider: + def matches(self, raw: str) -> bool: + return raw.startswith("local:") + + def parse_ref(self, raw: str) -> LocalRef: + path = raw[len("local:"):] + if not path: + raise RefError("local: ref must specify a path") + return LocalRef(path=path) + + def resolve(self, ref: LocalRef) -> LocalRef: + return ref + + def fetch(self, ref: LocalRef, base_dir: Path) -> str: + target = (base_dir / ref.path).resolve() + if not target.is_file(): + raise EmbedderError(f"Local fragment not found: {target}") + return target.read_text(encoding="utf-8") + + def cache_key(self, ref: LocalRef) -> None: + return None diff --git a/src/embedder/refs.py b/src/embedder/refs.py index 52b3ff5..67054d2 100644 --- a/src/embedder/refs.py +++ b/src/embedder/refs.py @@ -39,7 +39,7 @@ def render(self) -> str: return f"github.com/{self.repository}@{self.tag}:{self.asset}" -def parse_ref(raw: str) -> GitHubAssetRef: +def parse_github_ref(raw: str) -> GitHubAssetRef: match = REF_RE.match(raw.strip()) if not match: raise RefError(f"Invalid embedder ref: {raw}") diff --git a/src/embedder/updater.py b/src/embedder/updater.py index ef41d11..6e7600d 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -10,17 +10,17 @@ iter_files, parse_blocks, ) -from embedder.github import GitHubClient +from embedder.providers import AnyRef, DEFAULT_REGISTRY, ProviderRegistry @dataclass(frozen=True) class CheckResult: block: EmbedderBlock - latest_tag: str + latest_ref: AnyRef @property def update_available(self) -> bool: - return self.block.ref.tag != self.latest_tag + return self.block.ref != self.latest_ref @dataclass(frozen=True) @@ -29,21 +29,32 @@ class FileUpdate: changed_blocks: list[CheckResult] -def check_blocks(blocks: list[EmbedderBlock], github: GitHubClient) -> list[CheckResult]: - latest_by_repository: dict[str, str] = {} +def check_blocks( + blocks: list[EmbedderBlock], + registry: ProviderRegistry = DEFAULT_REGISTRY, +) -> list[CheckResult]: + cache: dict[str, AnyRef] = {} results: list[CheckResult] = [] for block in blocks: - latest = latest_by_repository.get(block.ref.repository) - if latest is None: - latest = github.latest_tag(block.ref) - latest_by_repository[block.ref.repository] = latest - results.append(CheckResult(block=block, latest_tag=latest)) + provider = registry.get(block.ref.render()) + key = provider.cache_key(block.ref) + if key is not None: + latest = cache.get(key) + if latest is None: + latest = provider.resolve(block.ref) + cache[key] = latest + else: + latest = provider.resolve(block.ref) + results.append(CheckResult(block=block, latest_ref=latest)) return results -def update_files(paths: list[Path], github: GitHubClient) -> list[FileUpdate]: +def update_files( + paths: list[Path], + registry: ProviderRegistry = DEFAULT_REGISTRY, +) -> list[FileUpdate]: changed: list[FileUpdate] = [] for path in iter_files(paths): @@ -55,23 +66,24 @@ def update_files(paths: list[Path], github: GitHubClient) -> list[FileUpdate]: if not blocks: continue - checks = check_blocks(blocks, github) + checks = check_blocks(blocks, registry) updates: list[BlockUpdate] = [] changed_checks: list[CheckResult] = [] + for check in checks: if not check.update_available: continue - new_ref = check.block.ref.with_tag(check.latest_tag) - new_body = github.download_asset(new_ref) + provider = registry.get(check.latest_ref.render()) + new_body = provider.fetch(check.latest_ref, path.parent) updates.append( - BlockUpdate(block=check.block, new_ref=new_ref, new_body=new_body) + BlockUpdate(block=check.block, new_ref=check.latest_ref, new_body=new_body) ) changed_checks.append(check) if not updates: continue - new_text = apply_updates(text, updates) + new_text = apply_updates(path, text, updates) if new_text == text: continue path.write_text(new_text, encoding="utf-8") diff --git a/tests/helpers.py b/tests/helpers.py index 38d4632..5beb442 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,3 +4,11 @@ def marker(ref: str) -> str: def close_marker() -> str: return "" + + +def yaml_marker(ref: str) -> str: + return f"# embedder {ref}" + + +def yaml_close_marker() -> str: + return "# /embedder" diff --git a/tests/test_blocks.py b/tests/test_blocks.py index b82e07f..a5a51f3 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -3,7 +3,7 @@ import pytest from embedder.blocks import EmbedderError, parse_blocks -from tests.helpers import close_marker, marker +from tests.helpers import close_marker, marker, yaml_close_marker, yaml_marker def test_parse_single_block() -> None: @@ -93,3 +93,54 @@ def test_ignore_markers_inside_markdown_code_fence() -> None: ) assert parse_blocks(Path("README.md"), text) == [] + + +def test_parse_yaml_block() -> None: + text = "\n".join( + [ + "before: value", + yaml_marker("local:fragments/config.yaml"), + "key: managed", + yaml_close_marker(), + "after: value", + "", + ] + ) + + blocks = parse_blocks(Path("config.yaml"), text) + + assert len(blocks) == 1 + block = blocks[0] + assert block.ref.render() == "local:fragments/config.yaml" + assert block.body == "key: managed\n" + + +def test_yaml_ignores_markers_in_block_scalar() -> None: + text = "\n".join( + [ + "description: |", + f" {yaml_marker('local:file.yaml')}", + " some content", + f" {yaml_close_marker()}", + "other: value", + "", + ] + ) + + assert parse_blocks(Path("config.yaml"), text) == [] + + +def test_parse_local_ref_in_markdown() -> None: + text = "\n".join( + [ + marker("local:fragments/file.md"), + "managed content", + close_marker(), + "", + ] + ) + + blocks = parse_blocks(Path("README.md"), text) + + assert len(blocks) == 1 + assert blocks[0].ref.render() == "local:fragments/file.md" diff --git a/tests/test_cli.py b/tests/test_cli.py index 12d23ca..4d6c941 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,7 +23,7 @@ def test_scan_prints_blocks(tmp_path: Path, capsys) -> None: assert main(["scan", str(tmp_path)]) == int(ExitCode.OK) captured = capsys.readouterr() - assert "AGENTS.md:3 rubykatzen/embedder@v0.1.0:fragment.md" in captured.out + assert "AGENTS.md:3 github.com/rubykatzen/embedder@v0.1.0:fragment.md" in captured.out def test_scan_json(tmp_path: Path, capsys) -> None: diff --git a/tests/test_refs.py b/tests/test_refs.py index ca645a8..9b112b7 100644 --- a/tests/test_refs.py +++ b/tests/test_refs.py @@ -1,10 +1,12 @@ import pytest -from embedder.refs import RefError, parse_ref +from embedder.refs import GitHubAssetRef, RefError, parse_github_ref +from embedder.providers import parse_ref +from embedder.providers.local import LocalRef -def test_parse_ref() -> None: - ref = parse_ref("github.com/OWNER/repo-name@v0.3.0:agent-message-prefix.md") +def test_parse_github_ref() -> None: + ref = parse_github_ref("github.com/OWNER/repo-name@v0.3.0:agent-message-prefix.md") assert ref.owner == "OWNER" assert ref.repo == "repo-name" @@ -14,17 +16,36 @@ def test_parse_ref() -> None: assert ref.render() == "github.com/OWNER/repo-name@v0.3.0:agent-message-prefix.md" -def test_ref_with_new_tag() -> None: - ref = parse_ref("github.com/rubykatzen/embedder@v0.1.0:fragment.md") +def test_github_ref_with_new_tag() -> None: + ref = parse_github_ref("github.com/rubykatzen/embedder@v0.1.0:fragment.md") assert ref.with_tag("v0.2.0").render() == "github.com/rubykatzen/embedder@v0.2.0:fragment.md" -def test_reject_invalid_ref() -> None: +def test_reject_invalid_github_ref() -> None: with pytest.raises(RefError): - parse_ref("rubykatzen/embedder@v0.1.0:fragment.md") + parse_github_ref("rubykatzen/embedder@v0.1.0:fragment.md") def test_reject_nested_asset_path() -> None: with pytest.raises(RefError, match="basename"): - parse_ref("github.com/rubykatzen/embedder@v0.1.0:fragments/fragment.md") + parse_github_ref("github.com/rubykatzen/embedder@v0.1.0:fragments/fragment.md") + + +def test_parse_ref_dispatches_github() -> None: + ref = parse_ref("github.com/rubykatzen/embedder@v0.1.0:fragment.md") + + assert isinstance(ref, GitHubAssetRef) + + +def test_parse_ref_dispatches_local() -> None: + ref = parse_ref("local:fragments/file.md") + + assert isinstance(ref, LocalRef) + assert ref.path == "fragments/file.md" + assert ref.render() == "local:fragments/file.md" + + +def test_parse_ref_rejects_unknown_scheme() -> None: + with pytest.raises(RefError, match="Unknown ref scheme"): + parse_ref("s3://bucket/file.md") diff --git a/tests/test_updater.py b/tests/test_updater.py index 8f2091e..0b21d2a 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1,27 +1,41 @@ from pathlib import Path from embedder.blocks import BlockUpdate, parse_blocks +from embedder.providers import AnyRef, ProviderRegistry +from embedder.providers.local import LocalProvider, LocalRef +from embedder.refs import GitHubAssetRef, parse_github_ref from embedder.updater import check_blocks, update_files from tests.helpers import close_marker, marker -class FakeGitHub: +class FakeGitHubProvider: def __init__(self) -> None: - self.latest = { - "rubykatzen/embedder": "v0.2.0", - } - self.assets = { + self.latest: dict[str, str] = {"rubykatzen/embedder": "v0.2.0"} + self.assets: dict[str, str] = { "github.com/rubykatzen/embedder@v0.2.0:fragment.md": "new managed text\n", } - self.latest_calls = 0 + self.resolve_calls = 0 + + def matches(self, raw: str) -> bool: + return raw.startswith("github.com/") - def latest_tag(self, ref) -> str: - self.latest_calls += 1 - return self.latest[ref.repository] + def parse_ref(self, raw: str) -> GitHubAssetRef: + return parse_github_ref(raw) - def download_asset(self, ref) -> str: + def resolve(self, ref: GitHubAssetRef) -> GitHubAssetRef: + self.resolve_calls += 1 + return ref.with_tag(self.latest[ref.repository]) + + def fetch(self, ref: GitHubAssetRef, base_dir: Path) -> str: return self.assets[ref.render()] + def cache_key(self, ref: GitHubAssetRef) -> str: + return ref.repository + + +def fake_registry(**kwargs) -> ProviderRegistry: + return ProviderRegistry([FakeGitHubProvider(**kwargs) if kwargs else FakeGitHubProvider(), LocalProvider()]) + def test_check_blocks_marks_updates() -> None: text = "\n".join( @@ -34,10 +48,10 @@ def test_check_blocks_marks_updates() -> None: ) blocks = parse_blocks(Path("AGENTS.md"), text) - results = check_blocks(blocks, FakeGitHub()) + results = check_blocks(blocks, fake_registry()) assert len(results) == 1 - assert results[0].latest_tag == "v0.2.0" + assert results[0].latest_ref.render() == "github.com/rubykatzen/embedder@v0.2.0:fragment.md" assert results[0].update_available @@ -57,7 +71,7 @@ def test_update_files_replaces_only_managed_body(tmp_path: Path) -> None: encoding="utf-8", ) - changed = update_files([tmp_path], FakeGitHub()) + changed = update_files([tmp_path], fake_registry()) assert [item.path for item in changed] == [str(target)] assert target.read_text(encoding="utf-8") == "\n".join( @@ -86,7 +100,11 @@ def test_apply_update_keeps_body_trailing_newline() -> None: from embedder.blocks import apply_updates - updated = apply_updates(text, [BlockUpdate(block=block, new_ref=new_ref, new_body="new")]) + updated = apply_updates( + Path("AGENTS.md"), + text, + [BlockUpdate(block=block, new_ref=new_ref, new_body="new")], + ) assert updated == "\n".join( [ @@ -98,7 +116,7 @@ def test_apply_update_keeps_body_trailing_newline() -> None: ) -def test_check_blocks_caches_latest_release_per_repository() -> None: +def test_check_blocks_caches_resolve_per_repository() -> None: text = "\n".join( [ marker("github.com/rubykatzen/embedder@v0.1.0:first.md"), @@ -111,8 +129,42 @@ def test_check_blocks_caches_latest_release_per_repository() -> None: ] ) blocks = parse_blocks(Path("AGENTS.md"), text) - github = FakeGitHub() + provider = FakeGitHubProvider() + registry = ProviderRegistry([provider, LocalProvider()]) + + check_blocks(blocks, registry) + + assert provider.resolve_calls == 1 + + +def test_local_ref_is_always_current(tmp_path: Path) -> None: + fragment = tmp_path / "fragment.md" + fragment.write_text("local content\n", encoding="utf-8") + + target = tmp_path / "README.md" + target.write_text( + "\n".join( + [ + marker("local:fragment.md"), + "old content", + close_marker(), + "", + ] + ), + encoding="utf-8", + ) + + registry = ProviderRegistry([FakeGitHubProvider(), LocalProvider()]) + results = check_blocks(parse_blocks(target, target.read_text(encoding="utf-8")), registry) + + assert not results[0].update_available + + +def test_local_ref_fetch(tmp_path: Path) -> None: + fragment = tmp_path / "frag.md" + fragment.write_text("hello from local\n", encoding="utf-8") - check_blocks(blocks, github) + ref = LocalRef(path="frag.md") + content = LocalProvider().fetch(ref, tmp_path) - assert github.latest_calls == 1 + assert content == "hello from local\n" From d3ef623828f01791415148a8079f26dd2ae52191 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 14:03:24 +0200 Subject: [PATCH 02/25] fix: resolve ruff lint errors (F401, I001, UP037) Remove unused imports, sort import blocks, and remove unnecessary quotes from type annotations to match baseline ruff config. Co-Authored-By: Claude Sonnet 4.6 --- src/embedder/cli.py | 2 +- src/embedder/formats/markdown.py | 1 - src/embedder/formats/text.py | 1 - src/embedder/formats/yaml.py | 3 --- src/embedder/providers/__init__.py | 2 +- src/embedder/providers/base.py | 8 ++++---- src/embedder/updater.py | 2 +- tests/test_refs.py | 2 +- tests/test_updater.py | 2 +- 9 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/embedder/cli.py b/src/embedder/cli.py index 6fc4342..a319cf0 100644 --- a/src/embedder/cli.py +++ b/src/embedder/cli.py @@ -3,7 +3,7 @@ import argparse import json import sys -from dataclasses import asdict, fields, is_dataclass +from dataclasses import fields, is_dataclass from enum import IntEnum from pathlib import Path from typing import Any diff --git a/src/embedder/formats/markdown.py b/src/embedder/formats/markdown.py index aa056eb..6c4bbdb 100644 --- a/src/embedder/formats/markdown.py +++ b/src/embedder/formats/markdown.py @@ -5,7 +5,6 @@ from embedder.formats.text import TextFormat - FENCE_RE = re.compile(r"^\s*(?P`{3,}|~{3,})") diff --git a/src/embedder/formats/text.py b/src/embedder/formats/text.py index c8b511e..505a852 100644 --- a/src/embedder/formats/text.py +++ b/src/embedder/formats/text.py @@ -3,7 +3,6 @@ import re from pathlib import Path - OPEN_RE = re.compile(r"^\s*\s*$") CLOSE_RE = re.compile(r"^\s*\s*$") diff --git a/src/embedder/formats/yaml.py b/src/embedder/formats/yaml.py index fd15298..02ef3c8 100644 --- a/src/embedder/formats/yaml.py +++ b/src/embedder/formats/yaml.py @@ -3,9 +3,6 @@ import re from pathlib import Path -from embedder.formats.base import LineScanner - - OPEN_RE = re.compile(r"^\s*#\s+embedder\s+(?P\S+)\s*$") CLOSE_RE = re.compile(r"^\s*#\s+/embedder\s*$") diff --git a/src/embedder/providers/__init__.py b/src/embedder/providers/__init__.py index 0819968..8d66071 100644 --- a/src/embedder/providers/__init__.py +++ b/src/embedder/providers/__init__.py @@ -2,7 +2,7 @@ from embedder.providers.base import Provider from embedder.providers.github import GitHubProvider -from embedder.providers.local import LocalRef, LocalProvider +from embedder.providers.local import LocalProvider, LocalRef from embedder.refs import GitHubAssetRef, RefError AnyRef = GitHubAssetRef | LocalRef diff --git a/src/embedder/providers/base.py b/src/embedder/providers/base.py index e7eeb0d..90b6018 100644 --- a/src/embedder/providers/base.py +++ b/src/embedder/providers/base.py @@ -11,16 +11,16 @@ class Provider(Protocol): def matches(self, raw: str) -> bool: ... - def parse_ref(self, raw: str) -> "AnyRef": ... + def parse_ref(self, raw: str) -> AnyRef: ... - def resolve(self, ref: "AnyRef") -> "AnyRef": + def resolve(self, ref: AnyRef) -> AnyRef: """Return the ref updated to the latest available version.""" ... - def fetch(self, ref: "AnyRef", base_dir: Path) -> str: + def fetch(self, ref: AnyRef, base_dir: Path) -> str: """Return the fragment content.""" ... - def cache_key(self, ref: "AnyRef") -> str | None: + def cache_key(self, ref: AnyRef) -> str | None: """Deduplication key for resolve() calls. None means no caching.""" ... diff --git a/src/embedder/updater.py b/src/embedder/updater.py index 6e7600d..99e9c65 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -10,7 +10,7 @@ iter_files, parse_blocks, ) -from embedder.providers import AnyRef, DEFAULT_REGISTRY, ProviderRegistry +from embedder.providers import DEFAULT_REGISTRY, AnyRef, ProviderRegistry @dataclass(frozen=True) diff --git a/tests/test_refs.py b/tests/test_refs.py index 9b112b7..0c33bd8 100644 --- a/tests/test_refs.py +++ b/tests/test_refs.py @@ -1,8 +1,8 @@ import pytest -from embedder.refs import GitHubAssetRef, RefError, parse_github_ref from embedder.providers import parse_ref from embedder.providers.local import LocalRef +from embedder.refs import GitHubAssetRef, RefError, parse_github_ref def test_parse_github_ref() -> None: diff --git a/tests/test_updater.py b/tests/test_updater.py index 0b21d2a..0acc69a 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1,7 +1,7 @@ from pathlib import Path from embedder.blocks import BlockUpdate, parse_blocks -from embedder.providers import AnyRef, ProviderRegistry +from embedder.providers import ProviderRegistry from embedder.providers.local import LocalProvider, LocalRef from embedder.refs import GitHubAssetRef, parse_github_ref from embedder.updater import check_blocks, update_files From 4ea22fd08d6600178317b9247c35c30e2ac15eca Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 14:34:31 +0200 Subject: [PATCH 03/25] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20cache=20asset=20bug,=20local=20refresh,=20YAML=20bl?= =?UTF-8?q?anks,=20path=20traversal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - check_blocks: cache was reusing the first block's asset for subsequent blocks from the same repo; fix with resolve_cached() that applies the cached tag to the current block's own ref - update_files: local: refs never refreshed because update_available was always false; add always_refresh() to Provider protocol so LocalProvider opts in to unconditional re-fetch - _YamlScanner: blank lines inside YAML block scalars cleared block state prematurely, causing false-positive marker detection after blank lines - LocalProvider: path traversal allowed arbitrary file reads; validate resolved path stays within base_dir - update-fragments-shared.yml: bump pinned version to v0.0.6 and make jq backward-compatible with both latest_ref.tag and legacy latest_tag Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/update-fragments-shared.yml | 4 +- src/embedder/formats/yaml.py | 5 +- src/embedder/providers/base.py | 13 +++++ src/embedder/providers/github.py | 6 ++ src/embedder/providers/local.py | 11 ++++ src/embedder/updater.py | 15 +++-- tests/test_blocks.py | 17 ++++++ tests/test_updater.py | 56 ++++++++++++++++++- 8 files changed, 117 insertions(+), 10 deletions(-) diff --git a/.github/workflows/update-fragments-shared.yml b/.github/workflows/update-fragments-shared.yml index 7a133bd..1d71f71 100644 --- a/.github/workflows/update-fragments-shared.yml +++ b/.github/workflows/update-fragments-shared.yml @@ -23,7 +23,7 @@ jobs: with: python-version: "3.13" - name: Install embedder - run: pip install "git+https://github.com/rubykatzen/embedder.git@v0.0.5" + run: pip install "git+https://github.com/rubykatzen/embedder.git@v0.0.6" - name: Run embedder update env: GH_TOKEN: ${{ secrets.token || github.token }} @@ -54,7 +54,7 @@ jobs: body=$(jq -r ' "## Embedded fragment updates\n\n| File | Source | Asset | Old | New |\n|---|---|---|---:|---:|\n" + ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(.block.ref.owner)/\(.block.ref.repo) | \(.block.ref.asset) | \(.block.ref.tag) | \(.latest_ref.tag) |" + "| \(.block.path) | \(.block.ref.owner)/\(.block.ref.repo) | \(.block.ref.asset) | \(.block.ref.tag) | \(.latest_ref.tag // .latest_tag) |" ] | join("\n")) ' /tmp/embedder-result.json) existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') diff --git a/src/embedder/formats/yaml.py b/src/embedder/formats/yaml.py index 02ef3c8..bfeba5b 100644 --- a/src/embedder/formats/yaml.py +++ b/src/embedder/formats/yaml.py @@ -20,7 +20,10 @@ def advance(self, line: str) -> bool: stripped = line.rstrip("\n") if self._block_indent is not None: - if stripped and not stripped[0].isspace(): + if not stripped: + # Blank lines are valid inside a block scalar. + return True + if not stripped[0].isspace(): self._block_indent = None elif len(stripped) - len(stripped.lstrip()) > self._block_indent: return True diff --git a/src/embedder/providers/base.py b/src/embedder/providers/base.py index 90b6018..b0b3b62 100644 --- a/src/embedder/providers/base.py +++ b/src/embedder/providers/base.py @@ -17,6 +17,19 @@ def resolve(self, ref: AnyRef) -> AnyRef: """Return the ref updated to the latest available version.""" ... + def resolve_cached(self, ref: AnyRef, cached: AnyRef) -> AnyRef: + """Apply the version from a previously resolved ref to a different ref. + + Used to reuse a cached provider lookup (e.g. latest tag) for a block + with the same cache key but a different asset or path. + """ + ... + + def always_refresh(self, ref: AnyRef) -> bool: + """Return True if this ref should always be re-fetched on update, + even when resolve() reports no version change.""" + ... + def fetch(self, ref: AnyRef, base_dir: Path) -> str: """Return the fragment content.""" ... diff --git a/src/embedder/providers/github.py b/src/embedder/providers/github.py index c2798ca..3947023 100644 --- a/src/embedder/providers/github.py +++ b/src/embedder/providers/github.py @@ -27,6 +27,12 @@ def parse_ref(self, raw: str) -> GitHubAssetRef: def resolve(self, ref: GitHubAssetRef) -> GitHubAssetRef: return ref.with_tag(self._latest_tag(ref)) + def resolve_cached(self, ref: GitHubAssetRef, cached: GitHubAssetRef) -> GitHubAssetRef: + return ref.with_tag(cached.tag) + + def always_refresh(self, ref: GitHubAssetRef) -> bool: + return False + def fetch(self, ref: GitHubAssetRef, base_dir: Path) -> str: return self._download_asset(ref) diff --git a/src/embedder/providers/local.py b/src/embedder/providers/local.py index 6f9dd04..2b7c69b 100644 --- a/src/embedder/providers/local.py +++ b/src/embedder/providers/local.py @@ -28,8 +28,19 @@ def parse_ref(self, raw: str) -> LocalRef: def resolve(self, ref: LocalRef) -> LocalRef: return ref + def resolve_cached(self, ref: LocalRef, cached: LocalRef) -> LocalRef: + return ref + + def always_refresh(self, ref: LocalRef) -> bool: + return True + def fetch(self, ref: LocalRef, base_dir: Path) -> str: + resolved_base = base_dir.resolve() target = (base_dir / ref.path).resolve() + if not target.is_relative_to(resolved_base): + raise EmbedderError( + f"Local fragment path escapes base directory: {ref.path!r}" + ) if not target.is_file(): raise EmbedderError(f"Local fragment not found: {target}") return target.read_text(encoding="utf-8") diff --git a/src/embedder/updater.py b/src/embedder/updater.py index 99e9c65..1f707d5 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -33,17 +33,18 @@ def check_blocks( blocks: list[EmbedderBlock], registry: ProviderRegistry = DEFAULT_REGISTRY, ) -> list[CheckResult]: - cache: dict[str, AnyRef] = {} + resolved_cache: dict[str, AnyRef] = {} results: list[CheckResult] = [] for block in blocks: provider = registry.get(block.ref.render()) key = provider.cache_key(block.ref) if key is not None: - latest = cache.get(key) - if latest is None: + if key in resolved_cache: + latest = provider.resolve_cached(block.ref, resolved_cache[key]) + else: latest = provider.resolve(block.ref) - cache[key] = latest + resolved_cache[key] = latest else: latest = provider.resolve(block.ref) results.append(CheckResult(block=block, latest_ref=latest)) @@ -71,10 +72,12 @@ def update_files( changed_checks: list[CheckResult] = [] for check in checks: - if not check.update_available: + provider = registry.get(check.block.ref.render()) + if not check.update_available and not provider.always_refresh(check.block.ref): continue - provider = registry.get(check.latest_ref.render()) new_body = provider.fetch(check.latest_ref, path.parent) + if new_body == check.block.body and not check.update_available: + continue updates.append( BlockUpdate(block=check.block, new_ref=check.latest_ref, new_body=new_body) ) diff --git a/tests/test_blocks.py b/tests/test_blocks.py index a5a51f3..895a3b8 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -130,6 +130,23 @@ def test_yaml_ignores_markers_in_block_scalar() -> None: assert parse_blocks(Path("config.yaml"), text) == [] +def test_yaml_ignores_markers_after_blank_line_in_block_scalar() -> None: + text = "\n".join( + [ + "description: |", + " first paragraph", + "", + f" {yaml_marker('local:file.yaml')}", + " second paragraph", + f" {yaml_close_marker()}", + "other: value", + "", + ] + ) + + assert parse_blocks(Path("config.yaml"), text) == [] + + def test_parse_local_ref_in_markdown() -> None: text = "\n".join( [ diff --git a/tests/test_updater.py b/tests/test_updater.py index 0acc69a..710ce08 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1,6 +1,8 @@ from pathlib import Path -from embedder.blocks import BlockUpdate, parse_blocks +import pytest + +from embedder.blocks import BlockUpdate, EmbedderError, parse_blocks from embedder.providers import ProviderRegistry from embedder.providers.local import LocalProvider, LocalRef from embedder.refs import GitHubAssetRef, parse_github_ref @@ -26,6 +28,12 @@ def resolve(self, ref: GitHubAssetRef) -> GitHubAssetRef: self.resolve_calls += 1 return ref.with_tag(self.latest[ref.repository]) + def resolve_cached(self, ref: GitHubAssetRef, cached: GitHubAssetRef) -> GitHubAssetRef: + return ref.with_tag(cached.tag) + + def always_refresh(self, ref: GitHubAssetRef) -> bool: + return False + def fetch(self, ref: GitHubAssetRef, base_dir: Path) -> str: return self.assets[ref.render()] @@ -168,3 +176,49 @@ def test_local_ref_fetch(tmp_path: Path) -> None: content = LocalProvider().fetch(ref, tmp_path) assert content == "hello from local\n" + + +def test_cache_uses_correct_asset_per_block() -> None: + """Two blocks from the same repo must not share each other's asset.""" + text = "\n".join( + [ + marker("github.com/rubykatzen/embedder@v0.1.0:first.md"), + "old", + close_marker(), + marker("github.com/rubykatzen/embedder@v0.1.0:second.md"), + "old", + close_marker(), + "", + ] + ) + blocks = parse_blocks(Path("AGENTS.md"), text) + registry = ProviderRegistry([FakeGitHubProvider(), LocalProvider()]) + + results = check_blocks(blocks, registry) + + assert results[0].latest_ref.render() == "github.com/rubykatzen/embedder@v0.2.0:first.md" + assert results[1].latest_ref.render() == "github.com/rubykatzen/embedder@v0.2.0:second.md" + + +def test_local_ref_body_refreshed_on_update(tmp_path: Path) -> None: + """update_files re-fetches local refs even when the ref itself hasn't changed.""" + fragment = tmp_path / "frag.md" + fragment.write_text("v1 content\n", encoding="utf-8") + + target = tmp_path / "README.md" + target.write_text( + "\n".join([marker("local:frag.md"), "old content", close_marker(), ""]), + encoding="utf-8", + ) + + fragment.write_text("v2 content\n", encoding="utf-8") + registry = ProviderRegistry([FakeGitHubProvider(), LocalProvider()]) + update_files([tmp_path], registry) + + assert "v2 content" in target.read_text(encoding="utf-8") + + +def test_local_ref_path_traversal_rejected(tmp_path: Path) -> None: + ref = LocalRef(path="../../etc/passwd") + with pytest.raises(EmbedderError, match="escapes base directory"): + LocalProvider().fetch(ref, tmp_path) From c74730232f08ccdc9f0d159b45254758e32cbaf9 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 14:54:08 +0200 Subject: [PATCH 04/25] fix: address second round of review feedback - formats/__init__: split _FORMATS into _SPECIFIC_FORMATS + _FALLBACK to make the TextFormat fallback explicit and remove unreachable code - local.py: fix cache_key return annotation to str | None per protocol - yaml.py: extend block-scalar regex to cover indentation indicators (|2, >9), chomp+indent combos (|2-, >-2) and trailing comments (| # note) - update-fragments.yml: build ref string from fields in jq instead of using .render (a method, not a JSON field, always serialises as null) Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/update-fragments.yml | 2 +- src/embedder/formats/__init__.py | 8 ++++---- src/embedder/formats/yaml.py | 6 ++++-- src/embedder/providers/local.py | 2 +- tests/test_blocks.py | 26 ++++++++++++++++++++++++++ 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/update-fragments.yml b/.github/workflows/update-fragments.yml index ff4d3ac..baa0e9f 100644 --- a/.github/workflows/update-fragments.yml +++ b/.github/workflows/update-fragments.yml @@ -43,7 +43,7 @@ jobs: body=$(jq -r ' "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(.block.ref.render // .block.ref.path // (.block.ref.owner + "/" + .block.ref.repo + ":" + .block.ref.asset)) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" + "| \(.block.path) | \(if .block.ref.owner then "github.com/\(.block.ref.owner)/\(.block.ref.repo)@\(.block.ref.tag):\(.block.ref.asset)" else "local:\(.block.ref.path)" end) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" ] | join("\n")) ' /tmp/embedder-result.json) existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') diff --git a/src/embedder/formats/__init__.py b/src/embedder/formats/__init__.py index 0011774..bb6cc81 100644 --- a/src/embedder/formats/__init__.py +++ b/src/embedder/formats/__init__.py @@ -7,15 +7,15 @@ from embedder.formats.text import TextFormat from embedder.formats.yaml import YamlFormat -_FORMATS: list[Format] = [ +_SPECIFIC_FORMATS: list[Format] = [ MarkdownFormat(), YamlFormat(), - TextFormat(), ] +_FALLBACK: Format = TextFormat() def get_format(path: Path) -> Format: - for fmt in _FORMATS: + for fmt in _SPECIFIC_FORMATS: if fmt.matches(path): return fmt - return TextFormat() + return _FALLBACK diff --git a/src/embedder/formats/yaml.py b/src/embedder/formats/yaml.py index bfeba5b..356c59a 100644 --- a/src/embedder/formats/yaml.py +++ b/src/embedder/formats/yaml.py @@ -6,8 +6,10 @@ OPEN_RE = re.compile(r"^\s*#\s+embedder\s+(?P\S+)\s*$") CLOSE_RE = re.compile(r"^\s*#\s+/embedder\s*$") -# Matches the start of a block scalar (key: | or key: >), capturing indentation level. -_BLOCK_SCALAR_RE = re.compile(r"^(?P\s*)\S.*:\s*[|>][+-]?\s*$") +# Matches YAML block scalar headers: key: |, key: |2, key: |-2, key: | # note, etc. +# Per YAML spec, after | or > comes an optional indentation indicator (digit), +# optional chomp indicator (+/-), and an optional trailing comment. +_BLOCK_SCALAR_RE = re.compile(r"^(?P\s*)\S.*:\s*[|>][0-9]?[+-]?[0-9]?(?:\s+#.*)?\s*$") class _YamlScanner: diff --git a/src/embedder/providers/local.py b/src/embedder/providers/local.py index 2b7c69b..085fd6c 100644 --- a/src/embedder/providers/local.py +++ b/src/embedder/providers/local.py @@ -45,5 +45,5 @@ def fetch(self, ref: LocalRef, base_dir: Path) -> str: raise EmbedderError(f"Local fragment not found: {target}") return target.read_text(encoding="utf-8") - def cache_key(self, ref: LocalRef) -> None: + def cache_key(self, ref: LocalRef) -> str | None: return None diff --git a/tests/test_blocks.py b/tests/test_blocks.py index 895a3b8..0fddcd0 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -147,6 +147,32 @@ def test_yaml_ignores_markers_after_blank_line_in_block_scalar() -> None: assert parse_blocks(Path("config.yaml"), text) == [] +def test_yaml_ignores_markers_in_block_scalar_with_indent_indicator() -> None: + text = "\n".join( + [ + "description: |2", + f" {yaml_marker('local:file.yaml')}", + "other: value", + "", + ] + ) + + assert parse_blocks(Path("config.yaml"), text) == [] + + +def test_yaml_ignores_markers_in_block_scalar_with_trailing_comment() -> None: + text = "\n".join( + [ + "description: | # inline comment", + f" {yaml_marker('local:file.yaml')}", + "other: value", + "", + ] + ) + + assert parse_blocks(Path("config.yaml"), text) == [] + + def test_parse_local_ref_in_markdown() -> None: text = "\n".join( [ From 1aa025db3dbe900826b32cb6d7d72bb3d9463874 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 17:18:49 +0200 Subject: [PATCH 05/25] fix: indent replacement body to match YAML marker indentation When an embedder block marker is indented (e.g. under a parent YAML key), apply_updates now re-indents each body line by the same prefix so the replacement content stays structurally valid after an update. Co-Authored-By: Claude Sonnet 4.6 --- src/embedder/blocks.py | 5 +++++ tests/test_blocks.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/embedder/blocks.py b/src/embedder/blocks.py index 48d88b5..fd58b6e 100644 --- a/src/embedder/blocks.py +++ b/src/embedder/blocks.py @@ -159,6 +159,11 @@ def apply_updates(path: Path, text: str, updates: list[BlockUpdate]) -> str: replacement_body = update.new_body if replacement_body and not replacement_body.endswith("\n"): replacement_body += "\n" + if indent and replacement_body: + replacement_body = "".join( + indent + line if line.rstrip("\n") else line + for line in replacement_body.splitlines(keepends=True) + ) output.extend(lines[cursor:start]) output.append(f"{indent}{fmt.render_open(update.new_ref.render())}{newline}") diff --git a/tests/test_blocks.py b/tests/test_blocks.py index 0fddcd0..c3bf8f4 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -2,7 +2,7 @@ import pytest -from embedder.blocks import EmbedderError, parse_blocks +from embedder.blocks import BlockUpdate, EmbedderError, apply_updates, parse_blocks from tests.helpers import close_marker, marker, yaml_close_marker, yaml_marker @@ -187,3 +187,33 @@ def test_parse_local_ref_in_markdown() -> None: assert len(blocks) == 1 assert blocks[0].ref.render() == "local:fragments/file.md" + + +def test_apply_updates_indents_body_to_match_yaml_marker() -> None: + text = "\n".join( + [ + "parent:", + f" {yaml_marker('local:frag.yaml')}", + " old: value", + f" {yaml_close_marker()}", + "", + ] + ) + path = Path("config.yaml") + block = parse_blocks(path, text)[0] + + updated = apply_updates( + path, + text, + [BlockUpdate(block=block, new_ref=block.ref, new_body="new: value\n")], + ) + + assert updated == "\n".join( + [ + "parent:", + f" {yaml_marker('local:frag.yaml')}", + " new: value", + f" {yaml_close_marker()}", + "", + ] + ) From 0ef3f0c57ee70cd9680713b798199b469bcfd8b4 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 17:24:25 +0200 Subject: [PATCH 06/25] refactor: remove TextFormat fallback, process only known formats get_format now returns None for unrecognised file types instead of falling back to TextFormat with HTML comment markers. parse_blocks and apply_updates silently skip files with no matching format. MarkdownFormat is now standalone and no longer inherits from TextFormat. Co-Authored-By: Claude Sonnet 4.6 --- src/embedder/blocks.py | 4 ++++ src/embedder/formats/__init__.py | 10 ++++------ src/embedder/formats/markdown.py | 20 ++++++++++++++++--- src/embedder/formats/text.py | 34 -------------------------------- 4 files changed, 25 insertions(+), 43 deletions(-) delete mode 100644 src/embedder/formats/text.py diff --git a/src/embedder/blocks.py b/src/embedder/blocks.py index fd58b6e..346da6c 100644 --- a/src/embedder/blocks.py +++ b/src/embedder/blocks.py @@ -49,6 +49,8 @@ def parse_blocks(path: Path, text: str) -> list[EmbedderBlock]: from embedder.providers import parse_ref fmt = get_format(path) + if fmt is None: + return [] scanner = fmt.make_scanner() lines = text.splitlines(keepends=True) blocks: list[EmbedderBlock] = [] @@ -143,6 +145,8 @@ def apply_updates(path: Path, text: str, updates: list[BlockUpdate]) -> str: return text fmt = get_format(path) + if fmt is None: + return text lines = text.splitlines(keepends=True) output: list[str] = [] cursor = 0 diff --git a/src/embedder/formats/__init__.py b/src/embedder/formats/__init__.py index bb6cc81..93bd8fe 100644 --- a/src/embedder/formats/__init__.py +++ b/src/embedder/formats/__init__.py @@ -4,18 +4,16 @@ from embedder.formats.base import Format from embedder.formats.markdown import MarkdownFormat -from embedder.formats.text import TextFormat from embedder.formats.yaml import YamlFormat -_SPECIFIC_FORMATS: list[Format] = [ +_FORMATS: list[Format] = [ MarkdownFormat(), YamlFormat(), ] -_FALLBACK: Format = TextFormat() -def get_format(path: Path) -> Format: - for fmt in _SPECIFIC_FORMATS: +def get_format(path: Path) -> Format | None: + for fmt in _FORMATS: if fmt.matches(path): return fmt - return _FALLBACK + return None diff --git a/src/embedder/formats/markdown.py b/src/embedder/formats/markdown.py index 6c4bbdb..660c642 100644 --- a/src/embedder/formats/markdown.py +++ b/src/embedder/formats/markdown.py @@ -3,8 +3,8 @@ import re from pathlib import Path -from embedder.formats.text import TextFormat - +OPEN_RE = re.compile(r"^\s*\s*$") +CLOSE_RE = re.compile(r"^\s*\s*$") FENCE_RE = re.compile(r"^\s*(?P`{3,}|~{3,})") @@ -24,9 +24,23 @@ def advance(self, line: str) -> bool: return self._fence is not None -class MarkdownFormat(TextFormat): +class MarkdownFormat: + @property + def open_re(self) -> re.Pattern[str]: + return OPEN_RE + + @property + def close_re(self) -> re.Pattern[str]: + return CLOSE_RE + def matches(self, path: Path) -> bool: return path.suffix.lower() in {".md", ".markdown"} def make_scanner(self) -> _MarkdownScanner: return _MarkdownScanner() + + def render_open(self, ref_str: str) -> str: + return f"" + + def render_close(self) -> str: + return "" diff --git a/src/embedder/formats/text.py b/src/embedder/formats/text.py deleted file mode 100644 index 505a852..0000000 --- a/src/embedder/formats/text.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path - -OPEN_RE = re.compile(r"^\s*\s*$") -CLOSE_RE = re.compile(r"^\s*\s*$") - - -class _NoopScanner: - def advance(self, line: str) -> bool: - return False - - -class TextFormat: - @property - def open_re(self) -> re.Pattern[str]: - return OPEN_RE - - @property - def close_re(self) -> re.Pattern[str]: - return CLOSE_RE - - def matches(self, path: Path) -> bool: - return True - - def make_scanner(self) -> _NoopScanner: - return _NoopScanner() - - def render_open(self, ref_str: str) -> str: - return f"" - - def render_close(self) -> str: - return "" From 4c795cc72e79a08fed858db734dcccefd6968c64 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 17:38:10 +0200 Subject: [PATCH 07/25] refactor: consolidate modules, eliminate circular imports - Extract EmbedderError, EmbedderEnvironmentError, RefError to errors.py - Move GitHubAssetRef + parse_github_ref into providers/github.py (symmetrical with LocalRef in providers/local.py) - Inline Format/LineScanner protocols into formats/__init__.py - Inline Provider protocol into providers/__init__.py - Replace ProviderRegistry class with get_provider() + DEFAULT_PROVIDERS - Remove lazy imports from parse_blocks/apply_updates (circular dep fixed) - Delete refs.py, formats/base.py, providers/base.py Co-Authored-By: Claude Sonnet 4.6 --- src/embedder/blocks.py | 31 +++++++++-------- src/embedder/cli.py | 8 ++--- src/embedder/errors.py | 13 +++++++ src/embedder/formats/__init__.py | 26 +++++++++++++- src/embedder/formats/base.py | 29 ---------------- src/embedder/providers/__init__.py | 42 ++++++++++++++-------- src/embedder/providers/base.py | 39 --------------------- src/embedder/providers/github.py | 56 +++++++++++++++++++++++------- src/embedder/providers/local.py | 3 +- src/embedder/refs.py | 54 ---------------------------- src/embedder/updater.py | 12 +++---- tests/test_blocks.py | 3 +- tests/test_refs.py | 3 +- tests/test_updater.py | 24 ++++++------- 14 files changed, 151 insertions(+), 192 deletions(-) create mode 100644 src/embedder/errors.py delete mode 100644 src/embedder/formats/base.py delete mode 100644 src/embedder/providers/base.py delete mode 100644 src/embedder/refs.py diff --git a/src/embedder/blocks.py b/src/embedder/blocks.py index 346da6c..eda2fcc 100644 --- a/src/embedder/blocks.py +++ b/src/embedder/blocks.py @@ -4,7 +4,9 @@ from dataclasses import dataclass from pathlib import Path -from embedder.refs import RefError +from embedder.errors import EmbedderEnvironmentError, EmbedderError, RefError +from embedder.formats import get_format +from embedder.providers import parse_ref SKIP_DIRS = { ".git", @@ -20,14 +22,6 @@ } -class EmbedderError(Exception): - pass - - -class EmbedderEnvironmentError(EmbedderError): - pass - - @dataclass(frozen=True) class EmbedderBlock: path: str @@ -45,12 +39,10 @@ class BlockUpdate: def parse_blocks(path: Path, text: str) -> list[EmbedderBlock]: - from embedder.formats import get_format - from embedder.providers import parse_ref - fmt = get_format(path) if fmt is None: return [] + scanner = fmt.make_scanner() lines = text.splitlines(keepends=True) blocks: list[EmbedderBlock] = [] @@ -139,14 +131,13 @@ def scan_paths(paths: list[Path]) -> list[EmbedderBlock]: def apply_updates(path: Path, text: str, updates: list[BlockUpdate]) -> str: - from embedder.formats import get_format - if not updates: return text fmt = get_format(path) if fmt is None: return text + lines = text.splitlines(keepends=True) output: list[str] = [] cursor = 0 @@ -177,3 +168,15 @@ def apply_updates(path: Path, text: str, updates: list[BlockUpdate]) -> str: output.extend(lines[cursor:]) return "".join(output) + + +__all__ = [ + "BlockUpdate", + "EmbedderBlock", + "EmbedderEnvironmentError", + "EmbedderError", + "apply_updates", + "iter_files", + "parse_blocks", + "scan_paths", +] diff --git a/src/embedder/cli.py b/src/embedder/cli.py index a319cf0..a7d03f1 100644 --- a/src/embedder/cli.py +++ b/src/embedder/cli.py @@ -9,12 +9,8 @@ from typing import Any from embedder import __version__ -from embedder.blocks import ( - EmbedderBlock, - EmbedderEnvironmentError, - EmbedderError, - scan_paths, -) +from embedder.blocks import EmbedderBlock, scan_paths +from embedder.errors import EmbedderEnvironmentError, EmbedderError from embedder.providers.github import GitHubProvider from embedder.updater import check_blocks, update_files diff --git a/src/embedder/errors.py b/src/embedder/errors.py new file mode 100644 index 0000000..fd790d3 --- /dev/null +++ b/src/embedder/errors.py @@ -0,0 +1,13 @@ +from __future__ import annotations + + +class EmbedderError(Exception): + pass + + +class EmbedderEnvironmentError(EmbedderError): + pass + + +class RefError(ValueError): + pass diff --git a/src/embedder/formats/__init__.py b/src/embedder/formats/__init__.py index 93bd8fe..e6eb1e2 100644 --- a/src/embedder/formats/__init__.py +++ b/src/embedder/formats/__init__.py @@ -1,11 +1,35 @@ from __future__ import annotations +import re from pathlib import Path +from typing import Protocol, runtime_checkable -from embedder.formats.base import Format from embedder.formats.markdown import MarkdownFormat from embedder.formats.yaml import YamlFormat + +@runtime_checkable +class LineScanner(Protocol): + def advance(self, line: str) -> bool: ... + + +@runtime_checkable +class Format(Protocol): + @property + def open_re(self) -> re.Pattern[str]: ... + + @property + def close_re(self) -> re.Pattern[str]: ... + + def matches(self, path: Path) -> bool: ... + + def make_scanner(self) -> LineScanner: ... + + def render_open(self, ref_str: str) -> str: ... + + def render_close(self) -> str: ... + + _FORMATS: list[Format] = [ MarkdownFormat(), YamlFormat(), diff --git a/src/embedder/formats/base.py b/src/embedder/formats/base.py deleted file mode 100644 index 6941806..0000000 --- a/src/embedder/formats/base.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path -from typing import Protocol, runtime_checkable - - -@runtime_checkable -class LineScanner(Protocol): - def advance(self, line: str) -> bool: - """Advance by one line. Returns True if the line should be skipped.""" - ... - - -@runtime_checkable -class Format(Protocol): - @property - def open_re(self) -> re.Pattern[str]: ... - - @property - def close_re(self) -> re.Pattern[str]: ... - - def matches(self, path: Path) -> bool: ... - - def make_scanner(self) -> LineScanner: ... - - def render_open(self, ref_str: str) -> str: ... - - def render_close(self) -> str: ... diff --git a/src/embedder/providers/__init__.py b/src/embedder/providers/__init__.py index 8d66071..3739f8a 100644 --- a/src/embedder/providers/__init__.py +++ b/src/embedder/providers/__init__.py @@ -1,29 +1,41 @@ from __future__ import annotations -from embedder.providers.base import Provider -from embedder.providers.github import GitHubProvider +from pathlib import Path +from typing import Protocol, runtime_checkable + +from embedder.errors import RefError +from embedder.providers.github import GitHubAssetRef, GitHubProvider from embedder.providers.local import LocalProvider, LocalRef -from embedder.refs import GitHubAssetRef, RefError AnyRef = GitHubAssetRef | LocalRef -class ProviderRegistry: - def __init__(self, providers: list[Provider]) -> None: - self._providers = providers +@runtime_checkable +class Provider(Protocol): + def matches(self, raw: str) -> bool: ... + + def parse_ref(self, raw: str) -> AnyRef: ... + + def resolve(self, ref: AnyRef) -> AnyRef: ... + + def resolve_cached(self, ref: AnyRef, cached: AnyRef) -> AnyRef: ... + + def always_refresh(self, ref: AnyRef) -> bool: ... + + def fetch(self, ref: AnyRef, base_dir: Path) -> str: ... + + def cache_key(self, ref: AnyRef) -> str | None: ... - def get(self, raw: str) -> Provider: - for provider in self._providers: - if provider.matches(raw): - return provider - raise RefError(f"Unknown ref scheme: {raw!r}") - def parse_ref(self, raw: str) -> AnyRef: - return self.get(raw).parse_ref(raw) +DEFAULT_PROVIDERS: list[Provider] = [GitHubProvider(), LocalProvider()] -DEFAULT_REGISTRY = ProviderRegistry([GitHubProvider(), LocalProvider()]) +def get_provider(raw: str, providers: list[Provider] = DEFAULT_PROVIDERS) -> Provider: + for provider in providers: + if provider.matches(raw): + return provider + raise RefError(f"Unknown ref scheme: {raw!r}") def parse_ref(raw: str) -> AnyRef: - return DEFAULT_REGISTRY.parse_ref(raw) + return get_provider(raw).parse_ref(raw) diff --git a/src/embedder/providers/base.py b/src/embedder/providers/base.py deleted file mode 100644 index b0b3b62..0000000 --- a/src/embedder/providers/base.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import TYPE_CHECKING, Protocol, runtime_checkable - -if TYPE_CHECKING: - from embedder.providers import AnyRef - - -@runtime_checkable -class Provider(Protocol): - def matches(self, raw: str) -> bool: ... - - def parse_ref(self, raw: str) -> AnyRef: ... - - def resolve(self, ref: AnyRef) -> AnyRef: - """Return the ref updated to the latest available version.""" - ... - - def resolve_cached(self, ref: AnyRef, cached: AnyRef) -> AnyRef: - """Apply the version from a previously resolved ref to a different ref. - - Used to reuse a cached provider lookup (e.g. latest tag) for a block - with the same cache key but a different asset or path. - """ - ... - - def always_refresh(self, ref: AnyRef) -> bool: - """Return True if this ref should always be re-fetched on update, - even when resolve() reports no version change.""" - ... - - def fetch(self, ref: AnyRef, base_dir: Path) -> str: - """Return the fragment content.""" - ... - - def cache_key(self, ref: AnyRef) -> str | None: - """Deduplication key for resolve() calls. None means no caching.""" - ... diff --git a/src/embedder/providers/github.py b/src/embedder/providers/github.py index 3947023..ab01eac 100644 --- a/src/embedder/providers/github.py +++ b/src/embedder/providers/github.py @@ -1,13 +1,54 @@ from __future__ import annotations +import re import shutil import subprocess import tempfile from dataclasses import dataclass from pathlib import Path -from embedder.blocks import EmbedderEnvironmentError, EmbedderError -from embedder.refs import GitHubAssetRef, parse_github_ref +from embedder.errors import EmbedderEnvironmentError, EmbedderError, RefError + +_REF_RE = re.compile( + r"^github\.com/" + r"(?P[A-Za-z0-9_.-]+)/" + r"(?P[A-Za-z0-9_.-]+)" + r"@(?P[^:\s]+)" + r":(?P[^\s]+)$" +) + + +@dataclass(frozen=True) +class GitHubAssetRef: + owner: str + repo: str + tag: str + asset: str + + @property + def repository(self) -> str: + return f"{self.owner}/{self.repo}" + + def with_tag(self, tag: str) -> GitHubAssetRef: + return GitHubAssetRef(owner=self.owner, repo=self.repo, tag=tag, asset=self.asset) + + def render(self) -> str: + return f"github.com/{self.repository}@{self.tag}:{self.asset}" + + +def parse_github_ref(raw: str) -> GitHubAssetRef: + match = _REF_RE.match(raw.strip()) + if not match: + raise RefError(f"Invalid embedder ref: {raw}") + asset = match["asset"] + if "/" in asset or "\\" in asset: + raise RefError(f"Release asset must be a basename: {asset}") + return GitHubAssetRef( + owner=match["owner"], + repo=match["repo"], + tag=match["tag"], + asset=asset, + ) @dataclass(frozen=True) @@ -68,16 +109,7 @@ def auth_ok(self) -> bool: def _latest_tag(self, ref: GitHubAssetRef) -> str: result = self.run( - [ - "release", - "view", - "--repo", - ref.repository, - "--json", - "tagName", - "--jq", - ".tagName", - ] + ["release", "view", "--repo", ref.repository, "--json", "tagName", "--jq", ".tagName"] ) if not result.stdout or result.stdout == "null": raise EmbedderError(f"Could not resolve latest release for {ref.repository}") diff --git a/src/embedder/providers/local.py b/src/embedder/providers/local.py index 085fd6c..fe73da0 100644 --- a/src/embedder/providers/local.py +++ b/src/embedder/providers/local.py @@ -3,8 +3,7 @@ from dataclasses import dataclass from pathlib import Path -from embedder.blocks import EmbedderError -from embedder.refs import RefError +from embedder.errors import EmbedderError, RefError @dataclass(frozen=True) diff --git a/src/embedder/refs.py b/src/embedder/refs.py deleted file mode 100644 index 67054d2..0000000 --- a/src/embedder/refs.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass - -REF_RE = re.compile( - r"^github\.com/" - r"(?P[A-Za-z0-9_.-]+)/" - r"(?P[A-Za-z0-9_.-]+)" - r"@(?P[^:\s]+)" - r":(?P[^\s]+)$" -) - - -class RefError(ValueError): - pass - - -@dataclass(frozen=True) -class GitHubAssetRef: - owner: str - repo: str - tag: str - asset: str - - @property - def repository(self) -> str: - return f"{self.owner}/{self.repo}" - - def with_tag(self, tag: str) -> GitHubAssetRef: - return GitHubAssetRef( - owner=self.owner, - repo=self.repo, - tag=tag, - asset=self.asset, - ) - - def render(self) -> str: - return f"github.com/{self.repository}@{self.tag}:{self.asset}" - - -def parse_github_ref(raw: str) -> GitHubAssetRef: - match = REF_RE.match(raw.strip()) - if not match: - raise RefError(f"Invalid embedder ref: {raw}") - asset = match["asset"] - if "/" in asset or "\\" in asset: - raise RefError(f"Release asset must be a basename: {asset}") - return GitHubAssetRef( - owner=match["owner"], - repo=match["repo"], - tag=match["tag"], - asset=asset, - ) diff --git a/src/embedder/updater.py b/src/embedder/updater.py index 1f707d5..cf6d766 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -10,7 +10,7 @@ iter_files, parse_blocks, ) -from embedder.providers import DEFAULT_REGISTRY, AnyRef, ProviderRegistry +from embedder.providers import DEFAULT_PROVIDERS, AnyRef, Provider, get_provider @dataclass(frozen=True) @@ -31,13 +31,13 @@ class FileUpdate: def check_blocks( blocks: list[EmbedderBlock], - registry: ProviderRegistry = DEFAULT_REGISTRY, + providers: list[Provider] = DEFAULT_PROVIDERS, ) -> list[CheckResult]: resolved_cache: dict[str, AnyRef] = {} results: list[CheckResult] = [] for block in blocks: - provider = registry.get(block.ref.render()) + provider = get_provider(block.ref.render(), providers) key = provider.cache_key(block.ref) if key is not None: if key in resolved_cache: @@ -54,7 +54,7 @@ def check_blocks( def update_files( paths: list[Path], - registry: ProviderRegistry = DEFAULT_REGISTRY, + providers: list[Provider] = DEFAULT_PROVIDERS, ) -> list[FileUpdate]: changed: list[FileUpdate] = [] @@ -67,12 +67,12 @@ def update_files( if not blocks: continue - checks = check_blocks(blocks, registry) + checks = check_blocks(blocks, providers) updates: list[BlockUpdate] = [] changed_checks: list[CheckResult] = [] for check in checks: - provider = registry.get(check.block.ref.render()) + provider = get_provider(check.block.ref.render(), providers) if not check.update_available and not provider.always_refresh(check.block.ref): continue new_body = provider.fetch(check.latest_ref, path.parent) diff --git a/tests/test_blocks.py b/tests/test_blocks.py index c3bf8f4..204f73e 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -2,7 +2,8 @@ import pytest -from embedder.blocks import BlockUpdate, EmbedderError, apply_updates, parse_blocks +from embedder.blocks import BlockUpdate, apply_updates, parse_blocks +from embedder.errors import EmbedderError from tests.helpers import close_marker, marker, yaml_close_marker, yaml_marker diff --git a/tests/test_refs.py b/tests/test_refs.py index 0c33bd8..6a6464d 100644 --- a/tests/test_refs.py +++ b/tests/test_refs.py @@ -1,8 +1,9 @@ import pytest +from embedder.errors import RefError from embedder.providers import parse_ref +from embedder.providers.github import GitHubAssetRef, parse_github_ref from embedder.providers.local import LocalRef -from embedder.refs import GitHubAssetRef, RefError, parse_github_ref def test_parse_github_ref() -> None: diff --git a/tests/test_updater.py b/tests/test_updater.py index 710ce08..a6e3f81 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -2,10 +2,11 @@ import pytest -from embedder.blocks import BlockUpdate, EmbedderError, parse_blocks -from embedder.providers import ProviderRegistry +from embedder.blocks import BlockUpdate, parse_blocks +from embedder.errors import EmbedderError +from embedder.providers import Provider +from embedder.providers.github import GitHubAssetRef, parse_github_ref from embedder.providers.local import LocalProvider, LocalRef -from embedder.refs import GitHubAssetRef, parse_github_ref from embedder.updater import check_blocks, update_files from tests.helpers import close_marker, marker @@ -41,8 +42,8 @@ def cache_key(self, ref: GitHubAssetRef) -> str: return ref.repository -def fake_registry(**kwargs) -> ProviderRegistry: - return ProviderRegistry([FakeGitHubProvider(**kwargs) if kwargs else FakeGitHubProvider(), LocalProvider()]) +def fake_providers() -> list[Provider]: + return [FakeGitHubProvider(), LocalProvider()] def test_check_blocks_marks_updates() -> None: @@ -56,7 +57,7 @@ def test_check_blocks_marks_updates() -> None: ) blocks = parse_blocks(Path("AGENTS.md"), text) - results = check_blocks(blocks, fake_registry()) + results = check_blocks(blocks, fake_providers()) assert len(results) == 1 assert results[0].latest_ref.render() == "github.com/rubykatzen/embedder@v0.2.0:fragment.md" @@ -79,7 +80,7 @@ def test_update_files_replaces_only_managed_body(tmp_path: Path) -> None: encoding="utf-8", ) - changed = update_files([tmp_path], fake_registry()) + changed = update_files([tmp_path], fake_providers()) assert [item.path for item in changed] == [str(target)] assert target.read_text(encoding="utf-8") == "\n".join( @@ -138,9 +139,8 @@ def test_check_blocks_caches_resolve_per_repository() -> None: ) blocks = parse_blocks(Path("AGENTS.md"), text) provider = FakeGitHubProvider() - registry = ProviderRegistry([provider, LocalProvider()]) - check_blocks(blocks, registry) + check_blocks(blocks, [provider, LocalProvider()]) assert provider.resolve_calls == 1 @@ -162,7 +162,7 @@ def test_local_ref_is_always_current(tmp_path: Path) -> None: encoding="utf-8", ) - registry = ProviderRegistry([FakeGitHubProvider(), LocalProvider()]) + registry = [FakeGitHubProvider(), LocalProvider()] results = check_blocks(parse_blocks(target, target.read_text(encoding="utf-8")), registry) assert not results[0].update_available @@ -192,7 +192,7 @@ def test_cache_uses_correct_asset_per_block() -> None: ] ) blocks = parse_blocks(Path("AGENTS.md"), text) - registry = ProviderRegistry([FakeGitHubProvider(), LocalProvider()]) + registry = [FakeGitHubProvider(), LocalProvider()] results = check_blocks(blocks, registry) @@ -212,7 +212,7 @@ def test_local_ref_body_refreshed_on_update(tmp_path: Path) -> None: ) fragment.write_text("v2 content\n", encoding="utf-8") - registry = ProviderRegistry([FakeGitHubProvider(), LocalProvider()]) + registry = [FakeGitHubProvider(), LocalProvider()] update_files([tmp_path], registry) assert "v2 content" in target.read_text(encoding="utf-8") From 6c9628fc449395c4176318c579566453a14aafbf Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:05:54 +0200 Subject: [PATCH 08/25] fix: address second round of review feedback - YAML scanner: extend _BLOCK_SCALAR_RE to match sequence block scalars (- | and - >) in addition to mapping values (key: |), so markers inside sequence scalar content are correctly ignored - parse_blocks: accept optional providers list so custom provider schemes work during scanning; thread through scan_paths and update_files - Replace mutable list defaults with None sentinel in check_blocks, update_files, and get_provider to prevent accidental cross-call mutation - update-fragments-shared.yml: align jq ref rendering with update-fragments.yml (handles both GitHub and local refs, removes null fields) Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/update-fragments-shared.yml | 4 +- src/embedder/blocks.py | 20 ++++-- src/embedder/formats/yaml.py | 6 +- src/embedder/providers/__init__.py | 5 +- src/embedder/updater.py | 14 ++-- tests/test_blocks.py | 66 +++++++++++++++++++ tests/test_updater.py | 19 ++++++ 7 files changed, 117 insertions(+), 17 deletions(-) diff --git a/.github/workflows/update-fragments-shared.yml b/.github/workflows/update-fragments-shared.yml index 1d71f71..7231135 100644 --- a/.github/workflows/update-fragments-shared.yml +++ b/.github/workflows/update-fragments-shared.yml @@ -52,9 +52,9 @@ jobs: git commit -m "chore: update embedded fragments" git push origin "$branch" --force body=$(jq -r ' - "## Embedded fragment updates\n\n| File | Source | Asset | Old | New |\n|---|---|---|---:|---:|\n" + + "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(.block.ref.owner)/\(.block.ref.repo) | \(.block.ref.asset) | \(.block.ref.tag) | \(.latest_ref.tag // .latest_tag) |" + "| \(.block.path) | \(if .block.ref.owner then "github.com/\(.block.ref.owner)/\(.block.ref.repo)@\(.block.ref.tag):\(.block.ref.asset)" else "local:\(.block.ref.path)" end) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" ] | join("\n")) ' /tmp/embedder-result.json) existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') diff --git a/src/embedder/blocks.py b/src/embedder/blocks.py index eda2fcc..49697d8 100644 --- a/src/embedder/blocks.py +++ b/src/embedder/blocks.py @@ -6,7 +6,7 @@ from embedder.errors import EmbedderEnvironmentError, EmbedderError, RefError from embedder.formats import get_format -from embedder.providers import parse_ref +from embedder.providers import DEFAULT_PROVIDERS, Provider, get_provider SKIP_DIRS = { ".git", @@ -38,7 +38,12 @@ class BlockUpdate: new_body: str -def parse_blocks(path: Path, text: str) -> list[EmbedderBlock]: +def parse_blocks( + path: Path, + text: str, + providers: list[Provider] | None = None, +) -> list[EmbedderBlock]: + _providers = providers if providers is not None else DEFAULT_PROVIDERS fmt = get_format(path) if fmt is None: return [] @@ -61,7 +66,9 @@ def parse_blocks(path: Path, text: str) -> list[EmbedderBlock]: f"{path}:{index + 1}: nested embedder block is not allowed" ) try: - active_ref = parse_ref(open_match["ref"]) + active_ref = get_provider(open_match["ref"], _providers).parse_ref( + open_match["ref"] + ) except RefError as error: raise EmbedderError(f"{path}:{index + 1}: {error}") from error active_start = index @@ -115,7 +122,10 @@ def iter_files(paths: list[Path]) -> list[Path]: return sorted(files) -def scan_paths(paths: list[Path]) -> list[EmbedderBlock]: +def scan_paths( + paths: list[Path], + providers: list[Provider] | None = None, +) -> list[EmbedderBlock]: blocks: list[EmbedderBlock] = [] for path in iter_files(paths): if not is_probably_text(path): @@ -126,7 +136,7 @@ def scan_paths(paths: list[Path]) -> list[EmbedderBlock]: continue except OSError as error: raise EmbedderError(f"Could not read {path}: {error}") from error - blocks.extend(parse_blocks(path, text)) + blocks.extend(parse_blocks(path, text, providers)) return blocks diff --git a/src/embedder/formats/yaml.py b/src/embedder/formats/yaml.py index 356c59a..aaffeeb 100644 --- a/src/embedder/formats/yaml.py +++ b/src/embedder/formats/yaml.py @@ -6,10 +6,12 @@ OPEN_RE = re.compile(r"^\s*#\s+embedder\s+(?P\S+)\s*$") CLOSE_RE = re.compile(r"^\s*#\s+/embedder\s*$") -# Matches YAML block scalar headers: key: |, key: |2, key: |-2, key: | # note, etc. +# Matches YAML block scalar headers in mapping values (key: |) and sequence items (- |). # Per YAML spec, after | or > comes an optional indentation indicator (digit), # optional chomp indicator (+/-), and an optional trailing comment. -_BLOCK_SCALAR_RE = re.compile(r"^(?P\s*)\S.*:\s*[|>][0-9]?[+-]?[0-9]?(?:\s+#.*)?\s*$") +_BLOCK_SCALAR_RE = re.compile( + r"^(?P\s*)(?:\S.*:\s*|-\s*)[|>][0-9]?[+-]?[0-9]?(?:\s+#.*)?\s*$" +) class _YamlScanner: diff --git a/src/embedder/providers/__init__.py b/src/embedder/providers/__init__.py index 3739f8a..a7a7711 100644 --- a/src/embedder/providers/__init__.py +++ b/src/embedder/providers/__init__.py @@ -30,8 +30,9 @@ def cache_key(self, ref: AnyRef) -> str | None: ... DEFAULT_PROVIDERS: list[Provider] = [GitHubProvider(), LocalProvider()] -def get_provider(raw: str, providers: list[Provider] = DEFAULT_PROVIDERS) -> Provider: - for provider in providers: +def get_provider(raw: str, providers: list[Provider] | None = None) -> Provider: + _providers = providers if providers is not None else DEFAULT_PROVIDERS + for provider in _providers: if provider.matches(raw): return provider raise RefError(f"Unknown ref scheme: {raw!r}") diff --git a/src/embedder/updater.py b/src/embedder/updater.py index cf6d766..39de917 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -31,13 +31,14 @@ class FileUpdate: def check_blocks( blocks: list[EmbedderBlock], - providers: list[Provider] = DEFAULT_PROVIDERS, + providers: list[Provider] | None = None, ) -> list[CheckResult]: + _providers = providers if providers is not None else DEFAULT_PROVIDERS resolved_cache: dict[str, AnyRef] = {} results: list[CheckResult] = [] for block in blocks: - provider = get_provider(block.ref.render(), providers) + provider = get_provider(block.ref.render(), _providers) key = provider.cache_key(block.ref) if key is not None: if key in resolved_cache: @@ -54,8 +55,9 @@ def check_blocks( def update_files( paths: list[Path], - providers: list[Provider] = DEFAULT_PROVIDERS, + providers: list[Provider] | None = None, ) -> list[FileUpdate]: + _providers = providers if providers is not None else DEFAULT_PROVIDERS changed: list[FileUpdate] = [] for path in iter_files(paths): @@ -63,16 +65,16 @@ def update_files( text = path.read_text(encoding="utf-8") except UnicodeDecodeError: continue - blocks = parse_blocks(path, text) + blocks = parse_blocks(path, text, _providers) if not blocks: continue - checks = check_blocks(blocks, providers) + checks = check_blocks(blocks, _providers) updates: list[BlockUpdate] = [] changed_checks: list[CheckResult] = [] for check in checks: - provider = get_provider(check.block.ref.render(), providers) + provider = get_provider(check.block.ref.render(), _providers) if not check.update_available and not provider.always_refresh(check.block.ref): continue new_body = provider.fetch(check.latest_ref, path.parent) diff --git a/tests/test_blocks.py b/tests/test_blocks.py index 204f73e..e765ff5 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -190,6 +190,72 @@ def test_parse_local_ref_in_markdown() -> None: assert blocks[0].ref.render() == "local:fragments/file.md" +def test_yaml_ignores_markers_in_sequence_block_scalar() -> None: + text = "\n".join( + [ + "items:", + " - |", + f" {yaml_marker('local:file.yaml')}", + " some content", + f" {yaml_close_marker()}", + "other: value", + "", + ] + ) + + assert parse_blocks(Path("config.yaml"), text) == [] + + +def test_parse_blocks_uses_injected_providers() -> None: + from dataclasses import dataclass + from pathlib import Path as _Path + + @dataclass(frozen=True) + class FakeRef: + name: str + + def render(self) -> str: + return f"fake:{self.name}" + + class FakeProvider: + def matches(self, raw: str) -> bool: + return raw.startswith("fake:") + + def parse_ref(self, raw: str) -> FakeRef: + return FakeRef(name=raw[len("fake:"):]) + + def resolve(self, ref: FakeRef) -> FakeRef: + return ref + + def resolve_cached(self, ref: FakeRef, cached: FakeRef) -> FakeRef: + return ref + + def always_refresh(self, ref: FakeRef) -> bool: + return False + + def fetch(self, ref: FakeRef, base_dir: _Path) -> str: + return "" + + def cache_key(self, ref: FakeRef) -> str | None: + return None + + text = "\n".join( + [ + marker("fake:thing"), + "body", + close_marker(), + "", + ] + ) + + with pytest.raises(EmbedderError): + parse_blocks(Path("README.md"), text) + + blocks = parse_blocks(Path("README.md"), text, providers=[FakeProvider()]) + assert len(blocks) == 1 + assert blocks[0].ref.render() == "fake:thing" + + def test_apply_updates_indents_body_to_match_yaml_marker() -> None: text = "\n".join( [ diff --git a/tests/test_updater.py b/tests/test_updater.py index a6e3f81..b292c24 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -222,3 +222,22 @@ def test_local_ref_path_traversal_rejected(tmp_path: Path) -> None: ref = LocalRef(path="../../etc/passwd") with pytest.raises(EmbedderError, match="escapes base directory"): LocalProvider().fetch(ref, tmp_path) + + +def test_default_providers_not_mutated_between_calls() -> None: + from embedder.providers import DEFAULT_PROVIDERS + + text = "\n".join( + [ + marker("github.com/rubykatzen/embedder@v0.1.0:fragment.md"), + "old", + close_marker(), + "", + ] + ) + blocks = parse_blocks(Path("AGENTS.md"), text) + + providers_before = list(DEFAULT_PROVIDERS) + check_blocks(blocks, fake_providers()) + check_blocks(blocks, fake_providers()) + assert list(DEFAULT_PROVIDERS) == providers_before From 9f38b553a54257548fca80234102d289b65677e1 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:15:06 +0200 Subject: [PATCH 09/25] chore: add dependabot config with local embedder fragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fragments/dependabot-updates.yaml holds the canonical list of update targets; .github/dependabot.yml embeds it via a local: ref. Demonstrates YAML format support and local provider working end-to-end — the body (unindented in the fragment) is automatically re-indented to match the two-space marker indent under updates: on apply. Co-Authored-By: Claude Sonnet 4.6 --- .github/dependabot.yml | 12 ++++++++++++ fragments/dependabot-updates.yaml | 8 ++++++++ 2 files changed, 20 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 fragments/dependabot-updates.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d70548e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + # embedder local:fragments/dependabot-updates.yaml + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + # /embedder diff --git a/fragments/dependabot-updates.yaml b/fragments/dependabot-updates.yaml new file mode 100644 index 0000000..96afbc6 --- /dev/null +++ b/fragments/dependabot-updates.yaml @@ -0,0 +1,8 @@ +- package-ecosystem: pip + directory: "/" + schedule: + interval: weekly +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly From 520f0fcc3582dce1768b049e3834643a9a234dcf Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:17:23 +0200 Subject: [PATCH 10/25] chore: move entire dependabot config into a single local fragment fragments/dependabot.yaml holds the canonical config; .github/dependabot.yml embeds it in full via local:fragments/dependabot.yaml. Co-Authored-By: Claude Sonnet 4.6 --- .github/dependabot.yml | 4 ++-- fragments/dependabot.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 fragments/dependabot.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d70548e..76e2b73 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,6 @@ +# embedder local:fragments/dependabot.yaml version: 2 updates: - # embedder local:fragments/dependabot-updates.yaml - package-ecosystem: pip directory: "/" schedule: @@ -9,4 +9,4 @@ updates: directory: "/" schedule: interval: weekly - # /embedder +# /embedder diff --git a/fragments/dependabot.yaml b/fragments/dependabot.yaml new file mode 100644 index 0000000..563cc5c --- /dev/null +++ b/fragments/dependabot.yaml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly From a68409bb5720b1d4a721ea1a324fac1563505c18 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:22:03 +0200 Subject: [PATCH 11/25] chore: refresh local fragments during release preparation Runs embedder update before pushing the release branch so that any local fragments (e.g. dependabot.yml) are up-to-date in the release. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/prepare-release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 8aaa5bd..f4426ad 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -39,6 +39,13 @@ jobs: - uses: rubykatzen/releaser/.github/actions/bump-pyproject-version@v0.3.4 with: version: ${{ inputs.version }} + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Refresh local fragments + run: | + pip install -e . + embedder update . - uses: rubykatzen/releaser/.github/actions/push-release-branch@v0.3.4 with: version: ${{ inputs.version }} From 18d40748469d399ef570fbb7a14a7a2c05a095d6 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:27:00 +0200 Subject: [PATCH 12/25] feat: add --local-only flag to embedder update update_files(local_only=True) refreshes blocks whose provider has always_refresh=True (i.e. local refs) without calling resolve() on external providers. GitHub refs are left completely untouched. Used in prepare-release.yml so local fragments (e.g. dependabot.yml) are synced from disk without bumping external dependency versions. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/prepare-release.yml | 2 +- src/embedder/cli.py | 7 ++++- src/embedder/updater.py | 8 ++++- tests/test_updater.py | 45 +++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index f4426ad..1463c9b 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -45,7 +45,7 @@ jobs: - name: Refresh local fragments run: | pip install -e . - embedder update . + embedder update . --local-only - uses: rubykatzen/releaser/.github/actions/push-release-branch@v0.3.4 with: version: ${{ inputs.version }} diff --git a/src/embedder/cli.py b/src/embedder/cli.py index a7d03f1..cd3c396 100644 --- a/src/embedder/cli.py +++ b/src/embedder/cli.py @@ -75,7 +75,7 @@ def command_check(args: argparse.Namespace) -> int: def command_update(args: argparse.Namespace) -> int: - changed = update_files([Path(path) for path in args.paths]) + changed = update_files([Path(path) for path in args.paths], local_only=args.local_only) if args.json: to_json({"changed": changed}) return int(ExitCode.OK) @@ -141,6 +141,11 @@ def parser() -> argparse.ArgumentParser: help="Files or directories to update. Defaults to the current directory.", ) update.add_argument("--json", action="store_true", help="Print machine-readable JSON output.") + update.add_argument( + "--local-only", + action="store_true", + help="Refresh local fragments only; skip external provider resolution.", + ) update.set_defaults(func=command_update) doctor = subparsers.add_parser("doctor", help="Check local embedder prerequisites.") diff --git a/src/embedder/updater.py b/src/embedder/updater.py index 39de917..0489df6 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -56,6 +56,8 @@ def check_blocks( def update_files( paths: list[Path], providers: list[Provider] | None = None, + *, + local_only: bool = False, ) -> list[FileUpdate]: _providers = providers if providers is not None else DEFAULT_PROVIDERS changed: list[FileUpdate] = [] @@ -69,12 +71,16 @@ def update_files( if not blocks: continue - checks = check_blocks(blocks, _providers) + checks = check_blocks(blocks, _providers) if not local_only else [ + CheckResult(block=block, latest_ref=block.ref) for block in blocks + ] updates: list[BlockUpdate] = [] changed_checks: list[CheckResult] = [] for check in checks: provider = get_provider(check.block.ref.render(), _providers) + if local_only and not provider.always_refresh(check.block.ref): + continue if not check.update_available and not provider.always_refresh(check.block.ref): continue new_body = provider.fetch(check.latest_ref, path.parent) diff --git a/tests/test_updater.py b/tests/test_updater.py index b292c24..88e1961 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -224,6 +224,51 @@ def test_local_ref_path_traversal_rejected(tmp_path: Path) -> None: LocalProvider().fetch(ref, tmp_path) +def test_local_only_skips_github_blocks(tmp_path: Path) -> None: + """update_files(local_only=True) refreshes local refs but leaves GitHub refs untouched.""" + fragment = tmp_path / "frag.md" + fragment.write_text("new local\n", encoding="utf-8") + + target = tmp_path / "README.md" + target.write_text( + "\n".join( + [ + marker("github.com/rubykatzen/embedder@v0.1.0:fragment.md"), + "old github", + close_marker(), + marker("local:frag.md"), + "old local", + close_marker(), + "", + ] + ), + encoding="utf-8", + ) + + update_files([tmp_path], fake_providers(), local_only=True) + + result = target.read_text(encoding="utf-8") + assert "old github" in result + assert "new local" in result + + +def test_local_only_does_not_call_github_resolve(tmp_path: Path) -> None: + """update_files(local_only=True) makes no resolve() calls to external providers.""" + fragment = tmp_path / "frag.md" + fragment.write_text("content\n", encoding="utf-8") + + target = tmp_path / "README.md" + target.write_text( + "\n".join([marker("local:frag.md"), "old", close_marker(), ""]), + encoding="utf-8", + ) + + provider = FakeGitHubProvider() + update_files([tmp_path], [provider, LocalProvider()], local_only=True) + + assert provider.resolve_calls == 0 + + def test_default_providers_not_mutated_between_calls() -> None: from embedder.providers import DEFAULT_PROVIDERS From c935e62fc67a7e280ac7a72f7aa17de84b54dcd9 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:29:48 +0200 Subject: [PATCH 13/25] feat: extract refresh-local-fragments composite action Wraps pip install + embedder update --local-only into a reusable composite action. prepare-release.yml uses it via ./.github/actions/ and external repos can reference it by tag. The action installs embedder from its own source tree via github.action_path so no separate install step is needed. Co-Authored-By: Claude Sonnet 4.6 --- .../refresh-local-fragments/action.yml | 25 +++++++++++++++++++ .github/workflows/prepare-release.yml | 8 +----- 2 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 .github/actions/refresh-local-fragments/action.yml diff --git a/.github/actions/refresh-local-fragments/action.yml b/.github/actions/refresh-local-fragments/action.yml new file mode 100644 index 0000000..9e7d0de --- /dev/null +++ b/.github/actions/refresh-local-fragments/action.yml @@ -0,0 +1,25 @@ +name: Refresh local fragments +description: Re-embed all local: fragment refs in the workspace without bumping external dependencies. +inputs: + paths: + description: Files or directories to scan. Defaults to the current directory. + required: false + default: "." + python-version: + description: Python version to use. + required: false + default: "3.13" +runs: + using: composite + steps: + - uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + - name: Refresh local fragments + shell: bash + env: + EMBEDDER_PATHS: ${{ inputs.paths }} + run: | + pip install -e "${{ github.action_path }}/../../.." + read -ra paths <<< "$EMBEDDER_PATHS" + embedder update "${paths[@]}" --local-only diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 1463c9b..8701bc4 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -39,13 +39,7 @@ jobs: - uses: rubykatzen/releaser/.github/actions/bump-pyproject-version@v0.3.4 with: version: ${{ inputs.version }} - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - name: Refresh local fragments - run: | - pip install -e . - embedder update . --local-only + - uses: ./.github/actions/refresh-local-fragments - uses: rubykatzen/releaser/.github/actions/push-release-branch@v0.3.4 with: version: ${{ inputs.version }} From 52d1c46f59e3eafaa85cca99ea0320b5e34fafee Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:33:30 +0200 Subject: [PATCH 14/25] fix: move dependabot fragment to .github/fragments/ local: refs resolve relative to the embedding file's directory, so fragments/dependabot.yaml was invisible from .github/dependabot.yml. Moved to .github/fragments/dependabot.yaml where it resolves correctly. Also fix YAML parse error in action.yml description (unquoted local:). Co-Authored-By: Claude Sonnet 4.6 --- .github/actions/refresh-local-fragments/action.yml | 2 +- .github/fragments/dependabot.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .github/fragments/dependabot.yaml diff --git a/.github/actions/refresh-local-fragments/action.yml b/.github/actions/refresh-local-fragments/action.yml index 9e7d0de..507a543 100644 --- a/.github/actions/refresh-local-fragments/action.yml +++ b/.github/actions/refresh-local-fragments/action.yml @@ -1,5 +1,5 @@ name: Refresh local fragments -description: Re-embed all local: fragment refs in the workspace without bumping external dependencies. +description: "Re-embed all local: fragment refs in the workspace without bumping external dependencies." inputs: paths: description: Files or directories to scan. Defaults to the current directory. diff --git a/.github/fragments/dependabot.yaml b/.github/fragments/dependabot.yaml new file mode 100644 index 0000000..563cc5c --- /dev/null +++ b/.github/fragments/dependabot.yaml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly From 9b1ad0f7c78706642febb1311d542b124e43c6a1 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:38:42 +0200 Subject: [PATCH 15/25] chore: drop setup-python from refresh-local-fragments action Python is already available on the runner; no need to install it. Co-Authored-By: Claude Sonnet 4.6 --- .github/actions/refresh-local-fragments/action.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/actions/refresh-local-fragments/action.yml b/.github/actions/refresh-local-fragments/action.yml index 507a543..8cc727f 100644 --- a/.github/actions/refresh-local-fragments/action.yml +++ b/.github/actions/refresh-local-fragments/action.yml @@ -5,16 +5,9 @@ inputs: description: Files or directories to scan. Defaults to the current directory. required: false default: "." - python-version: - description: Python version to use. - required: false - default: "3.13" runs: using: composite steps: - - uses: actions/setup-python@v6 - with: - python-version: ${{ inputs.python-version }} - name: Refresh local fragments shell: bash env: From e766b25a608d0be3c569fe9252d54ea72120e949 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:39:59 +0200 Subject: [PATCH 16/25] Delete dependabot.yaml --- fragments/dependabot.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 fragments/dependabot.yaml diff --git a/fragments/dependabot.yaml b/fragments/dependabot.yaml deleted file mode 100644 index 563cc5c..0000000 --- a/fragments/dependabot.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -updates: - - package-ecosystem: pip - directory: "/" - schedule: - interval: weekly - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly From 259e1648674a39684ef349b25a6a2e1d461542a7 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:43:58 +0200 Subject: [PATCH 17/25] refactor: resolve local: refs relative to base_dir (default CWD) Previously local: paths resolved relative to each embedding file's parent directory, making cross-directory refs (e.g. from .github/ to fragments/) impossible without path traversal. update_files now accepts an explicit base_dir (default Path.cwd()) and passes it to provider.fetch() for all blocks. This makes all local: refs repo-root-relative when running embedder from the project root. Move fragments/dependabot.yaml back to repo-root fragments/ and remove the orphaned dependabot-updates.yaml fragment. Co-Authored-By: Claude Sonnet 4.6 --- fragments/dependabot-updates.yaml | 8 -------- {.github/fragments => fragments}/dependabot.yaml | 0 src/embedder/updater.py | 4 +++- tests/test_updater.py | 8 ++++---- 4 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 fragments/dependabot-updates.yaml rename {.github/fragments => fragments}/dependabot.yaml (100%) diff --git a/fragments/dependabot-updates.yaml b/fragments/dependabot-updates.yaml deleted file mode 100644 index 96afbc6..0000000 --- a/fragments/dependabot-updates.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- package-ecosystem: pip - directory: "/" - schedule: - interval: weekly -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly diff --git a/.github/fragments/dependabot.yaml b/fragments/dependabot.yaml similarity index 100% rename from .github/fragments/dependabot.yaml rename to fragments/dependabot.yaml diff --git a/src/embedder/updater.py b/src/embedder/updater.py index 0489df6..0f1eb63 100644 --- a/src/embedder/updater.py +++ b/src/embedder/updater.py @@ -58,8 +58,10 @@ def update_files( providers: list[Provider] | None = None, *, local_only: bool = False, + base_dir: Path | None = None, ) -> list[FileUpdate]: _providers = providers if providers is not None else DEFAULT_PROVIDERS + _base_dir = base_dir if base_dir is not None else Path.cwd() changed: list[FileUpdate] = [] for path in iter_files(paths): @@ -83,7 +85,7 @@ def update_files( continue if not check.update_available and not provider.always_refresh(check.block.ref): continue - new_body = provider.fetch(check.latest_ref, path.parent) + new_body = provider.fetch(check.latest_ref, _base_dir) if new_body == check.block.body and not check.update_available: continue updates.append( diff --git a/tests/test_updater.py b/tests/test_updater.py index 88e1961..2800529 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -80,7 +80,7 @@ def test_update_files_replaces_only_managed_body(tmp_path: Path) -> None: encoding="utf-8", ) - changed = update_files([tmp_path], fake_providers()) + changed = update_files([tmp_path], fake_providers(), base_dir=tmp_path) assert [item.path for item in changed] == [str(target)] assert target.read_text(encoding="utf-8") == "\n".join( @@ -213,7 +213,7 @@ def test_local_ref_body_refreshed_on_update(tmp_path: Path) -> None: fragment.write_text("v2 content\n", encoding="utf-8") registry = [FakeGitHubProvider(), LocalProvider()] - update_files([tmp_path], registry) + update_files([tmp_path], registry, base_dir=tmp_path) assert "v2 content" in target.read_text(encoding="utf-8") @@ -245,7 +245,7 @@ def test_local_only_skips_github_blocks(tmp_path: Path) -> None: encoding="utf-8", ) - update_files([tmp_path], fake_providers(), local_only=True) + update_files([tmp_path], fake_providers(), local_only=True, base_dir=tmp_path) result = target.read_text(encoding="utf-8") assert "old github" in result @@ -264,7 +264,7 @@ def test_local_only_does_not_call_github_resolve(tmp_path: Path) -> None: ) provider = FakeGitHubProvider() - update_files([tmp_path], [provider, LocalProvider()], local_only=True) + update_files([tmp_path], [provider, LocalProvider()], local_only=True, base_dir=tmp_path) assert provider.resolve_calls == 0 From 1c4296fb6a80cf33dcfada44ba489bdec5e6ceae Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 21:53:16 +0200 Subject: [PATCH 18/25] refactor: extract PR creation into open-fragments-pr composite action Co-Authored-By: Claude Sonnet 4.6 --- .github/actions/open-fragments-pr/action.yml | 40 +++++++++++++++++++ .github/workflows/update-fragments-shared.yml | 30 +------------- 2 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 .github/actions/open-fragments-pr/action.yml diff --git a/.github/actions/open-fragments-pr/action.yml b/.github/actions/open-fragments-pr/action.yml new file mode 100644 index 0000000..f5b601c --- /dev/null +++ b/.github/actions/open-fragments-pr/action.yml @@ -0,0 +1,40 @@ +name: Open fragments PR +description: "Commit fragment updates and open a pull request." +inputs: + branch: + description: Branch name for the PR. + required: false + default: embedder/update-fragments + result_file: + description: Path to the embedder JSON result file. + required: false + default: /tmp/embedder-result.json +runs: + using: composite + steps: + - name: Commit and open pull request + shell: bash + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ inputs.branch }} + RESULT_FILE: ${{ inputs.result_file }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH" + git add -A + git commit -m "chore: update embedded fragments" + git push origin "$BRANCH" --force + body=$(jq -r ' + "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + + ([.changed[] | .changed_blocks[] | + "| \(.block.path) | \(if .block.ref.owner then "github.com/\(.block.ref.owner)/\(.block.ref.repo)@\(.block.ref.tag):\(.block.ref.asset)" else "local:\(.block.ref.path)" end) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" + ] | join("\n")) + ' "$RESULT_FILE") + existing=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number // empty') + if [ -z "$existing" ]; then + gh pr create \ + --title "chore: update embedded fragments" \ + --body "$body" \ + --head "$BRANCH" + fi diff --git a/.github/workflows/update-fragments-shared.yml b/.github/workflows/update-fragments-shared.yml index 7231135..4373380 100644 --- a/.github/workflows/update-fragments-shared.yml +++ b/.github/workflows/update-fragments-shared.yml @@ -19,16 +19,12 @@ jobs: pull-requests: write steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - name: Install embedder - run: pip install "git+https://github.com/rubykatzen/embedder.git@v0.0.6" - name: Run embedder update env: GH_TOKEN: ${{ secrets.token || github.token }} EMBEDDER_PATHS: ${{ inputs.paths }} run: | + pip install "git+https://github.com/rubykatzen/embedder.git@v0.0.6" read -ra paths <<< "$EMBEDDER_PATHS" embedder update "${paths[@]}" --json > /tmp/embedder-result.json - name: Detect changes @@ -41,26 +37,4 @@ jobs: fi - name: Open pull request if: steps.diff.outputs.changed == 'true' - env: - GH_TOKEN: ${{ github.token }} - run: | - branch="embedder/update-fragments" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "$branch" - git add -A - git commit -m "chore: update embedded fragments" - git push origin "$branch" --force - body=$(jq -r ' - "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + - ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(if .block.ref.owner then "github.com/\(.block.ref.owner)/\(.block.ref.repo)@\(.block.ref.tag):\(.block.ref.asset)" else "local:\(.block.ref.path)" end) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" - ] | join("\n")) - ' /tmp/embedder-result.json) - existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') - if [ -z "$existing" ]; then - gh pr create \ - --title "chore: update embedded fragments" \ - --body "$body" \ - --head "$branch" - fi + uses: ./.github/actions/open-fragments-pr From f16f811c00bce2db2edd0a95b79679b632bd4b01 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:09:57 +0200 Subject: [PATCH 19/25] refactor: align update-fragments.yml with shared workflow pattern Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/update-fragments.yml | 33 ++++---------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/.github/workflows/update-fragments.yml b/.github/workflows/update-fragments.yml index baa0e9f..215caa8 100644 --- a/.github/workflows/update-fragments.yml +++ b/.github/workflows/update-fragments.yml @@ -11,15 +11,12 @@ jobs: pull-requests: write steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - name: Install embedder - run: pip install -e . - name: Run embedder update env: GH_TOKEN: ${{ github.token }} - run: embedder update . --json > /tmp/embedder-result.json + run: | + pip install -e . + embedder update . --json > /tmp/embedder-result.json - name: Detect changes id: diff run: | @@ -30,26 +27,4 @@ jobs: fi - name: Open pull request if: steps.diff.outputs.changed == 'true' - env: - GH_TOKEN: ${{ github.token }} - run: | - branch="embedder/update-fragments" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "$branch" - git add -A - git commit -m "chore: update embedded fragments" - git push origin "$branch" --force - body=$(jq -r ' - "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + - ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(if .block.ref.owner then "github.com/\(.block.ref.owner)/\(.block.ref.repo)@\(.block.ref.tag):\(.block.ref.asset)" else "local:\(.block.ref.path)" end) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" - ] | join("\n")) - ' /tmp/embedder-result.json) - existing=$(gh pr list --head "$branch" --json number --jq '.[0].number // empty') - if [ -z "$existing" ]; then - gh pr create \ - --title "chore: update embedded fragments" \ - --body "$body" \ - --head "$branch" - fi + uses: ./.github/actions/open-fragments-pr From b4a25e90b19b245e8ec3c44ec9478d97ca13707b Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:15:36 +0200 Subject: [PATCH 20/25] chore: extract pytest into separate spec workflow Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/lint.yml | 5 ----- .github/workflows/spec.yml | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/spec.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 66bd173..e39345e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,8 +12,3 @@ jobs: - uses: rubykatzen/baseline/.github/actions/lint-yamllint@v0.5.2 - uses: rubykatzen/baseline/.github/actions/lint-ruff@v0.5.2 - uses: rubykatzen/baseline/.github/actions/lint-actionlint@v0.5.2 - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - run: pip install -e . pytest - - run: pytest diff --git a/.github/workflows/spec.yml b/.github/workflows/spec.yml new file mode 100644 index 0000000..f4c3a3c --- /dev/null +++ b/.github/workflows/spec.yml @@ -0,0 +1,16 @@ +name: Spec +on: + push: + branches: ["main"] + pull_request: + workflow_dispatch: +jobs: + spec: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: pip install -e . pytest + - run: pytest From 48922d93b079ce747d906d800dba55734c7c6080 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:22:36 +0200 Subject: [PATCH 21/25] refactor: simplify PR body to git diff --stat, drop JSON output Co-Authored-By: Claude Sonnet 4.6 --- .github/actions/open-fragments-pr/action.yml | 14 ++------------ .github/workflows/update-fragments-shared.yml | 2 +- .github/workflows/update-fragments.yml | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/actions/open-fragments-pr/action.yml b/.github/actions/open-fragments-pr/action.yml index f5b601c..f547c64 100644 --- a/.github/actions/open-fragments-pr/action.yml +++ b/.github/actions/open-fragments-pr/action.yml @@ -5,10 +5,6 @@ inputs: description: Branch name for the PR. required: false default: embedder/update-fragments - result_file: - description: Path to the embedder JSON result file. - required: false - default: /tmp/embedder-result.json runs: using: composite steps: @@ -17,24 +13,18 @@ runs: env: GH_TOKEN: ${{ github.token }} BRANCH: ${{ inputs.branch }} - RESULT_FILE: ${{ inputs.result_file }} run: | + stat=$(git diff --stat) git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "$BRANCH" git add -A git commit -m "chore: update embedded fragments" git push origin "$BRANCH" --force - body=$(jq -r ' - "## Embedded fragment updates\n\n| File | Ref | Old | New |\n|---|---|---:|---:|\n" + - ([.changed[] | .changed_blocks[] | - "| \(.block.path) | \(if .block.ref.owner then "github.com/\(.block.ref.owner)/\(.block.ref.repo)@\(.block.ref.tag):\(.block.ref.asset)" else "local:\(.block.ref.path)" end) | \(.block.ref.tag // "current") | \(.latest_ref.tag // "current") |" - ] | join("\n")) - ' "$RESULT_FILE") existing=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number // empty') if [ -z "$existing" ]; then gh pr create \ --title "chore: update embedded fragments" \ - --body "$body" \ + --body "$stat" \ --head "$BRANCH" fi diff --git a/.github/workflows/update-fragments-shared.yml b/.github/workflows/update-fragments-shared.yml index 4373380..c321706 100644 --- a/.github/workflows/update-fragments-shared.yml +++ b/.github/workflows/update-fragments-shared.yml @@ -26,7 +26,7 @@ jobs: run: | pip install "git+https://github.com/rubykatzen/embedder.git@v0.0.6" read -ra paths <<< "$EMBEDDER_PATHS" - embedder update "${paths[@]}" --json > /tmp/embedder-result.json + embedder update "${paths[@]}" - name: Detect changes id: diff run: | diff --git a/.github/workflows/update-fragments.yml b/.github/workflows/update-fragments.yml index 215caa8..a4f72bb 100644 --- a/.github/workflows/update-fragments.yml +++ b/.github/workflows/update-fragments.yml @@ -16,7 +16,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | pip install -e . - embedder update . --json > /tmp/embedder-result.json + embedder update . - name: Detect changes id: diff run: | From 15fa6fa822dd8312f6224b05f3cdc61cf88a3b76 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:27:23 +0200 Subject: [PATCH 22/25] feat: auto-bump embedder version in shared workflow on release Co-Authored-By: Claude Sonnet 4.6 --- .../bump-shared-workflow-version/action.yml | 22 +++++++++++++++++++ .github/workflows/prepare-release.yml | 3 +++ 2 files changed, 25 insertions(+) create mode 100644 .github/actions/bump-shared-workflow-version/action.yml diff --git a/.github/actions/bump-shared-workflow-version/action.yml b/.github/actions/bump-shared-workflow-version/action.yml new file mode 100644 index 0000000..8150d10 --- /dev/null +++ b/.github/actions/bump-shared-workflow-version/action.yml @@ -0,0 +1,22 @@ +name: Bump shared workflow embedder version +description: Bump the embedder pip install version in update-fragments-shared.yml, commit and push. +inputs: + version: + description: Release version without v prefix. + required: true +runs: + using: composite + steps: + - name: Bump and commit + shell: bash + run: | + set -euo pipefail + file=.github/workflows/update-fragments-shared.yml + if ! grep -q 'embedder.git@v' "$file" 2>/dev/null; then + exit 0 + fi + sed -i "s|embedder\.git@v[^\"]*|embedder.git@v${{ inputs.version }}|" "$file" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "$file" + git diff --cached --quiet || git commit -m "chore: bump shared workflow to ${{ inputs.version }}" diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 8701bc4..82a12d0 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -39,6 +39,9 @@ jobs: - uses: rubykatzen/releaser/.github/actions/bump-pyproject-version@v0.3.4 with: version: ${{ inputs.version }} + - uses: ./.github/actions/bump-shared-workflow-version + with: + version: ${{ inputs.version }} - uses: ./.github/actions/refresh-local-fragments - uses: rubykatzen/releaser/.github/actions/push-release-branch@v0.3.4 with: From 38ae54f64c03aac8199b118df68880f50857d8d1 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:34:44 +0200 Subject: [PATCH 23/25] refactor: extract install-embedder action, version pin lives in uses: ref Co-Authored-By: Claude Sonnet 4.6 --- .github/actions/bump-shared-workflow-version/action.yml | 4 ++-- .github/actions/install-embedder/action.yml | 8 ++++++++ .github/workflows/update-fragments-shared.yml | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 .github/actions/install-embedder/action.yml diff --git a/.github/actions/bump-shared-workflow-version/action.yml b/.github/actions/bump-shared-workflow-version/action.yml index 8150d10..0743b23 100644 --- a/.github/actions/bump-shared-workflow-version/action.yml +++ b/.github/actions/bump-shared-workflow-version/action.yml @@ -12,10 +12,10 @@ runs: run: | set -euo pipefail file=.github/workflows/update-fragments-shared.yml - if ! grep -q 'embedder.git@v' "$file" 2>/dev/null; then + if ! grep -q 'install-embedder@v' "$file" 2>/dev/null; then exit 0 fi - sed -i "s|embedder\.git@v[^\"]*|embedder.git@v${{ inputs.version }}|" "$file" + sed -i "s|/install-embedder@v[^'\"]*|/install-embedder@v${{ inputs.version }}|" "$file" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add "$file" diff --git a/.github/actions/install-embedder/action.yml b/.github/actions/install-embedder/action.yml new file mode 100644 index 0000000..03fc10b --- /dev/null +++ b/.github/actions/install-embedder/action.yml @@ -0,0 +1,8 @@ +name: Install embedder +description: Install embedder from this repository's source tree. +runs: + using: composite + steps: + - name: Install + shell: bash + run: pip install -e "${{ github.action_path }}/../../.." diff --git a/.github/workflows/update-fragments-shared.yml b/.github/workflows/update-fragments-shared.yml index c321706..5ee64be 100644 --- a/.github/workflows/update-fragments-shared.yml +++ b/.github/workflows/update-fragments-shared.yml @@ -19,12 +19,12 @@ jobs: pull-requests: write steps: - uses: actions/checkout@v7 + - uses: rubykatzen/embedder/.github/actions/install-embedder@v0.0.6 - name: Run embedder update env: GH_TOKEN: ${{ secrets.token || github.token }} EMBEDDER_PATHS: ${{ inputs.paths }} run: | - pip install "git+https://github.com/rubykatzen/embedder.git@v0.0.6" read -ra paths <<< "$EMBEDDER_PATHS" embedder update "${paths[@]}" - name: Detect changes From ce1deff8296a569c5d39cdeb6eba90e103a266ec Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:43:48 +0200 Subject: [PATCH 24/25] chore: install lint tools explicitly for baseline v0.6.2 compatibility Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/lint.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e39345e..ab98336 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,6 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: rubykatzen/baseline/.github/actions/lint-yamllint@v0.5.2 - - uses: rubykatzen/baseline/.github/actions/lint-ruff@v0.5.2 - - uses: rubykatzen/baseline/.github/actions/lint-actionlint@v0.5.2 + - run: pip install ruff yamllint + - run: | + bash <(curl -sL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + sudo install actionlint /usr/local/bin/actionlint + - uses: rubykatzen/baseline/.github/actions/lint-yamllint@v0.6.2 + - uses: rubykatzen/baseline/.github/actions/lint-ruff@v0.6.2 + - uses: rubykatzen/baseline/.github/actions/lint-actionlint@v0.6.2 From 6c3c224ab1c015496fa10cbc26328899c6717cef Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sun, 21 Jun 2026 22:46:23 +0200 Subject: [PATCH 25/25] chore: use setup-runtimes from baseline instead of manual installs Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/lint.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ab98336..af10e16 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,10 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - run: pip install ruff yamllint - - run: | - bash <(curl -sL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) - sudo install actionlint /usr/local/bin/actionlint + - uses: rubykatzen/baseline/.github/actions/setup-runtimes@v0.6.2 - uses: rubykatzen/baseline/.github/actions/lint-yamllint@v0.6.2 - uses: rubykatzen/baseline/.github/actions/lint-ruff@v0.6.2 - uses: rubykatzen/baseline/.github/actions/lint-actionlint@v0.6.2