From 6f36c9d8ced20bf9acbf62144c133c9311bd0a1f Mon Sep 17 00:00:00 2001 From: tiagotaveiragomes <150598480+tiagotaveiragomes@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:34:31 +0100 Subject: [PATCH 1/4] feat: support @-relative includes and $ENV expansion in urlpaths A leading '@' in an include urlpath now resolves relative to the directory of the file doing the including, instead of base_dir / the current working directory (falls back to cwd at the top level, where there is no including file yet). Env vars embedded in urlpaths (e.g. $HOME/foo.yaml) are expanded via os.path.expandvars before any other resolution. --- src/yaml_include/constructor.py | 32 ++++++++++- tests/data/atinclude/nested/target.yaml | 1 + tests/data/atinclude/outer.yaml | 1 + tests/test_at_relative.py | 74 +++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/data/atinclude/nested/target.yaml create mode 100644 tests/data/atinclude/outer.yaml create mode 100644 tests/test_at_relative.py diff --git a/src/yaml_include/constructor.py b/src/yaml_include/constructor.py index 1bfba18..794dc15 100644 --- a/src/yaml_include/constructor.py +++ b/src/yaml_include/constructor.py @@ -4,6 +4,7 @@ from __future__ import annotations +import os import re import sys from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence @@ -43,6 +44,10 @@ WILDCARDS_PATTERN = re.compile(r"^(.*)([\*\?\[\]]+)(.*)$") +#: A leading ``@`` means "resolve relative to the directory of the file doing the including", +#: instead of relative to `base_dir` / the current working directory. +LOCAL_PATTERN = re.compile(r"^@") + if yaml.__with_libyaml__: # pragma: no cover @@ -191,6 +196,15 @@ def read(self, length: int = ...) -> bytes | str: ... typing.Any: The parsed result. """ + _including_file_dir: Optional[str] = field(default=None, init=False, repr=False, compare=False) + """Directory of the file currently being parsed, used to resolve ``@``-relative include paths. + + Tracks the innermost include's directory while its content is being loaded, so that a nested + ``!inc @relative/path.yaml`` resolves relative to the file that contains it, not to `base_dir` + or the current working directory. Restored to the previous value once the nested include + finishes loading, so sibling/parent includes are unaffected. + """ + @contextmanager def managed_autoload(self, autoload: bool) -> Iterator[Self]: """Context manager for temporarily setting the :attr:`autoload` attribute. @@ -364,7 +378,13 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: yaml.load(fp, Loader) """ base_dir = self.base_dir - urlpath = data.urlpath + urlpath = os.path.expandvars(data.urlpath) + + if LOCAL_PATTERN.match(urlpath): + # `@`-relative: resolve against the directory of the file doing the including, + # instead of `base_dir` / the current working directory. + including_dir = Path(self._including_file_dir) if self._including_file_dir else Path.cwd() + urlpath = including_dir.joinpath(urlpath[1:]).as_posix() url_sr = urlsplit(urlpath) if base_dir is not None: @@ -374,7 +394,8 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: base_dir = Path(base_dir) if url_sr.scheme: urlpath = urlunsplit(chain(url_sr[:2], (base_dir.joinpath(url_sr[2]).as_posix(),), url_sr[3:])) - else: + elif not LOCAL_PATTERN.match(data.urlpath): + # `@`-relative paths are already absolute at this point; don't re-anchor them to `base_dir`. urlpath = base_dir.joinpath(urlpath).as_posix() # If protocol/scheme in path, we shall open it directly with fs's default open method @@ -457,7 +478,12 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: # else if no wildcards, return a single object with self.fs.open(urlpath, *data.sequence_params, **data.mapping_params) as of_: - result = load_open_file(of_, loader_type, urlpath, self.custom_loader) + previous_including_file_dir = self._including_file_dir + self._including_file_dir = str(Path(urlpath).parent) + try: + result = load_open_file(of_, loader_type, urlpath, self.custom_loader) + finally: + self._including_file_dir = previous_including_file_dir return result diff --git a/tests/data/atinclude/nested/target.yaml b/tests/data/atinclude/nested/target.yaml new file mode 100644 index 0000000..c02d6e1 --- /dev/null +++ b/tests/data/atinclude/nested/target.yaml @@ -0,0 +1 @@ +value: hello-from-nested diff --git a/tests/data/atinclude/outer.yaml b/tests/data/atinclude/outer.yaml new file mode 100644 index 0000000..a128cbc --- /dev/null +++ b/tests/data/atinclude/outer.yaml @@ -0,0 +1 @@ +key: !inc "@nested/target.yaml" diff --git a/tests/test_at_relative.py b/tests/test_at_relative.py new file mode 100644 index 0000000..e05fea3 --- /dev/null +++ b/tests/test_at_relative.py @@ -0,0 +1,74 @@ +""" +Tests for sigil-added features: + +- ``@``-relative include paths (resolved against the directory of the + including file, not `base_dir` / the current working directory). +- ``$ENV``-style environment variable expansion in include urlpaths. +""" + +import os +from io import StringIO + +import pytest +import yaml + +from yaml_include import Constructor + +from ._internal import YAML_LOADERS +from .conftest import YAML_INCLUDE_TAG, cleanup_constructor + + +class TestAtRelativeInclude: + """`@`-relative resolution: relative to the directory of the including file.""" + + @pytest.fixture(autouse=True) + def setup_constructor(self, test_data_dir): + self.ctor = Constructor(base_dir=str(test_data_dir)) + for loader_cls in YAML_LOADERS: + yaml.add_constructor(YAML_INCLUDE_TAG, self.ctor, loader_cls) + yield + cleanup_constructor(YAML_INCLUDE_TAG, YAML_LOADERS) + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_at_relative_resolves_to_including_file_dir(self, loader): + """`@nested/target.yaml` in `atinclude/outer.yaml` resolves relative to + `atinclude/`, not `base_dir` (the top-level `tests/data` dir).""" + yml = "outer: !inc atinclude/outer.yaml" + data = yaml.load(StringIO(yml), loader) + assert data == {"outer": {"key": {"value": "hello-from-nested"}}} + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_at_relative_top_level_falls_back_to_cwd(self, loader, test_data_dir, monkeypatch): + """A top-level `@`-relative include (no including file yet) falls back to cwd.""" + monkeypatch.chdir(test_data_dir) + yml = 'file1: !inc "@include.d/1.yaml"' + data = yaml.load(StringIO(yml), loader) + assert data == {"file1": {"name": "1"}} + + +class TestEnvVarExpansion: + """`$ENV`-style environment variable expansion in include urlpaths.""" + + @pytest.fixture(autouse=True) + def setup_constructor(self, test_data_dir): + self.ctor = Constructor(base_dir=str(test_data_dir)) + for loader_cls in YAML_LOADERS: + yaml.add_constructor(YAML_INCLUDE_TAG, self.ctor, loader_cls) + yield + cleanup_constructor(YAML_INCLUDE_TAG, YAML_LOADERS) + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_expandvars_in_urlpath(self, loader, monkeypatch): + monkeypatch.setenv("PYYAML_INCLUDE_TEST_DIR", "include.d") + yml = "file1: !inc $PYYAML_INCLUDE_TEST_DIR/1.yaml" + data = yaml.load(StringIO(yml), loader) + assert data == {"file1": {"name": "1"}} + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_expandvars_combined_with_at_relative(self, loader, monkeypatch, test_data_dir): + # Top-level `@`-relative includes fall back to cwd (there is no including file yet). + monkeypatch.chdir(test_data_dir) + monkeypatch.setenv("PYYAML_INCLUDE_TEST_FILE", "outer.yaml") + yml = 'outer: !inc "@atinclude/$PYYAML_INCLUDE_TEST_FILE"' + data = yaml.load(StringIO(yml), loader) + assert data == {"outer": {"key": {"value": "hello-from-nested"}}} From 892e234537a34ac7796e4b1439ac45c597699b89 Mon Sep 17 00:00:00 2001 From: tiagotaveiragomes <150598480+tiagotaveiragomes@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:35:38 +0100 Subject: [PATCH 2/4] feat: support @modulename/path python-module-relative includes '@modulename/relative/path.yaml' resolves relative to the imported module's package directory (importlib.import_module(modulename).__path__[0]). If modulename isn't importable, falls back to plain @-relative resolution, so '@include.d/1.yaml'-style paths (module-name-shaped but not actually a module) keep working. --- src/yaml_include/constructor.py | 30 +++++++++++-- tests/fixture_pkg/__init__.py | 0 tests/fixture_pkg/data/module_include.yaml | 1 + tests/test_python_module_include.py | 52 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 tests/fixture_pkg/__init__.py create mode 100644 tests/fixture_pkg/data/module_include.yaml create mode 100644 tests/test_python_module_include.py diff --git a/src/yaml_include/constructor.py b/src/yaml_include/constructor.py index 794dc15..506d57f 100644 --- a/src/yaml_include/constructor.py +++ b/src/yaml_include/constructor.py @@ -4,6 +4,7 @@ from __future__ import annotations +import importlib import os import re import sys @@ -48,6 +49,10 @@ #: instead of relative to `base_dir` / the current working directory. LOCAL_PATTERN = re.compile(r"^@") +#: ``@modulename/relative/path.yaml`` resolves relative to the imported Python module's +#: (first) package directory, i.e. ``importlib.import_module("modulename").__path__``. +PYTHON_MODULE_PATTERN = re.compile(r"^@([A-Za-z0-9_][A-Za-z0-9_.]*)/(.*)$") + if yaml.__with_libyaml__: # pragma: no cover @@ -379,12 +384,29 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: """ base_dir = self.base_dir urlpath = os.path.expandvars(data.urlpath) - - if LOCAL_PATTERN.match(urlpath): + at_resolved = False + + module_match = PYTHON_MODULE_PATTERN.match(urlpath) + module = None + if module_match: + # `@modulename/relative/path.yaml`: resolve against the imported module's package directory. + # If `modulename` isn't importable, this isn't module syntax after all -- fall back to + # treating the whole thing as a plain `@`-relative path (e.g. `@include.d/1.yaml`). + try: + module = importlib.import_module(module_match.group(1)) + except ModuleNotFoundError: + module = None + if module is not None: + if not module.__path__: # pragma: no cover + raise ImportError(f"module {module_match.group(1)!r} has no package directory (__path__ is empty)") + urlpath = Path(module.__path__[0]).joinpath(module_match.group(2)).as_posix() + at_resolved = True + elif LOCAL_PATTERN.match(urlpath): # `@`-relative: resolve against the directory of the file doing the including, # instead of `base_dir` / the current working directory. including_dir = Path(self._including_file_dir) if self._including_file_dir else Path.cwd() urlpath = including_dir.joinpath(urlpath[1:]).as_posix() + at_resolved = True url_sr = urlsplit(urlpath) if base_dir is not None: @@ -394,8 +416,8 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: base_dir = Path(base_dir) if url_sr.scheme: urlpath = urlunsplit(chain(url_sr[:2], (base_dir.joinpath(url_sr[2]).as_posix(),), url_sr[3:])) - elif not LOCAL_PATTERN.match(data.urlpath): - # `@`-relative paths are already absolute at this point; don't re-anchor them to `base_dir`. + elif not at_resolved: + # `@`-resolved paths are already absolute at this point; don't re-anchor them to `base_dir`. urlpath = base_dir.joinpath(urlpath).as_posix() # If protocol/scheme in path, we shall open it directly with fs's default open method diff --git a/tests/fixture_pkg/__init__.py b/tests/fixture_pkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixture_pkg/data/module_include.yaml b/tests/fixture_pkg/data/module_include.yaml new file mode 100644 index 0000000..0c70b37 --- /dev/null +++ b/tests/fixture_pkg/data/module_include.yaml @@ -0,0 +1 @@ +value: hello-from-module diff --git a/tests/test_python_module_include.py b/tests/test_python_module_include.py new file mode 100644 index 0000000..0da36a4 --- /dev/null +++ b/tests/test_python_module_include.py @@ -0,0 +1,52 @@ +""" +Tests for `@modulename/relative/path.yaml` include syntax: resolves relative +to the imported Python module's package directory +(``importlib.import_module("modulename").__path__``). +""" + +from io import StringIO + +import pytest +import yaml + +from yaml_include import Constructor + +from ._internal import YAML_LOADERS +from .conftest import YAML_INCLUDE_TAG, cleanup_constructor + + +class TestPythonModuleInclude: + @pytest.fixture(autouse=True) + def setup_constructor(self, test_data_dir): + self.ctor = Constructor(base_dir=str(test_data_dir)) + for loader_cls in YAML_LOADERS: + yaml.add_constructor(YAML_INCLUDE_TAG, self.ctor, loader_cls) + yield + cleanup_constructor(YAML_INCLUDE_TAG, YAML_LOADERS) + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_module_relative_include(self, loader): + yml = 'file1: !inc "@tests.fixture_pkg/data/module_include.yaml"' + data = yaml.load(StringIO(yml), loader) + assert data == {"file1": {"value": "hello-from-module"}} + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_module_relative_include_ignores_base_dir(self, loader): + """Module-relative includes are already absolute; `base_dir` must not be re-applied.""" + ctor = Constructor(base_dir="/some/unrelated/dir/that/does/not/exist") + for loader_cls in YAML_LOADERS: + yaml.add_constructor(YAML_INCLUDE_TAG, ctor, loader_cls) + try: + yml = 'file1: !inc "@tests.fixture_pkg/data/module_include.yaml"' + data = yaml.load(StringIO(yml), loader) + assert data == {"file1": {"value": "hello-from-module"}} + finally: + cleanup_constructor(YAML_INCLUDE_TAG, YAML_LOADERS) + + def test_unresolvable_module_falls_back_to_at_relative(self, test_data_dir, monkeypatch): + """If `modulename` in `@modulename/path` isn't importable, it's not module syntax after + all -- fall back to plain `@`-relative resolution of the whole path.""" + monkeypatch.chdir(test_data_dir) + yml = 'file1: !inc "@no.such.module/x.yaml"' + with pytest.raises(FileNotFoundError): + yaml.load(StringIO(yml), yaml.SafeLoader) From 36a3855815ea1f83db718ea63f456bfb782446c7 Mon Sep 17 00:00:00 2001 From: tiagotaveiragomes <150598480+tiagotaveiragomes@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:37:02 +0100 Subject: [PATCH 3/4] feat: support path:key.subkey fragment extraction on includes 'some/file.yaml:key.subkey' loads the file, then descends into the loaded mapping via key.subkey (split on '.'), returning only that nested value. Only applies to plain local, non-wildcard paths; real URLs (which always contain '://') and glob patterns are left alone. --- src/yaml_include/constructor.py | 11 +++++++ tests/data/fragment.yaml | 7 ++++ tests/test_fragment_include.py | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 tests/data/fragment.yaml create mode 100644 tests/test_fragment_include.py diff --git a/src/yaml_include/constructor.py b/src/yaml_include/constructor.py index 506d57f..03698c6 100644 --- a/src/yaml_include/constructor.py +++ b/src/yaml_include/constructor.py @@ -408,6 +408,14 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: urlpath = including_dir.joinpath(urlpath[1:]).as_posix() at_resolved = True + # `path/to/file.yaml:key.subkey` loads `path/to/file.yaml`, then descends into the + # loaded mapping/sequence via `key.subkey`, returning only that nested value. + # Only applies to plain local paths: real URLs always contain "://", and wildcarded + # paths have no single file to descend into. + objpath: Optional[str] = None + if "://" not in urlpath and ":" in urlpath and not WILDCARDS_PATTERN.match(urlpath): + urlpath, _, objpath = urlpath.partition(":") + url_sr = urlsplit(urlpath) if base_dir is not None: if callable(base_dir): @@ -506,6 +514,9 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: result = load_open_file(of_, loader_type, urlpath, self.custom_loader) finally: self._including_file_dir = previous_including_file_dir + if objpath: + for piece in objpath.split("."): + result = result[piece] return result diff --git a/tests/data/fragment.yaml b/tests/data/fragment.yaml new file mode 100644 index 0000000..8dd44a2 --- /dev/null +++ b/tests/data/fragment.yaml @@ -0,0 +1,7 @@ +top: + mid: + leaf: fragment-value + other: ignored +list_value: + - a + - b diff --git a/tests/test_fragment_include.py b/tests/test_fragment_include.py new file mode 100644 index 0000000..ccb74ce --- /dev/null +++ b/tests/test_fragment_include.py @@ -0,0 +1,58 @@ +""" +Tests for `path/to/file.yaml:key.subkey` fragment-extraction include syntax: +loads the file, then descends into the loaded mapping via `key.subkey` +(``.split(".")``), returning only that nested value. +""" + +from io import StringIO + +import pytest +import yaml + +from yaml_include import Constructor + +from ._internal import YAML_LOADERS +from .conftest import YAML_INCLUDE_TAG, cleanup_constructor + + +class TestFragmentInclude: + @pytest.fixture(autouse=True) + def setup_constructor(self, test_data_dir): + self.ctor = Constructor(base_dir=str(test_data_dir)) + for loader_cls in YAML_LOADERS: + yaml.add_constructor(YAML_INCLUDE_TAG, self.ctor, loader_cls) + yield + cleanup_constructor(YAML_INCLUDE_TAG, YAML_LOADERS) + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_single_level_key(self, loader): + yml = "value: !inc fragment.yaml:top" + data = yaml.load(StringIO(yml), loader) + assert data == {"value": {"mid": {"leaf": "fragment-value"}, "other": "ignored"}} + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_nested_key_path(self, loader): + yml = "value: !inc fragment.yaml:top.mid.leaf" + data = yaml.load(StringIO(yml), loader) + assert data == {"value": "fragment-value"} + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_without_fragment_returns_whole_document(self, loader): + yml = "value: !inc fragment.yaml" + data = yaml.load(StringIO(yml), loader) + assert data == { + "value": {"top": {"mid": {"leaf": "fragment-value"}, "other": "ignored"}, "list_value": ["a", "b"]} + } + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_unknown_key_raises(self, loader): + yml = "value: !inc fragment.yaml:top.nope" + with pytest.raises(KeyError): + yaml.load(StringIO(yml), loader) + + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_fragment_combined_with_at_relative(self, loader, test_data_dir, monkeypatch): + monkeypatch.chdir(test_data_dir) + yml = 'value: !inc "@fragment.yaml:top.mid.leaf"' + data = yaml.load(StringIO(yml), loader) + assert data == {"value": "fragment-value"} From 1193c80c2227a58ee4e8d7246a88f090924fb73b Mon Sep 17 00:00:00 2001 From: tiagotaveiragomes <150598480+tiagotaveiragomes@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:43:15 +0100 Subject: [PATCH 4/4] fix: gate module includes, fix module/fragment/at-relative edge cases Address reviewer findings against the sigil upstream feature branch: 1. SECURITY: @modulename/path no longer implicitly imports arbitrary installed packages as a side effect of yaml.load. Gated behind a new Constructor(allow_module_include=False) opt-in flag (default False); when disabled, @modulename/path falls through to plain @-relative resolution instead of raising. 2. Non-package modules (e.g. os.path, no __path__) now fall back to @-relative resolution instead of raising a raw AttributeError. 3. Namespace packages with multiple __path__ entries are now resolved by trying every entry (first that exists wins), not just entry [0]. 4. path:key.subkey fragment splitting no longer misparses a Windows drive letter (C:\... / C:/...) as a fragment separator. 5. _including_file_dir is now tracked correctly for files reached via a wildcard/glob include (set per matched file); for URL-scheme includes it's explicitly reset rather than left stale, so a nested @-relative include can't silently resolve against the wrong directory. Also documents all 4 new include features (@-relative, @module/path + its opt-in flag, $ENV expansion, path:key.subkey fragments) in the Constructor docstring, and adds a CHANGELOG entry. Known limitations left as-is (out of scope for this PR): fragment extraction only supports dotted mapping keys, not list indices; env-var expansion is unconditional (no opt-out). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 13 +++ src/yaml_include/constructor.py | 119 ++++++++++++++++++++++++---- tests/test_at_relative.py | 10 +++ tests/test_fragment_parsing.py | 44 ++++++++++ tests/test_python_module_include.py | 78 +++++++++++++++++- 5 files changed, 247 insertions(+), 17 deletions(-) create mode 100644 tests/test_fragment_parsing.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7425b61..eb24749 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ - ❎ Breaking Changes: - Drop support for Python 3.8 and below +- New: + + - `@relative/path.yaml` includes resolve relative to the directory of the *including* file, instead of `base_dir` / cwd + - `@modulename/relative/path.yaml` includes resolve relative to an importable Python module's package directory. Gated behind a new `Constructor(allow_module_include=False)` flag (default `False`, opt-in) since resolving it imports the named module, executing its top-level code + - `$ENV_VAR`-style environment variable expansion in include urlpaths + - `path/to/file.yaml:key.subkey` fragment syntax to load a file and extract only a nested key + +- Bug fix: + + - Non-package modules (e.g. `os.path`, no `__path__`) and namespace packages (multiple `__path__` entries) are now handled correctly by `@modulename/path` resolution instead of raising `AttributeError` or only trying the first entry + - `path:key` fragment splitting no longer misparses a Windows drive letter (`C:\...`, `C:/...`) as a fragment separator + - `@`-relative resolution state is now tracked correctly for files reached via a wildcard/glob include, instead of only the single-file no-scheme path; it's explicitly reset (not left stale) for URL-scheme includes + ## 2.2 > 📅 **Date** 2024-11-9 diff --git a/src/yaml_include/constructor.py b/src/yaml_include/constructor.py index 03698c6..d750795 100644 --- a/src/yaml_include/constructor.py +++ b/src/yaml_include/constructor.py @@ -53,6 +53,12 @@ #: (first) package directory, i.e. ``importlib.import_module("modulename").__path__``. PYTHON_MODULE_PATTERN = re.compile(r"^@([A-Za-z0-9_][A-Za-z0-9_.]*)/(.*)$") +#: Matches a ``:key.subkey``-style fragment suffix, used to split ``path/to/file.yaml:key`` +#: without misinterpreting a Windows drive letter (``C:\...`` / ``C:/...``) as a fragment +#: separator. A drive letter is a single alpha character at the very start of the string, +#: immediately followed by ``:`` and then a path separator -- that shape is excluded here. +FRAGMENT_PATTERN = re.compile(r":(?!$)(?!\\)(?!/)[A-Za-z_][\w.]*$") + if yaml.__with_libyaml__: # pragma: no cover @@ -112,6 +118,37 @@ class Constructor: files: !inc foo/**/*.yml + * Include a file relative to the directory of the file doing the including + (rather than relative to `base_dir` or the current working directory), with + a leading ``@``: + + .. code-block:: yaml + + file: !inc @sibling/baz.yml + + * Include a file relative to an importable Python package's directory, with + ``@modulename/relative/path``. This requires opting in via + ``Constructor(allow_module_include=True)`` (default ``False``) -- see the + :attr:`.allow_module_include` attribute docs for why it's gated: + + .. code-block:: yaml + + file: !inc @my_package/data/baz.yml + + * Expand ``$ENV_VAR`` / ``${ENV_VAR}``-style environment variables anywhere in + the urlpath (applied via :func:`os.path.expandvars` before any other resolution): + + .. code-block:: yaml + + file: !inc $CONFIG_DIR/baz.yml + + * Load a file and descend into it, extracting only a nested key (or dotted + ``key.subkey`` path) via a trailing ``:key.subkey`` fragment: + + .. code-block:: yaml + + value: !inc foo/baz.yml:some.nested.key + #. Load the YAML in Python:: data = yaml.load(yaml_string, yaml.Loader) @@ -201,6 +238,24 @@ def read(self, length: int = ...) -> bytes | str: ... typing.Any: The parsed result. """ + allow_module_include: bool = False + """Whether ``@modulename/relative/path.yaml`` includes are permitted. + + Resolving this syntax calls :func:`importlib.import_module` on the ``modulename`` + component, which executes that module's top-level code as a side effect of + :func:`yaml.load`. This can let untrusted YAML trigger arbitrary code execution + via any importable package, so it is an explicit, opt-in-only feature. + + * If ``False`` (the default): + ``@modulename/path`` is never treated as module syntax; it falls through to + plain ``@``-relative resolution (as if ``modulename`` were not importable). + + * If ``True``: + ``@modulename/path`` first attempts to import ``modulename`` and resolve + against its package directory, falling back to plain ``@``-relative + resolution only if the module can't be imported or has no usable ``__path__``. + """ + _including_file_dir: Optional[str] = field(default=None, init=False, repr=False, compare=False) """Directory of the file currently being parsed, used to resolve ``@``-relative include paths. @@ -386,20 +441,31 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: urlpath = os.path.expandvars(data.urlpath) at_resolved = False - module_match = PYTHON_MODULE_PATTERN.match(urlpath) - module = None + module_match = PYTHON_MODULE_PATTERN.match(urlpath) if self.allow_module_include else None + module_resolved_path: Optional[str] = None if module_match: # `@modulename/relative/path.yaml`: resolve against the imported module's package directory. - # If `modulename` isn't importable, this isn't module syntax after all -- fall back to - # treating the whole thing as a plain `@`-relative path (e.g. `@include.d/1.yaml`). + # If `modulename` isn't importable, or has no usable `__path__` (e.g. it's a plain module, + # not a package), this isn't module syntax after all -- fall back to treating the whole + # thing as a plain `@`-relative path (e.g. `@include.d/1.yaml`). try: module = importlib.import_module(module_match.group(1)) - except ModuleNotFoundError: - module = None - if module is not None: - if not module.__path__: # pragma: no cover - raise ImportError(f"module {module_match.group(1)!r} has no package directory (__path__ is empty)") - urlpath = Path(module.__path__[0]).joinpath(module_match.group(2)).as_posix() + module_paths = list(module.__path__) + except (ModuleNotFoundError, AttributeError): + module_paths = [] + rel = module_match.group(2) + for candidate_dir in module_paths: + candidate = Path(candidate_dir).joinpath(rel) + if candidate.exists(): + module_resolved_path = candidate.as_posix() + break + else: + if module_paths: + # Module found but none of its `__path__` entries contain the target file; + # use the first entry so downstream open/glob still produces a sensible error. + module_resolved_path = Path(module_paths[0]).joinpath(rel).as_posix() + if module_resolved_path is not None: + urlpath = module_resolved_path at_resolved = True elif LOCAL_PATTERN.match(urlpath): # `@`-relative: resolve against the directory of the file doing the including, @@ -413,8 +479,10 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: # Only applies to plain local paths: real URLs always contain "://", and wildcarded # paths have no single file to descend into. objpath: Optional[str] = None - if "://" not in urlpath and ":" in urlpath and not WILDCARDS_PATTERN.match(urlpath): - urlpath, _, objpath = urlpath.partition(":") + if "://" not in urlpath and not WILDCARDS_PATTERN.match(urlpath): + fragment_match = FRAGMENT_PATTERN.search(urlpath) + if fragment_match: + urlpath, objpath = urlpath[: fragment_match.start()], urlpath[fragment_match.start() + 1 :] url_sr = urlsplit(urlpath) if base_dir is not None: @@ -430,19 +498,35 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: # If protocol/scheme in path, we shall open it directly with fs's default open method if url_sr.scheme: + # `@`-relative resolution is scoped to the local file system: it can't map a remote + # URL's location onto a local directory. Rather than let a nested `!inc @...` silently + # resolve against a stale (unrelated) `_including_file_dir` left over from some earlier + # local include, explicitly clear it for the duration of URL-scheme loads, so any nested + # `@`-relative include falls back to `cwd` (its documented top-level behavior) instead of + # a wrong directory. if WILDCARDS_PATTERN.match(urlpath): # if wildcards in path, return a Sequence/List result = [] with fsspec.open_files(urlpath, *data.sequence_params, **data.mapping_params) as ofs: for of_ in ofs: - loaded_data = load_open_file(of_, loader_type, urlpath, self.custom_loader) + previous_including_file_dir = self._including_file_dir + self._including_file_dir = None + try: + loaded_data = load_open_file(of_, loader_type, urlpath, self.custom_loader) + finally: + self._including_file_dir = previous_including_file_dir result.append(loaded_data) return result # else if no wildcard, returns a single object with fsspec.open(urlpath, *data.sequence_params, **data.mapping_params) as of_: if isinstance(of_, list): # pragma: no cover raise RuntimeError(f"`fsspec.open()` returns a `list` ({of_})") - result = load_open_file(of_, loader_type, urlpath, self.custom_loader) + previous_including_file_dir = self._including_file_dir + self._including_file_dir = None + try: + result = load_open_file(of_, loader_type, urlpath, self.custom_loader) + finally: + self._including_file_dir = previous_including_file_dir return result # if no protocol / scheme in path, we shall use the `fs` object @@ -499,7 +583,12 @@ def load(self, loader_type: Type[Union[_Loader, _CLoader]], data: Data) -> Any: if not isinstance(file, str): # pragma: no cover raise RuntimeError(f"`fs.glob()` function does not return a `str` ({file})") with open_fn(file) as of_: - loaded_data = load_open_file(of_, loader_type, file, self.custom_loader) + previous_including_file_dir = self._including_file_dir + self._including_file_dir = str(Path(file).parent) + try: + loaded_data = load_open_file(of_, loader_type, file, self.custom_loader) + finally: + self._including_file_dir = previous_including_file_dir result.append(loaded_data) if data.flatten: return [child for item in result for child in item] diff --git a/tests/test_at_relative.py b/tests/test_at_relative.py index e05fea3..c60f2aa 100644 --- a/tests/test_at_relative.py +++ b/tests/test_at_relative.py @@ -45,6 +45,16 @@ def test_at_relative_top_level_falls_back_to_cwd(self, loader, test_data_dir, mo data = yaml.load(StringIO(yml), loader) assert data == {"file1": {"name": "1"}} + @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) + def test_at_relative_inside_wildcard_glob_include(self, loader): + """A file reached via a wildcard/glob include (`atinclude/outer.yaml`, matched by + `atinclude/*.yaml`) that itself contains a nested `@`-relative include must resolve + that nested include against its OWN directory (`atinclude/`), not stale state left + over from some earlier include.""" + yml = "outer: !inc atinclude/*.yaml" + data = yaml.load(StringIO(yml), loader) + assert data == {"outer": [{"key": {"value": "hello-from-nested"}}]} + class TestEnvVarExpansion: """`$ENV`-style environment variable expansion in include urlpaths.""" diff --git a/tests/test_fragment_parsing.py b/tests/test_fragment_parsing.py new file mode 100644 index 0000000..6fad2fe --- /dev/null +++ b/tests/test_fragment_parsing.py @@ -0,0 +1,44 @@ +""" +Tests for the internal `path:key.subkey` fragment-splitting logic, specifically that it +does not misparse a Windows drive letter (``C:\\...`` / ``C:/...``) as a fragment separator. + +These exercise the string-splitting logic directly (via `FRAGMENT_PATTERN`), so they run on +any OS -- no real Windows filesystem is required. +""" + +from yaml_include.constructor import FRAGMENT_PATTERN + + +def _split(urlpath): + """Mirror the splitting logic in `Constructor.load`.""" + match = FRAGMENT_PATTERN.search(urlpath) + if match: + return urlpath[: match.start()], urlpath[match.start() + 1 :] + return urlpath, None + + +class TestFragmentSplittingVsDriveLetters: + def test_windows_backslash_path_not_split(self): + urlpath, objpath = _split(r"C:\Users\foo\bar.yaml") + assert urlpath == r"C:\Users\foo\bar.yaml" + assert objpath is None + + def test_windows_forward_slash_path_not_split(self): + urlpath, objpath = _split("C:/Users/foo/bar.yaml") + assert urlpath == "C:/Users/foo/bar.yaml" + assert objpath is None + + def test_plain_fragment_still_split(self): + urlpath, objpath = _split("path/to/file.yaml:top.mid.leaf") + assert urlpath == "path/to/file.yaml" + assert objpath == "top.mid.leaf" + + def test_single_level_fragment_still_split(self): + urlpath, objpath = _split("fragment.yaml:top") + assert urlpath == "fragment.yaml" + assert objpath == "top" + + def test_no_fragment_no_colon(self): + urlpath, objpath = _split("plain/path.yaml") + assert urlpath == "plain/path.yaml" + assert objpath is None diff --git a/tests/test_python_module_include.py b/tests/test_python_module_include.py index 0da36a4..1ad1aae 100644 --- a/tests/test_python_module_include.py +++ b/tests/test_python_module_include.py @@ -4,12 +4,14 @@ (``importlib.import_module("modulename").__path__``). """ +import types from io import StringIO import pytest import yaml from yaml_include import Constructor +from yaml_include import constructor as constructor_module from ._internal import YAML_LOADERS from .conftest import YAML_INCLUDE_TAG, cleanup_constructor @@ -18,7 +20,7 @@ class TestPythonModuleInclude: @pytest.fixture(autouse=True) def setup_constructor(self, test_data_dir): - self.ctor = Constructor(base_dir=str(test_data_dir)) + self.ctor = Constructor(base_dir=str(test_data_dir), allow_module_include=True) for loader_cls in YAML_LOADERS: yaml.add_constructor(YAML_INCLUDE_TAG, self.ctor, loader_cls) yield @@ -33,7 +35,7 @@ def test_module_relative_include(self, loader): @pytest.mark.parametrize("loader", YAML_LOADERS, ids=lambda cls: cls.__name__) def test_module_relative_include_ignores_base_dir(self, loader): """Module-relative includes are already absolute; `base_dir` must not be re-applied.""" - ctor = Constructor(base_dir="/some/unrelated/dir/that/does/not/exist") + ctor = Constructor(base_dir="/some/unrelated/dir/that/does/not/exist", allow_module_include=True) for loader_cls in YAML_LOADERS: yaml.add_constructor(YAML_INCLUDE_TAG, ctor, loader_cls) try: @@ -50,3 +52,75 @@ def test_unresolvable_module_falls_back_to_at_relative(self, test_data_dir, monk yml = 'file1: !inc "@no.such.module/x.yaml"' with pytest.raises(FileNotFoundError): yaml.load(StringIO(yml), yaml.SafeLoader) + + def test_non_package_module_falls_back_instead_of_raising_attributeerror(self, test_data_dir, monkeypatch): + """`os.path` is importable but has no `__path__` (it's not a package). This must fall + back to plain `@`-relative resolution, not raise a raw `AttributeError`.""" + monkeypatch.chdir(test_data_dir) + yml = 'file1: !inc "@os.path/x.yaml"' + with pytest.raises(FileNotFoundError): + yaml.load(StringIO(yml), yaml.SafeLoader) + + def test_namespace_package_tries_all_path_entries(self, loader=yaml.SafeLoader): + """A module whose `__path__` has multiple entries (namespace package) must be resolved + by trying every entry, not just the first, e.g. when the file lives under a later entry.""" + fake_module = types.SimpleNamespace( + __path__=[ + "/no/such/directory/at/all", + str((__import__("pathlib").Path(__file__).parent / "fixture_pkg").resolve()), + ] + ) + + def fake_import_module(name): + assert name == "fake.namespace.pkg" + return fake_module + + original = constructor_module.importlib.import_module + constructor_module.importlib.import_module = fake_import_module + try: + yml = 'file1: !inc "@fake.namespace.pkg/data/module_include.yaml"' + data = yaml.load(StringIO(yml), loader) + assert data == {"file1": {"value": "hello-from-module"}} + finally: + constructor_module.importlib.import_module = original + + +class TestModuleIncludeDisabledByDefault: + """`allow_module_include` defaults to `False`: `@modulename/path` must not trigger an + implicit `importlib.import_module()` call unless explicitly opted in.""" + + @pytest.fixture(autouse=True) + def setup_constructor(self, test_data_dir): + self.ctor = Constructor(base_dir=str(test_data_dir)) + for loader_cls in YAML_LOADERS: + yaml.add_constructor(YAML_INCLUDE_TAG, self.ctor, loader_cls) + yield + cleanup_constructor(YAML_INCLUDE_TAG, YAML_LOADERS) + + def test_default_constructor_never_imports_module(self): + assert self.ctor.allow_module_include is False + + def test_module_syntax_falls_back_to_at_relative_when_disabled(self, test_data_dir, monkeypatch): + """With the flag left at its default (`False`), `@tests.fixture_pkg/...` must NOT import + `tests.fixture_pkg` -- it must be treated as a plain `@`-relative path, which does not + exist relative to cwd, so this must raise `FileNotFoundError`, never succeed via import.""" + monkeypatch.chdir(test_data_dir) + yml = 'file1: !inc "@tests.fixture_pkg/data/module_include.yaml"' + with pytest.raises(FileNotFoundError): + yaml.load(StringIO(yml), yaml.SafeLoader) + + def test_module_import_not_attempted_when_disabled(self, monkeypatch, test_data_dir): + """Directly assert `importlib.import_module` is never called when the flag is off.""" + monkeypatch.chdir(test_data_dir) + calls = [] + original = constructor_module.importlib.import_module + + def spy_import_module(name): + calls.append(name) + return original(name) + + monkeypatch.setattr(constructor_module.importlib, "import_module", spy_import_module) + yml = 'file1: !inc "@tests.fixture_pkg/data/module_include.yaml"' + with pytest.raises(FileNotFoundError): + yaml.load(StringIO(yml), yaml.SafeLoader) + assert calls == []