diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index df32dde..a8fa828 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -72,6 +72,86 @@ jobs: - name: Root-owned install, cf-execd and canary (step 7) run: tests/deploy/linux-install.sh + # Upstream cfengine/core and libntech at the overlay bases plus the overlay + # (docs/architecture/upstream-overlay.md §4, §7), not the pinned fork. It is + # the proof that the overlay's all-or-nothing loader (overlay/core/0002) and + # the strict JSON parse (overlay/libntech/0005, 0006) build, and that the + # loader's unit and acceptance tests and run.sh (TENDCF_REQUIRE_FORK_LOADER=1; + # its check 4b probes the loader and fails on any build that kept a + # per-entry skip) pass on them as on the fork build. The loader source is the + # fork's bar the R1 accessor; behaviour is NOT proven equal for JSON numbers + # (B-4, B-10 and core's rlist.c fix are not in the overlay, + # upstream-overlay.md §7). One leg: the point is the overlay, not the + # platform matrix. `bin/overlay.py check` (weekly) only + # proves the patches apply to upstream master, not that they compile (R1). + overlay-build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Fetch core and libntech at the overlay bases + shell: bash + run: | + set -euo pipefail + m() { python3 -c "import json,sys;print(json.load(open('overlay/overlay.json'))['components'][sys.argv[1]][sys.argv[2]])" "$@"; } + core_base=$(m core base) + lib_base=$(m libntech base) + git init -q /tmp/core + git -C /tmp/core remote add origin "$(m core upstream)" + timeout 600 git -C /tmp/core fetch -q --depth 1 origin "$core_base" + git -C /tmp/core checkout -q FETCH_HEAD + # core's submodule pointer at its base is libntech's base (upstream-overlay.md §3) + pin=$(git -C /tmp/core rev-parse HEAD:libntech) + test "$pin" = "$lib_base" || { echo "libntech base $lib_base is not core's submodule pin $pin"; exit 1; } + git init -q /tmp/core/libntech + git -C /tmp/core/libntech remote add origin "$(m libntech upstream)" + timeout 600 git -C /tmp/core/libntech fetch -q --depth 1 origin "$lib_base" + git -C /tmp/core/libntech checkout -q FETCH_HEAD + - name: Apply the overlay (upstream-overlay.md §4) + shell: bash + run: | + set -euo pipefail + # python3, not the uv shebang: the script has no dependencies + python3 bin/overlay.py apply core --checkout /tmp/core + python3 bin/overlay.py apply libntech --checkout /tmp/core/libntech + git -C /tmp/core add libntech + { + echo "### CFEngine overlay build" + echo + for c in core libntech; do + n=$(python3 -c "import json,sys;print(len(json.load(open('overlay/overlay.json'))['components'][sys.argv[1]]['patches']))" "$c") + if [ "$c" = core ]; then dir=/tmp/core; else dir=/tmp/core/libntech; fi + echo "- $c: base \`$(python3 -c "import json,sys;print(json.load(open('overlay/overlay.json'))['components'][sys.argv[1]]['base'])" "$c")\` plus $n patches, tree \`$(git -C "$dir" rev-parse 'HEAD^{tree}')\` (asserted by apply)" + done + } >> "$GITHUB_STEP_SUMMARY" + - name: Build dependencies (core's own script; Debian/Ubuntu branch) + run: /tmp/core/ci/dependencies.sh + - name: Configure and build + shell: bash + run: | + set -euo pipefail + cd /tmp/core + ./autogen.sh --enable-debug --prefix=/tmp/overlay-cf --with-systemd-service=no + make -j"$(nproc)" CFLAGS="-Wall" + make install + /tmp/overlay-cf/bin/cf-agent --version + - name: Loader unit tests (cmdb_test) and libntech json and mustache tests + shell: bash + run: | + set -euo pipefail + make -C /tmp/core/tests/unit check TESTS=cmdb_test + make -C /tmp/core/libntech/tests/unit check TESTS="json_test mustache_test" + - name: Loader acceptance tests (06_host_specific_data) + shell: bash + run: | + set -euo pipefail + cd /tmp/core/tests/acceptance + chmod -R go-w . + ./testall --printlog --tests=common,errorlog,errorexit,errorpass 00_basics/06_host_specific_data + - name: Generic-bundle tests (the loader capability probe must pass) + env: + TENDCF_REQUIRE_FORK_LOADER: "1" + run: tests/policy/run.sh --bindir /tmp/overlay-cf/bin + debian-13: runs-on: ubuntu-24.04 container: debian:13 diff --git a/AGENTS.md b/AGENTS.md index 6ed3033..4e72b28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -359,8 +359,12 @@ a book: `site-private/AGENTS.md` ("Book knowledge base"). - Linux CI (`.github/workflows/linux.yml`) builds the fork at the commit in `overlay/fork.json` (`--prefix=/opt/tendcf`) and runs the fork's unit/acceptance subsets plus the scripts above on ubuntu-24.04/26.04/24.04-arm - and a `debian:13` container, which also uploads the pastebin artifact. Moving - the pin is a commit. `check.yml` runs `bin/schema_lint.py`, + and a `debian:13` container, which also uploads the pastebin artifact. The + `overlay-build` job builds upstream at the overlay bases plus `overlay/` + (the all-or-nothing loader and the strict parse; `upstream-overlay.md` §4, + §7) and runs `cmdb_test`, libntech's `json_test`/`mustache_test`, the + `06_host_specific_data` acceptance directory and `run.sh` with + `TENDCF_REQUIRE_FORK_LOADER=1` on it. Moving the pin is a commit. `check.yml` runs `bin/schema_lint.py`, `bin/xref_lint.py`, `bin/overlay.py lint`, the canary check and `shellcheck`. - Plan and status: `docs/architecture/linux-targets.md` §3; the NixOS side in `docs/architecture/nixos-passthrough-plan.md`. Live hosts: pastebin (root diff --git a/bin/overlay.py b/bin/overlay.py index 35d003b..edaabd1 100755 --- a/bin/overlay.py +++ b/bin/overlay.py @@ -50,7 +50,6 @@ import json import os import re -import shutil import subprocess import sys import tempfile @@ -66,7 +65,7 @@ SUBJECT_PREFIX = re.compile(r"^\[PATCH[^\]]*\]\s*") REQUIRED_PATCH_KEYS = ("commit", "file", "subject") -REQUIRED_COMPONENT_KEYS = ("base", "patches", "upstream") +REQUIRED_COMPONENT_KEYS = ("base", "patches", "upstream", "applied_tree") class Finding(Exception): @@ -76,14 +75,20 @@ class Finding(Exception): # ---------------------------------------------------------------- git helpers -def git(repo: Path | str, *args: str, env: dict | None = None, - check: bool = True, input_bytes: bytes | None = None) -> subprocess.CompletedProcess: +def git( + repo: Path | str, + *args: str, + env: dict | None = None, + check: bool = True, + input_bytes: bytes | None = None, +) -> subprocess.CompletedProcess: cmd = ["git", "-C", str(repo), *args] merged = dict(os.environ) if env: merged.update(env) - return subprocess.run(cmd, env=merged, check=check, capture_output=True, - input=input_bytes) + return subprocess.run( + cmd, env=merged, check=check, capture_output=True, input=input_bytes + ) def git_out(repo: Path | str, *args: str, **kw) -> str: @@ -121,15 +126,18 @@ def load_manifest(overlay: Path) -> dict: raise Finding(f"{path}: not JSON: {e}") from None canonical = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n" if canonical != text: - raise Finding(f"{path}: not in canonical form (2-space indent, sorted keys, " - f"trailing newline). Rewrite with: bin/overlay.py lint --fix") + raise Finding( + f"{path}: not in canonical form (2-space indent, sorted keys, " + f"trailing newline). Rewrite with: bin/overlay.py lint --fix" + ) return data def save_manifest(overlay: Path, data: dict) -> None: (overlay / MANIFEST_NAME).write_text( json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n", - encoding="utf-8") + encoding="utf-8", + ) def component(data: dict, name: str) -> dict: @@ -163,7 +171,7 @@ def parse_patch_header(path: Path) -> tuple[str, str]: continue break if line.startswith("Subject: "): - subject_lines.append(line[len("Subject: "):]) + subject_lines.append(line[len("Subject: ") :]) in_subject = True elif line == "": break @@ -194,6 +202,9 @@ def lint(overlay: Path) -> list[str]: base = comp.get("base", "") if not HEX40.match(base): findings.append(f"{name}: base {base!r} is not a 40-hex commit id") + tree = comp.get("applied_tree") + if tree is not None and not HEX40.match(tree): + findings.append(f"{name}: applied_tree {tree!r} is not a 40-hex tree id") cdir = overlay / name if not cdir.is_dir(): findings.append(f"{name}: directory {cdir} missing") @@ -201,14 +212,20 @@ def lint(overlay: Path) -> list[str]: on_disk = sorted(p.name for p in cdir.iterdir() if p.suffix == ".patch") listed = [p.get("file", "") for p in comp.get("patches", [])] if listed != sorted(listed): - findings.append(f"{name}: manifest patch order is not the NNNN order: {listed}") + findings.append( + f"{name}: manifest patch order is not the NNNN order: {listed}" + ) if sorted(listed) != on_disk: - findings.append(f"{name}: manifest lists {sorted(listed)} but disk has {on_disk}") + findings.append( + f"{name}: manifest lists {sorted(listed)} but disk has {on_disk}" + ) seen_numbers: set[str] = set() for entry in comp.get("patches", []): for key in REQUIRED_PATCH_KEYS: if key not in entry: - findings.append(f"{name}: patch entry {entry.get('file', '?')} missing {key!r}") + findings.append( + f"{name}: patch entry {entry.get('file', '?')} missing {key!r}" + ) fname = entry.get("file", "") m = PATCH_NAME.match(fname) if not m: @@ -228,9 +245,13 @@ def lint(overlay: Path) -> list[str]: findings.append(str(e)) continue if commit != entry.get("commit"): - findings.append(f"{name}/{fname}: From line {commit} != manifest commit {entry.get('commit')}") + findings.append( + f"{name}/{fname}: From line {commit} != manifest commit {entry.get('commit')}" + ) if subject != entry.get("subject"): - findings.append(f"{name}/{fname}: subject {subject!r} != manifest {entry.get('subject')!r}") + findings.append( + f"{name}/{fname}: subject {subject!r} != manifest {entry.get('subject')!r}" + ) for d in overlay.iterdir(): if d.is_dir() and d.name not in listed_dirs: findings.append(f"{d}: directory not listed in the manifest") @@ -250,25 +271,42 @@ def check(overlay: Path, name: str, repo: Path, against: str | None) -> list[str tree = git_out(repo, "rev-parse", "--verify", f"{rev}^{{tree}}") resolved = git_out(repo, "rev-parse", "--verify", f"{rev}^{{commit}}") except subprocess.CalledProcessError as e: - raise Finding(f"{name}: cannot resolve {rev!r} in {repo}: {e.stderr.decode().strip()}") from None - report = [f"{name}: base {comp['base'][:12]}; checking against {rev} = {resolved[:12]}" - + ("" if resolved == comp["base"] else " (NOT the pinned base: drift check)")] + raise Finding( + f"{name}: cannot resolve {rev!r} in {repo}: {e.stderr.decode().strip()}" + ) from None + report = [ + f"{name}: base {comp['base'][:12]}; checking against {rev} = {resolved[:12]}" + + ("" if resolved == comp["base"] else " (NOT the pinned base: drift check)") + ] with tempfile.TemporaryDirectory(prefix="tendcf-overlay-") as tmp: index = str(Path(tmp) / "index") env = {"GIT_INDEX_FILE": index} git(repo, "read-tree", tree, env=env) for path in patch_paths(overlay, name, comp): - r = git(repo, "apply", "--cached", "--whitespace=nowarn", str(path), - env=env, check=False) + r = git( + repo, + "apply", + "--cached", + "--whitespace=nowarn", + str(path), + env=env, + check=False, + ) if r.returncode != 0: err = r.stderr.decode().strip() raise Finding( - "\n".join(report + [ - f"DRIFT: {name}/{path.name} does not apply to {rev} ({resolved[:12]}):", - *(" " + ln for ln in err.splitlines()), - f"Rebase the fork branch onto {rev}, re-export with `bin/overlay.py export`, " - f"and bump {name}.base in {MANIFEST_NAME} — or retire the patch if upstream carries it.", - ])) + "\n".join( + report + + [ + f"DRIFT: {name}/{path.name} does not apply to {rev} ({resolved[:12]}):", + *(" " + ln for ln in err.splitlines()), + ( + f"Rebase the fork branch onto {rev}, re-export with `bin/overlay.py export`, " + f"and bump {name}.base in {MANIFEST_NAME} — or retire the patch if upstream carries it." + ), + ] + ) + ) report.append(f" ok {path.name}") return report @@ -282,15 +320,25 @@ def patch_id(repo: Path, patch_bytes: bytes) -> str | None: return out[0] if out else None -def retired(overlay: Path, name: str, repo: Path, against: str) -> tuple[list[str], bool]: +def retired( + overlay: Path, name: str, repo: Path, against: str +) -> tuple[list[str], bool]: data = load_manifest(overlay) comp = component(data, name) base = comp["base"] # Every upstream commit between the pinned base and `against`, by patch-id. - log = git(repo, "log", "-p", "--no-merges", "--format=commit %H%n%s", f"{base}..{against}").stdout - ids_out = git(repo, "patch-id", "--stable", input_bytes=log, check=False).stdout.decode() - upstream_ids = {ln.split()[0]: ln.split()[1] for ln in ids_out.splitlines() if ln.strip()} - subjects = git_out(repo, "log", "--no-merges", "--format=%s", f"{base}..{against}").splitlines() + log = git( + repo, "log", "-p", "--no-merges", "--format=commit %H%n%s", f"{base}..{against}" + ).stdout + ids_out = git( + repo, "patch-id", "--stable", input_bytes=log, check=False + ).stdout.decode() + upstream_ids = { + ln.split()[0]: ln.split()[1] for ln in ids_out.splitlines() if ln.strip() + } + subjects = git_out( + repo, "log", "--no-merges", "--format=%s", f"{base}..{against}" + ).splitlines() count = len(subjects) report = [f"{name}: {count} upstream commit(s) in {base[:12]}..{against}"] any_retire = False @@ -298,11 +346,15 @@ def retired(overlay: Path, name: str, repo: Path, against: str) -> tuple[list[st pid = patch_id(repo, path.read_bytes()) if pid and pid in upstream_ids: any_retire = True - report.append(f" RETIRE {path.name}: identical change is upstream as {upstream_ids[pid][:12]}") + report.append( + f" RETIRE {path.name}: identical change is upstream as {upstream_ids[pid][:12]}" + ) elif entry["subject"] in subjects: any_retire = True - report.append(f" VERIFY {path.name}: an upstream commit has the same subject but a different " - f"patch-id; check whether it supersedes this patch") + report.append( + f" VERIFY {path.name}: an upstream commit has the same subject but a different " + f"patch-id; check whether it supersedes this patch" + ) else: report.append(f" keep {path.name}") return report, any_retire @@ -311,23 +363,38 @@ def retired(overlay: Path, name: str, repo: Path, against: str) -> tuple[list[st # ------------------------------------------------------------------ apply -def apply_series(overlay: Path, name: str, checkout: Path, allow_mismatch: bool) -> list[str]: +def apply_series( + overlay: Path, name: str, checkout: Path, allow_mismatch: bool +) -> list[str]: data = load_manifest(overlay) comp = component(data, name) head = git_out(checkout, "rev-parse", "HEAD") report = [f"{name}: HEAD {head[:12]}, pinned base {comp['base'][:12]}"] if head != comp["base"] and not allow_mismatch: - raise Finding(f"{name}: {checkout} is at {head[:12]}, not the pinned base {comp['base'][:12]}. " - f"Check out the base, or pass --allow-base-mismatch after `check --against` is green.") + raise Finding( + f"{name}: {checkout} is at {head[:12]}, not the pinned base {comp['base'][:12]}. " + f"Check out the base, or pass --allow-base-mismatch after `check --against` is green." + ) dirty = git_out(checkout, "status", "--porcelain", "--untracked-files=no") if dirty: - raise Finding(f"{name}: {checkout} has uncommitted changes; refusing to `git am` onto them") + raise Finding( + f"{name}: {checkout} has uncommitted changes; refusing to `git am` onto them" + ) rebase_apply = git_path(checkout, "rebase-apply") for busy in (rebase_apply, git_path(checkout, "rebase-merge")): if busy.exists(): - raise Finding(f"{name}: {checkout} has a `git am` or rebase in progress ({busy}); resolve or abort it first") + raise Finding( + f"{name}: {checkout} has a `git am` or rebase in progress ({busy}); resolve or abort it first" + ) paths = patch_paths(overlay, name, comp) - r = git(checkout, *identity_args(checkout), "am", "--whitespace=nowarn", *map(str, paths), check=False) + r = git( + checkout, + *identity_args(checkout), + "am", + "--whitespace=nowarn", + *map(str, paths), + check=False, + ) if r.returncode != 0: err = r.stderr.decode().strip() + "\n" + r.stdout.decode().strip() # Abort only the am session this invocation started: rebase-apply was @@ -339,31 +406,83 @@ def apply_series(overlay: Path, name: str, checkout: Path, allow_mismatch: bool) abort_note = f"another `git am` or rebase session ({rebase_apply}) appeared meanwhile; left it alone" else: ab = git(checkout, *identity_args(checkout), "am", "--abort", check=False) - abort_note = (f"aborted, checkout restored to {head[:12]}" if ab.returncode == 0 else - f"and `git am --abort` ALSO failed ({ab.returncode}): {ab.stderr.decode().strip()} {ab.stdout.decode().strip()}") - raise Finding("\n".join(report + [ - f"DRIFT: `git am` failed in {checkout}; {abort_note}:", - *(" " + ln for ln in err.splitlines() if ln.strip()), - ])) + abort_note = ( + f"aborted, checkout restored to {head[:12]}" + if ab.returncode == 0 + else f"and `git am --abort` ALSO failed ({ab.returncode}): {ab.stderr.decode().strip()} {ab.stdout.decode().strip()}" + ) + raise Finding( + "\n".join( + report + + [ + f"DRIFT: `git am` failed in {checkout}; {abort_note}:", + *(" " + ln for ln in err.splitlines() if ln.strip()), + ] + ) + ) new_head = git_out(checkout, "rev-parse", "HEAD") report.append(f" applied {len(paths)} patch(es); HEAD now {new_head[:12]}") + expected = comp.get("applied_tree") + tree = git_out(checkout, "rev-parse", "HEAD^{tree}") + if head != comp["base"]: + # applied_tree is the tree after applying onto the pinned base only. + report.append( + f" applied tree {tree} not compared (HEAD is not the pinned base)" + ) + else: + if not expected: + raise Finding( + "\n".join( + report + + [ + f"{name}: the manifest has no applied_tree to assert after `git am`." + ] + ) + ) + if tree != expected: + raise Finding( + "\n".join( + report + + [ + ( + f"TREE MISMATCH: the applied tree is {tree}, the manifest's applied_tree is {expected}. " + f"A patch file's body differs from what was recorded (lint checks only its From line and subject). " + f"The checkout at {checkout} is left with the patches applied." + ), + ] + ) + ) + report.append(f" applied tree {tree} matches the manifest") return report # ----------------------------------------------------------------- export -def export(overlay: Path, name: str, repo: Path, base: str, tips: list[str]) -> list[str]: +def export( + overlay: Path, name: str, repo: Path, base: str, tips: list[str] +) -> list[str]: """One or more series, each `base..tip`, numbered consecutively. Several tips are for independent fork branches with no file overlap (issue #3's libntech case); `check` proves the combined order still applies.""" - data = load_manifest(overlay) if (overlay / MANIFEST_NAME).is_file() else {"schema_version": 1, "components": {}} + data = ( + load_manifest(overlay) + if (overlay / MANIFEST_NAME).is_file() + else {"schema_version": 1, "components": {}} + ) comps = data.setdefault("components", {}) comp = comps.setdefault(name, {"upstream": "", "base": base, "patches": []}) base_full = git_out(repo, "rev-parse", "--verify", f"{base}^{{commit}}") - tips_full = [git_out(repo, "rev-parse", "--verify", f"{t}^{{commit}}") for t in tips] + tips_full = [ + git_out(repo, "rev-parse", "--verify", f"{t}^{{commit}}") for t in tips + ] for tip, tip_full in zip(tips, tips_full): - if git(repo, "merge-base", "--is-ancestor", base_full, tip_full, check=False).returncode != 0: + if ( + git( + repo, "merge-base", "--is-ancestor", base_full, tip_full, check=False + ).returncode + != 0 + ): raise Finding(f"{name}: {base} is not an ancestor of {tip}") old_by_commit = {p["commit"]: p for p in comp.get("patches", [])} cdir = overlay / name @@ -374,9 +493,18 @@ def export(overlay: Path, name: str, repo: Path, base: str, tips: list[str]) -> cdir.mkdir(parents=True, exist_ok=True) files: list[str] = [] for tip_full in tips_full: - out = git_out(repo, "format-patch", "--no-signature", "--no-renames", "--full-index", - "--no-stat", f"--start-number={len(files) + 1}", "-o", str(cdir), - f"{base_full}..{tip_full}") + out = git_out( + repo, + "format-patch", + "--no-signature", + "--no-renames", + "--full-index", + "--no-stat", + f"--start-number={len(files) + 1}", + "-o", + str(cdir), + f"{base_full}..{tip_full}", + ) files.extend(Path(ln).name for ln in out.splitlines() if ln.strip()) patches = [] for fname in files: @@ -387,8 +515,10 @@ def export(overlay: Path, name: str, repo: Path, base: str, tips: list[str]) -> comp["base"] = base_full comp["patches"] = patches save_manifest(overlay, data) - return [f"{name}: exported {len(files)} patch(es) from {base_full[:12]}..{{{', '.join(t[:12] for t in tips_full)}}} into {cdir}", - *(f" {f}" for f in files)] + return [ + f"{name}: exported {len(files)} patch(es) from {base_full[:12]}..{{{', '.join(t[:12] for t in tips_full)}}} into {cdir}", + *(f" {f}" for f in files), + ] # -------------------------------------------------------------- self-test @@ -401,10 +531,17 @@ def self_test() -> list[str]: def ok(label: str, cond: bool, detail: str = "") -> None: if not cond: - raise Finding(f"self-test FAILED: {label}" + (f"\n{detail}" if detail else "")) + raise Finding( + f"self-test FAILED: {label}" + (f"\n{detail}" if detail else "") + ) report.append(f" pass {label}") - ident = ["-c", "user.name=tendcf overlay self-test", "-c", "user.email=overlay@tendcf.invalid"] + ident = [ + "-c", + "user.name=tendcf overlay self-test", + "-c", + "user.email=overlay@tendcf.invalid", + ] def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: (repo / rel).write_text(text, encoding="utf-8") @@ -424,8 +561,15 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: git(upstream, "checkout", "-q", "-b", "fix/sample") fixed = list(lines) fixed[4] = "line 5 fixed\n" - c1 = commit_file(upstream, "json.c", "".join(fixed), "Fixed line five decoding twice") - c2 = commit_file(upstream, "json_test.c", "test for line five\n", "Added a test for line five") + c1 = commit_file( + upstream, "json.c", "".join(fixed), "Fixed line five decoding twice" + ) + c2 = commit_file( + upstream, + "json_test.c", + "test for line five\n", + "Added a test for line five", + ) git(upstream, "checkout", "-q", "master") overlay = tmp_path / "overlay" @@ -434,8 +578,15 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: export(overlay, "sample", upstream, base, [c2]) data = load_manifest(overlay) data["components"]["sample"]["upstream"] = "file://" + str(upstream) + data["components"]["sample"]["applied_tree"] = git_out( + upstream, "rev-parse", f"{c2}^{{tree}}" + ) save_manifest(overlay, data) - ok("export produced two numbered patches", [p["file"][:5] for p in data["components"]["sample"]["patches"]] == ["0001-", "0002-"]) + ok( + "export produced two numbered patches", + [p["file"][:5] for p in data["components"]["sample"]["patches"]] + == ["0001-", "0002-"], + ) ok("lint is clean on the exported overlay", lint(overlay) == []) # 1. The series applies to the pinned base. @@ -444,7 +595,9 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: # 2. An unrelated upstream change does not count as drift. git(upstream, "checkout", "-q", "-b", "unrelated", base) - unrelated = commit_file(upstream, "README", "unrelated\n", "Unrelated upstream change") + unrelated = commit_file( + upstream, "README", "unrelated\n", "Unrelated upstream change" + ) check(overlay, "sample", upstream, unrelated) ok("check --against: unrelated upstream change still applies", True) @@ -452,11 +605,17 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: git(upstream, "checkout", "-q", "-b", "drifted", base) drifted_lines = list(lines) drifted_lines[4] = "line 5 rewritten upstream\n" - drifted = commit_file(upstream, "json.c", "".join(drifted_lines), "Rewrote line five differently") + drifted = commit_file( + upstream, "json.c", "".join(drifted_lines), "Rewrote line five differently" + ) try: check(overlay, "sample", upstream, drifted) except Finding as e: - ok("check --against: conflicting upstream change reports DRIFT", "DRIFT" in str(e) and "0001-" in str(e), str(e)) + ok( + "check --against: conflicting upstream change reports DRIFT", + "DRIFT" in str(e) and "0001-" in str(e), + str(e), + ) else: ok("check --against: conflicting upstream change reports DRIFT", False) @@ -466,7 +625,11 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: try: apply_series(overlay, "sample", clone, allow_mismatch=False) except Finding as e: - ok("apply: refuses a checkout whose HEAD is not the pinned base", "not the pinned base" in str(e), str(e)) + ok( + "apply: refuses a checkout whose HEAD is not the pinned base", + "not the pinned base" in str(e), + str(e), + ) else: ok("apply: refuses a checkout whose HEAD is not the pinned base", False) try: @@ -475,9 +638,14 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: status = git_out(clone, "status", "--porcelain") in_progress = git_path(clone, "rebase-apply").exists() head_now = git_out(clone, "rev-parse", "HEAD") - ok("apply --allow-base-mismatch: `git am` conflict fails loudly and leaves the checkout clean", - "DRIFT" in str(e) and status == "" and not in_progress and head_now == drifted, - f"status={status!r} rebase-apply={in_progress} HEAD={head_now[:12]} expected={drifted[:12]}\n{e}") + ok( + "apply --allow-base-mismatch: `git am` conflict fails loudly and leaves the checkout clean", + "DRIFT" in str(e) + and status == "" + and not in_progress + and head_now == drifted, + f"status={status!r} rebase-apply={in_progress} HEAD={head_now[:12]} expected={drifted[:12]}\n{e}", + ) else: ok("apply --allow-base-mismatch: `git am` conflict fails loudly", False) @@ -485,19 +653,29 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: # is a file), is refused and left alone, never aborted by us. wt = tmp_path / "linked-worktree" git(clone, "worktree", "add", "-q", "--detach", str(wt), drifted) - first_patch = overlay / "sample" / data["components"]["sample"]["patches"][0]["file"] + first_patch = ( + overlay / "sample" / data["components"]["sample"]["patches"][0]["file"] + ) user_am = git(wt, *ident, "am", str(first_patch), check=False) user_session = git_path(wt, "rebase-apply") - ok("linked worktree: the user's own `git am` stopped, its session outside /.git/", - user_am.returncode != 0 and user_session.exists() and (wt / ".git").is_file() - and not (wt / ".git" / "rebase-apply").exists(), - f"am rc={user_am.returncode} session={user_session} exists={user_session.exists()}") + ok( + "linked worktree: the user's own `git am` stopped, its session outside /.git/", + user_am.returncode != 0 + and user_session.exists() + and (wt / ".git").is_file() + and not (wt / ".git" / "rebase-apply").exists(), + f"am rc={user_am.returncode} session={user_session} exists={user_session.exists()}", + ) try: apply_series(overlay, "sample", wt, allow_mismatch=True) except Finding as e: - ok("apply: refuses a linked worktree with a `git am` in progress and leaves that session intact", - "in progress" in str(e) and user_session.exists() and git_out(wt, "rev-parse", "HEAD") == drifted, - f"session exists={user_session.exists()}\n{e}") + ok( + "apply: refuses a linked worktree with a `git am` in progress and leaves that session intact", + "in progress" in str(e) + and user_session.exists() + and git_out(wt, "rev-parse", "HEAD") == drifted, + f"session exists={user_session.exists()}\n{e}", + ) else: ok("apply: refuses a linked worktree with a `git am` in progress", False) git(wt, *ident, "am", "--abort") @@ -507,32 +685,120 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: git(clone, "checkout", "-q", base) apply_series(overlay, "sample", clone, allow_mismatch=False) subjects = git_out(clone, "log", "--format=%s", f"{base}..HEAD").splitlines() - ok("apply: `git am` lands both commits in order", - subjects == ["Added a test for line five", "Fixed line five decoding twice"], str(subjects)) - ok("apply: the applied tree matches the fork tip's tree", - git_out(clone, "rev-parse", "HEAD^{tree}") == git_out(upstream, "rev-parse", f"{c2}^{{tree}}")) + ok( + "apply: `git am` lands both commits in order", + subjects + == ["Added a test for line five", "Fixed line five decoding twice"], + str(subjects), + ) + ok( + "apply: the applied tree matches the fork tip's tree", + git_out(clone, "rev-parse", "HEAD^{tree}") + == git_out(upstream, "rev-parse", f"{c2}^{{tree}}"), + ) + + # 4b. The manifest's applied_tree is asserted after `git am`. + good_tree = git_out(clone, "rev-parse", "HEAD^{tree}") + for tree, label in ((good_tree, "matches"), ("0" * 40, "differs")): + data = load_manifest(overlay) + data["components"]["sample"]["applied_tree"] = tree + save_manifest(overlay, data) + git(clone, "reset", "-q", "--hard", base) + try: + rep = apply_series(overlay, "sample", clone, allow_mismatch=False) + ok( + f"apply: applied_tree that {label} is " + + ("accepted" if tree == good_tree else "refused"), + tree == good_tree + and any("matches the manifest" in ln for ln in rep), + ) + except Finding as e: + ok( + "apply: applied_tree that differs is refused with TREE MISMATCH", + tree != good_tree and "TREE MISMATCH" in str(e), + str(e), + ) + data = load_manifest(overlay) + data["components"]["sample"]["applied_tree"] = "xyz" + save_manifest(overlay, data) + ok( + "lint: a malformed applied_tree is a finding", + any("applied_tree" in f for f in lint(overlay)), + ) + del data["components"]["sample"]["applied_tree"] + save_manifest(overlay, data) + ok( + "lint: a component without applied_tree is a finding", + any("missing 'applied_tree'" in f for f in lint(overlay)), + ) + git(clone, "reset", "-q", "--hard", base) + try: + apply_series(overlay, "sample", clone, allow_mismatch=False) + except Finding as e: + ok( + "apply: on the pinned base a manifest without applied_tree is refused", + "no applied_tree" in str(e), + str(e), + ) + else: + ok( + "apply: on the pinned base a manifest without applied_tree is refused", + False, + ) + git(clone, "reset", "-q", "--hard", base) + + # 4c. Off the pinned base the tree is reported, not compared. + data = load_manifest(overlay) + data["components"]["sample"]["applied_tree"] = good_tree + save_manifest(overlay, data) + git(clone, "checkout", "-q", unrelated) + rep = apply_series(overlay, "sample", clone, allow_mismatch=True) + ok( + "apply --allow-base-mismatch: off the pinned base the applied tree is reported as not compared", + any("not compared" in ln for ln in rep) + and not any("TREE MISMATCH" in ln for ln in rep), + "\n".join(rep), + ) + git(clone, "checkout", "-q", base) # 5. Retirement: upstream cherry-picks the first fix. git(upstream, "checkout", "-q", "-b", "caught-up", base) git(upstream, *ident, "cherry-pick", c1) caught_up = git_out(upstream, "rev-parse", "HEAD") rep, any_retire = retired(overlay, "sample", upstream, caught_up) - ok("retired: a cherry-picked patch is reported RETIRE, the other kept", - any_retire and any("RETIRE" in ln and "0001-" in ln for ln in rep) and any("keep" in ln and "0002-" in ln for ln in rep), - "\n".join(rep)) + ok( + "retired: a cherry-picked patch is reported RETIRE, the other kept", + any_retire + and any("RETIRE" in ln and "0001-" in ln for ln in rep) + and any("keep" in ln and "0002-" in ln for ln in rep), + "\n".join(rep), + ) rep, any_retire = retired(overlay, "sample", upstream, unrelated) - ok("retired: nothing to retire against an unrelated change", not any_retire, "\n".join(rep)) + ok( + "retired: nothing to retire against an unrelated change", + not any_retire, + "\n".join(rep), + ) # 6. Lint catches a manifest that disagrees with the files. data = load_manifest(overlay) data["components"]["sample"]["patches"][0]["commit"] = "0" * 40 save_manifest(overlay, data) - ok("lint: a manifest commit that disagrees with the patch's From line is a finding", - any("From line" in f for f in lint(overlay))) + ok( + "lint: a manifest commit that disagrees with the patch's From line is a finding", + any("From line" in f for f in lint(overlay)), + ) stray = overlay / "sample" / "0003-stray.patch" - stray.write_text("From " + "1" * 40 + " Mon Sep 17 00:00:00 2001\nSubject: [PATCH] stray\n\n", encoding="utf-8") - ok("lint: a patch file the manifest does not list is a finding", - any("disk has" in f for f in lint(overlay))) + stray.write_text( + "From " + + "1" * 40 + + " Mon Sep 17 00:00:00 2001\nSubject: [PATCH] stray\n\n", + encoding="utf-8", + ) + ok( + "lint: a patch file the manifest does not list is a finding", + any("disk has" in f for f in lint(overlay)), + ) return report @@ -540,37 +806,65 @@ def commit_file(repo: Path, rel: str, text: str, msg: str) -> str: def main() -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--overlay", type=Path, default=DEFAULT_OVERLAY, - help=f"overlay directory (default {DEFAULT_OVERLAY})") + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--overlay", + type=Path, + default=DEFAULT_OVERLAY, + help=f"overlay directory (default {DEFAULT_OVERLAY})", + ) sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("lint", help="manifest and patch files agree").add_argument( - "--fix", action="store_true", help="rewrite the manifest in canonical form") + "--fix", action="store_true", help="rewrite the manifest in canonical form" + ) sub.add_parser("status", help="print the overlay") - p = sub.add_parser("check", help="patches apply to a tree (temporary index; repo untouched)") + p = sub.add_parser( + "check", help="patches apply to a tree (temporary index; repo untouched)" + ) p.add_argument("component") - p.add_argument("--repo", type=Path, required=True, help="any git repository holding the upstream objects") + p.add_argument( + "--repo", + type=Path, + required=True, + help="any git repository holding the upstream objects", + ) p.add_argument("--against", help="rev to apply onto (default: the pinned base)") p = sub.add_parser("retired", help="patches upstream already carries") p.add_argument("component") p.add_argument("--repo", type=Path, required=True) - p.add_argument("--against", required=True, help="upstream rev to compare with, e.g. upstream/master") - p.add_argument("--strict", action="store_true", help="exit 1 if any patch should be retired") - - p = sub.add_parser("apply", help="`git am` the series onto a checkout at the pinned base") + p.add_argument( + "--against", + required=True, + help="upstream rev to compare with, e.g. upstream/master", + ) + p.add_argument( + "--strict", action="store_true", help="exit 1 if any patch should be retired" + ) + + p = sub.add_parser( + "apply", help="`git am` the series onto a checkout at the pinned base" + ) p.add_argument("component") p.add_argument("--checkout", type=Path, required=True) p.add_argument("--allow-base-mismatch", action="store_true") - p = sub.add_parser("export", help="regenerate a component's patches from a fork branch") + p = sub.add_parser( + "export", help="regenerate a component's patches from a fork branch" + ) p.add_argument("component") p.add_argument("--repo", type=Path, required=True) p.add_argument("--base", required=True, help="upstream commit the series sits on") - p.add_argument("--tip", required=True, action="append", - help="fork branch tip; repeat for independent series on the same base") + p.add_argument( + "--tip", + required=True, + action="append", + help="fork branch tip; repeat for independent series on the same base", + ) sub.add_parser("self-test", help="prove the mechanism on a synthetic repository") @@ -589,24 +883,58 @@ def main() -> int: if args.cmd == "status": data = load_manifest(overlay) for name, comp in sorted(data["components"].items()): - print(f"{name}: upstream {comp.get('upstream', '?')} base {comp['base'][:12]}" - f"{' ' + comp['base_described'] if comp.get('base_described') else ''}") + print( + f"{name}: upstream {comp.get('upstream', '?')} base {comp['base'][:12]}" + f"{' ' + comp['base_described'] if comp.get('base_described') else ''}" + ) for e in comp["patches"]: - extra = ", ".join(f"{k}={e[k]}" for k in ("register", "role", "upstream_pr_state") if k in e) - print(f" {e['file']} {e['commit'][:12]} {e['subject']}" + (f" [{extra}]" if extra else "")) + extra = ", ".join( + f"{k}={e[k]}" + for k in ("register", "role", "upstream_pr_state") + if k in e + ) + print( + f" {e['file']} {e['commit'][:12]} {e['subject']}" + + (f" [{extra}]" if extra else "") + ) return 0 if args.cmd == "check": - print("\n".join(check(overlay, args.component, args.repo.resolve(), args.against))) + print( + "\n".join( + check(overlay, args.component, args.repo.resolve(), args.against) + ) + ) return 0 if args.cmd == "retired": - rep, any_retire = retired(overlay, args.component, args.repo.resolve(), args.against) + rep, any_retire = retired( + overlay, args.component, args.repo.resolve(), args.against + ) print("\n".join(rep)) return 1 if (args.strict and any_retire) else 0 if args.cmd == "apply": - print("\n".join(apply_series(overlay, args.component, args.checkout.resolve(), args.allow_base_mismatch))) + print( + "\n".join( + apply_series( + overlay, + args.component, + args.checkout.resolve(), + args.allow_base_mismatch, + ) + ) + ) return 0 if args.cmd == "export": - print("\n".join(export(overlay, args.component, args.repo.resolve(), args.base, args.tip))) + print( + "\n".join( + export( + overlay, + args.component, + args.repo.resolve(), + args.base, + args.tip, + ) + ) + ) return 0 if args.cmd == "self-test": rep = self_test() @@ -617,7 +945,10 @@ def main() -> int: print(str(e), file=sys.stderr) return 1 except subprocess.CalledProcessError as e: - print(f"{' '.join(e.cmd)} failed ({e.returncode}):\n{e.stderr.decode(errors='replace')}", file=sys.stderr) + print( + f"{' '.join(e.cmd)} failed ({e.returncode}):\n{e.stderr.decode(errors='replace')}", + file=sys.stderr, + ) return 1 return 2 diff --git a/docs/architecture/tendcf-validate-proposal-2026-11-03.md b/docs/architecture/tendcf-validate-proposal-2026-11-03.md index 6168dff..b34f067 100644 --- a/docs/architecture/tendcf-validate-proposal-2026-11-03.md +++ b/docs/architecture/tendcf-validate-proposal-2026-11-03.md @@ -1097,7 +1097,8 @@ CFEngine. opposite of overlay patch `8f0076b81` (B-5b in [`upstream-register.md`](upstream-register.md)), which tendcf introduced to narrow the blast radius, and it diverges from what was - proposed upstream. Projector and policy stay unchanged. + proposed upstream. (Done: the overlay replaced that patch with this + loader on 2026-11-14, `overlay/core/0002`.) Projector and policy stay unchanged. **(c) Consequences.** @@ -1257,6 +1258,13 @@ per-entry skip with the literal fixture that fixture, so check 9 sees both keys missing and the partial states stay tested only through the stand-in (check 10), which remains valid because the check compares the file with what was installed whatever the loader's policy. +Since 2026-11-14 the overlay (`overlay/core/0002`, with libntech `0005`/`0006`) +carries that same all-or-nothing loader source in place of `8f0076b81` (bar the +R1 accessor), and the `overlay-build` job runs the loader's tests and `run.sh` +on it with `TENDCF_REQUIRE_FORK_LOADER=1`; they pass on both builds. The two +builds still differ for JSON numbers (B-4, B-10 and the `rlist.c` fix are not in +the overlay; `upstream-overlay.md` §7), which this check's fixtures do not +exercise. (2) The check needs `readjson` and `variablesmatching` to compose on the loader's installed variables; they do on the dev build (checks 9-10 of diff --git a/docs/architecture/upstream-overlay.md b/docs/architecture/upstream-overlay.md index 58ff2ea..01d342f 100644 --- a/docs/architecture/upstream-overlay.md +++ b/docs/architecture/upstream-overlay.md @@ -47,6 +47,16 @@ Not in the overlay, by that rule: The overlay is therefore small by construction, and it shrinks: §6. +Load-bearing today, by that rule: libntech B-14 (backslash decoding of +`host_specific.json` strings), B-23 (mustache rendering of every launchd +plist) and B-24 (the strict JSON parse); core B-5b, **replaced** on +2026-11-14 by the all-or-nothing CMDB loader (operator ruling R13: one +invalid entry in `host_specific.json` installs nothing, because a skipped +interlock entry leaves the services it guards ungated), which carries B-20's +null guards (the loader keeps them but refuses the whole file, where B-20 +skipped the entry; its null validation relies on them) and B-7 (a warning that +rides along). B-5a, B-13 and B-22 are carriers. + **Ruling, 2026-10-29 (operator):** the Linux CI and hosts ([`linux-targets.md`](linux-targets.md)) run the full fork build, including the security fixes and `--simulate-json`, not upstream plus the overlay. The manifest above still lists what shipped @@ -59,7 +69,12 @@ pointer, fetched from the libntech fork). `.github/workflows/linux.yml` builds it on every Linux CI target through `.github/actions/build-fork`, writes core and libntech commits plus `cf-agent --version` to the job summary, and runs `tests/policy/run.sh` on the result. Moving the pin is a -commit here, reviewed like any other. `tendcf-integration` itself is +commit here, reviewed like any other. Since 2026-11-14 the `overlay-build` +job in `.github/workflows/linux.yml` also builds upstream plus the overlay at +the base (§4) and runs the loader's unit, acceptance and `tests/policy/run.sh` +tests (`TENDCF_REQUIRE_FORK_LOADER=1`) on it. What that proves: the loader +source is the fork's bar the R1 accessor, and the same tests pass on both +builds. It does not prove the same behaviour for JSON numbers (§7). `tendcf-integration` itself is refreshed by merging fork `master` and every current fix branch ([`upstream-register.md`](upstream-register.md) "Branch layout"), never rebased. @@ -124,9 +139,24 @@ cd ~/src/core-overlay && NO_CONFIGURE=1 ./autogen.sh && ./configure --enable-deb That is the recipe the #3 comment executed by hand on 2026-08-19 (full `make -j4` clean; `json_test` 74/74, `mustache_test` 17/17, -`06_host_specific_data` 14/14 under fakeroot). `apply` refuses a checkout -whose HEAD is not the pinned base and refuses a dirty tree, so the recipe -cannot silently build something else. +`06_host_specific_data` 14/14 under fakeroot), then with four libntech +patches and the per-entry-skip B-5b. It now applies six libntech patches +(0005 and 0006 add `JsonParseStrict()`/`JsonParseFileStrict()`, B-24, which +the core loader calls) and core's all-or-nothing loader; the commands are +unchanged. The `overlay-build` CI job (`linux.yml`) applies the overlay with +`bin/overlay.py apply`, then runs the same dependency, configure, build and +install steps as the `build-fork` action (`.github/actions/build-fork`: +`ci/dependencies.sh`, `./autogen.sh --enable-debug --prefix=... --with-systemd-service=no`, +`make -j"$(nproc)" CFLAGS="-Wall"`, `make install`), not the hand recipe above, +and then `cmdb_test`, `json_test`, `mustache_test`, the `06_host_specific_data` +acceptance directory and `tests/policy/run.sh` with +`TENDCF_REQUIRE_FORK_LOADER=1`. The manifest's `applied_tree` per component is +the expected tree after `apply` onto the pinned base, and `lint` requires it; +`apply` fails on a mismatch, so an edited patch body cannot pass for the recorded +one (with `--allow-base-mismatch` on another HEAD it reports the tree as not +compared). `apply` refuses a checkout whose HEAD is not the +pinned base and refuses a dirty tree, so the recipe cannot silently build +something else. **Later**, when device binaries have a build pipeline (map §13 Step 10+, "builder/cache"; this was first written for tendcf-agent, which map D45 @@ -169,6 +199,21 @@ Where it runs: not on every push. A red run is the overlay asking for §5's rebase or §6's retirement, nothing else. +**Caveat (R1): `check` is textual only.** It proves the series *applies*, +not that it *compiles*. The core loader at the overlay base calls +`EvalContextVariableGet(ctx, ref, &type)` (three arguments). Upstream master +since `671d53ff0` (2026-08-21, secrets tracking) has a four-argument +`EvalContextVariableGet(…, get_secret)` and a separate +`EvalContextVariableGetPlaintext()`; the fork's loader uses the latter, and +the overlay's spelling of it is the base's. So at upstream master +(`a8ac0dc7e`, 2026-11-13) `check core` is green (`git apply --cached` of both +patches succeeds) while a build would stop on that call (a signature comparison, not a build: nothing was compiled at that commit). The weekly +`overlay-drift` job cannot see this. The `overlay-build` job builds at the +pinned base only, so a bump of the base must re-run it before the base moves: +bumping to a commit at or after `671d53ff0` means spelling the call +`EvalContextVariableGetPlaintext()` again and re-checking B-5a and the four +older libntech patches against the newer base. + **Rebasing the overlay** when it drifts: in the fork, rebase the branch onto the new upstream commit and push it; then @@ -207,38 +252,75 @@ A patch can also retire because the code it protects stops shipping (`retire_when` says so). That is a judgment, recorded in the commit that drops it. -## 7. State verified on 2026-10-09 +## 7. State verified on 2026-10-09, updated 2026-11-14 (overlay phase B) | | libntech | core | | --- | --- | --- | | base | `0c0620d` (upstream master 2026-08-15) | `a0bca6aaf` (upstream master 2026-08-15; pins libntech `0c0620d`) | -| patches | 4: B-13 (carrier), **B-14**, B-22 (carrier), **B-23** | 2: B-5a (carrier), **B-5b** | -| upstream PRs | #293, #297, #296, #298 — all closed unmerged 2026-08-19 | #6315, #6320 — closed unmerged 2026-08-19 | -| `check` at base | 4/4 ok | 2/2 ok | -| `check --against` upstream master | ok at `3c21479` (master moved 2 commits, neither in `json.c`/`mustache.c`) | ok at `4b67bb7` (master moved 98 commits, none in `cmdb.c`) and at tag `3.27.1-build7` | -| `apply` into a clean clone at base | tree `5cbd6f7…`, **byte-identical to the local merge `45c816c`** the #3 comment built and tested | tree `c66a60d…`, **byte-identical to fork tip `8f0076b81`** | -| `retired --strict` against upstream master | nothing to retire | nothing to retire | -| `self-test` | 13/13 | | - -**Since 2026-11-02 (R13, PR #41) the core column's B-5b is not what the fork -the tests run does.** The pin in `overlay/fork.json` carries an all-or-nothing -CMDB loader that replaces B-5b's per-entry skip (a skipped interlock leaves the -services it guards unguarded). The manifest entry still reproduces `8f0076b81` -and carries a `superseded_on_fork` note; the table above is the 2026-10-09 -state and is not updated. Upstream plus this overlay skips a bad entry; the -pinned fork installs nothing from the file. Operator ruling, 2026-11-14, B11 -round 2 item 1 (adversary D5, Astra C2): "1. Replace". The overlay carries the -all-or-nothing CMDB loader (R13) instead of B-5b's per-entry skip, so -upstream-plus-overlay builds match the pinned fork; if the overlay cannot patch -libntech it carries the core part only (no strict parse). `8f0076b81` (fork PR -59) is retired, never offered upstream. Status: ruled; implementation pending -in a follow-up PR (overlay phase B), because it changes what the overlay -reproduces ([`upstream-register.md`](upstream-register.md), "Operator rulings of -2026-11-14 on the queue and the fork", item 2). - -So the overlay reproduces exactly the inputs whose build was verified on -2026-08-19, from this repository alone, and would still apply to a build -cut from today's upstream. +| patches | 6: B-13 (carrier), **B-14**, B-22 (carrier), **B-23**, **B-24** ×2 (`0005`, `0006`: `JsonParseFileStrict()`) | 2: B-5a (carrier), **B-5b replaced** (`0002`, the all-or-nothing loader, with B-20 and B-7 inside) | +| fork refs | the four 2026-08 branches; `fix/json-strict-parse` (fork PR 11: `dc3fba2`, `8a57e55`) | `fix/cmdb-name-offending-key`; `overlay/cmdb-all-or-nothing` (`4158906f2`, tag `tendcf-pin-4158906`) | +| upstream PRs | #293, #297, #296, #298 — closed unmerged 2026-08-19; B-24 has none (fork PR only) | #6315 — closed unmerged 2026-08-19; the loader has none (#6320, the per-entry skip it replaces, closed the same day) | +| `check` at base (2026-11-14) | 6/6 ok | 2/2 ok | +| `check --against` upstream master (2026-11-14) | ok at `3c21479` (6/6) | ok at `a8ac0dc7e` (2/2), **textual only: see R1 in §5** | +| `apply` into a clean clone at base | tree `8d5fda1f9938` (2026-10-09, four patches: `5cbd6f7…`, identical to the local merge `45c816c`) | tree `5c9f3ce18a6b`, **identical to the tree of `4158906f2`**, the head of `overlay/cmdb-all-or-nothing` (2026-10-09: `c66a60d…` = `8f0076b81`) | +| `retired --strict` against upstream master (2026-11-14) | nothing to retire | nothing to retire | +| `self-test` | 21 assertions pass | | + +The core patch is a squash. The fork's loader work sits on `tendcf-integration` +(`4443504`, the pin in `overlay/fork.json`), stacked on B-20, B-7, formatting +commits and an upstream merge that the overlay base does not have, so a +replay does not apply. `libpromises/cmdb.c`, `libpromises/cmdb.h`, +`tests/unit/cmdb_test.c` and `tests/acceptance/00_basics/06_host_specific_data` +are taken from `4443504` byte for byte, plus the one `cmdb_test` line in +`tests/unit/Makefile.am`. **The only difference from `4443504`** in those files is one +secrets-tracking accessor at three call sites: the base has no +`EvalContextVariableGetPlaintext()`, so the call in `cmdb.c` (one) and in +`cmdb_test.c` (two) is spelled `EvalContextVariableGet(ctx, ref, &type)`. +That is the R1 risk in §5. + +**What the manifest says since 2026-11-14.** The `core` `0002` entry has role +`load-bearing`, register `B-5b` (replaced) with `B-7` as a passenger and `B-20` +as load-bearing, and `requires` libntech `0005`/`0006` (B-24). It no longer +reproduces `8f0076b81`, and the `superseded_on_fork` note that said so is gone. +Operator ruling, 2026-11-14, B11 round 2 item 1 (adversary D5, Astra C2): +"1. Replace". Upstream plus this overlay now installs nothing from +`host_specific.json` if any entry is invalid, as the pinned fork does (the +JSON-number difference below aside). +`8f0076b81` (fork PR 59) is retired and never offered upstream. Before this +change (2026-11-02 to 2026-11-14) the manifest's B-5b reproduced the per-entry +skip, which is history ([`upstream-register.md`](upstream-register.md), B-5b +note and "Operator rulings of 2026-11-14 on the queue and the fork", item 2). + +**What is verified and what is not.** The tree equalities above and the +`apply`/`check` runs are mechanical and were run. The patched tree builds and +the loader's tests pass in the `overlay-build` job (`cmdb_test`, `json_test`, +`mustache_test`, the `06_host_specific_data` acceptance directory, +`tests/policy/run.sh` with `TENDCF_REQUIRE_FORK_LOADER=1`); on the Mac nothing +was compiled. What CI proves is that the loader source is the fork's, bar the +R1 accessor, and that its tests pass on both builds. It does not prove equal +behaviour on every input: the fork's libntech pin (`cc892efae`) has a `json.c` +that differs from overlay plus strict parse by 82 lines, and the fork's core +carries a number fix in `rlist.c`. + +**Known difference from the pinned fork: JSON numbers.** The loader source is +the fork's, but two things it calls are not. The fork's libntech +(`cc892efae`) returns a number as it was parsed in `JsonPrimitiveToString()` +and classifies exponent form as a real; the fork's core `rlist.c` does the +same in `RlistAppendContainerPrimitive()`. Neither change is in the overlay +(register B-4 and B-10, and the core `rlist.c` number fix; excluded by §2's +rule). On upstream plus overlay a top-level `vars`/`variables` scalar, or a +primitive array, holding an integer past `long` or exponent form without a dot +(`1e5`, which the parser classifies as an integer) reaches +`StringToLongExitOnError()` from `JsonPrimitiveGetAsInteger()` (libntech +`json.c:850` after apply) and `cf-agent` exits in the install step, after +validation passed: nothing is evaluated, so it is not a partial install, but the +policy cannot refuse or report it. A real installs as `%.2f` (`string_lib.c` +`StringFromDouble()`), so `0.00049` arrives as `0.00`; the fork installs +`0.00049`. Reach: the projector refuses floats (`bin/projector.py:523`) and +writes no exponents, and tendcf's variables are containers, so projected data +does not reach it; a hand-edited file does. No test holds a bare number, so CI +does not see it. Whether B-4, B-10 and the `rlist.c` fix join the overlay is +open with djbclark (not decided here). ## 8. Rejected alternatives diff --git a/docs/architecture/upstream-register.md b/docs/architecture/upstream-register.md index ac68107..7f1c96f 100644 --- a/docs/architecture/upstream-register.md +++ b/docs/architecture/upstream-register.md @@ -327,13 +327,14 @@ Legend: **done** · *pending* · — not applicable. | B-18 | **`SetTimeOut()` arms before `cf_popen()`'s fork publishes the child's pid**, so a short `exec_timeout` can fire in that window on a loaded host with nothing registered to terminate — the same failure B-17 closes, through a different door (arming order, not clearing order). `GenericCreatePipeAndFork()` publishes `ALARM_PID` at `pipes_unix.c:272`, after `SetTimeOut()` has already armed at `verify_exec.c:308`. Previously only a theoretical note in a prior review doc; **empirically confirmed this session** via a real flake in B-17's own unit test under parallel `make check` load, not reproduced on demand but observed directly | core | **done** `ab98a5df4`, cut as a sibling off `dbf759d16` (not stacked on B-17/B-19). `SetTimeOut()` stashes the timeout and leaves the clock stopped; new `StartTimeOutClock()` is called from `GenericCreatePipeAndFork()` immediately after `ALARM_PID = pid`, so no alarm can be pending without a process to kill. One-shot per `SetTimeOut()`, so a second fork under one timeout runs on the remaining time as before; `TIMEOUT_ARMED` is still set pre-fork, leaving the child's `setpgid()` decision untouched. Unit 6/6 → **10/10**, acceptance `08_commands/04_exec_timeout` 6/6 both sides (62s → 74s). Discrimination proved, not asserted: rebuilding with the pre-fix arming order emulated fails exactly 2 of 10 (`test_clock_does_not_run_before_the_fork`, `test_set_leaves_the_clock_stopped`), both passing again on revert | **done** [`fix/exec-timeout-prefork-race`](https://github.com/djbclark/core/tree/fix/exec-timeout-prefork-race) | — (Jira only now) | **done** — found by B-17's 3-model panel, then its own 2-seat panel (grok-4.6, gemini-3.1-pro-high) against a frozen brief; both ship-with-changes, both required changes applied pre-push. **grok found a real platform bug the author missed**: guarding the immediate-arm on `__MINGW32__` alone left **Cygwin** on the deferred path with no starter — `AM_CONDITIONAL([NT])` is `mingw|cygwin` and `pipes_unix.c` builds only under `!NT`, so `exec_timeout` would have silently stopped working there, and `timeout_test` is `!NT` too so no CI job would have caught it. Now `__MINGW32__ || __CYGWIN__`. grok also falsified the commit message's "a leaked armed timeout now goes inert" claim — `TIMEOUT_PENDING` stays set with no timer, and the next `GenericCreatePipeAndFork()` starts that **full** budget against an unrelated child, where the old leaked alarm could only spend its remainder; message corrected, leak sites left to B-15/CFE-4732. Took grok's optional U4 test rewrite (the second `StartTimeOutClock()` is now issued against a *running* clock; the old version's first `alarm(0)` cancelled the timer, so it never exercised the production sequence at `nfs.c:1459`) and its U7 changelog fix (`Changelog: Commit` dumps implementation detail, against `CONTRIBUTING.md`). Skipped U6 (`TIMEOUT_PENDING` → `volatile sig_atomic_t`) — reviewers split, adjudicated for gemini: the variable has no handler interaction, so the annotation would imply concurrent signal state that does not exist. gemini's sole required change was **rejected** on verification: it called the `cf-monitord/history.c:242` arm a "real and severe regression", but `GetMeasurementAttributes()` starts from `ZeroAttributes` and never populates `.contain`, so `a.contain.timeout` is always 0 and that `SetTimeOut()` is unreachable — grok caught this independently, per [[panel-reviewer-weighting]] | — (Jira only now) | **done** [CFE-4734](https://northerntech.atlassian.net/browse/CFE-4734) — updated by comment (159432) with the fix, branch, measurements, discrimination and panel outcome — **PR [#6311](https://github.com/cfengine/core/pull/6311)** opened 2026-08-18 | | B-19 | **`cf_popen*()`'s eight `fdopen()`-failure paths reap the child but never clear `ALARM_PID`, and `RepairExec()`'s `pfp == NULL` branch never calls `ClearTimeOut()`.** `pipes_unix.c:458,470,588,600,670,681,789,800` all do `cf_pwait(pid); return NULL;` without the new `ClearAlarmedPid()`; `verify_exec.c:371-375` returns `ACTION_RESULT_FAILED` without reaching the function's only `ClearTimeOut()` at `:502`. An armed alarm can then stay pointed at a reaped, recyclable pid for up to the full `exec_timeout` — same defect class as B-17, but a window of seconds-to-minutes rather than instructions. Pre-existing, byte-identical to the pre-B-17 parent, gated on rare `fdopen()` failures (ENOMEM/EMFILE). A milder sibling of B-16 (`ShellCommandReturnsZero()`'s identical uncleared-post-reap shape) | core | **done** `453cc264d`, cut from B-17's tip (`8f4ebedbd`). Mechanical: `ClearAlarmedPid(pid)` after each of the eight `cf_pwait()` calls (new forward declaration makes the `static` helper visible), plus `ClearTimeOut()` on `RepairExec()`'s `pfp == NULL` path under the same `a->contain.timeout != CF_NOINT` gate the function's normal-completion path already uses. Panel disproved the original "untestable" claim: `tests/unit/timeout_test.c` gained `test_leftover_alarm_does_not_kill_next_child` (portable POSIX, `RLIMIT_NOFILE` forces a `pipe()` failure, no interposition needed), discriminated by hand (fails exactly at `TimeOutHasFired()` with the fix's `ClearTimeOut()` call removed) | **done** [`fix/exec-timeout-alarm-leak`](https://github.com/djbclark/core/tree/fix/exec-timeout-alarm-leak) | — (Jira only now) | **done** — found by B-17's 3-model panel (fable-deep), then its own 2-seat delta panel (gemini-3.1-pro-high, grok-4.6). grok proved the eight insertions correct by object-code inspection (compiler merges them into four shared tails) and found two more real-but-out-of-scope leaks (raw fd leak on `fdopen()` failure; two more `RepairExec()` early returns skipping `ClearTimeOut()`) that gemini also independently raised — both named in the commit message and left to the existing B-15 family rather than grown into this SHA | — (Jira only now) | **done** [CFE-4735](https://northerntech.atlassian.net/browse/CFE-4735) — updated by comment with the fix, branch, panel outcome and discrimination — **PR [#6312](https://github.com/cfengine/core/pull/6312)** opened 2026-08-18 | | B-5a | Rejected CMDB file names no key, value or path — the `void *data` carrier already exists and is `ARG_UNUSED` | core | **done** `958d13949`, cut from upstream master `a0bca6aaf`. Both `JsonWalk()` visitors now use the previously-`ARG_UNUSED` carrier; the three sections (`vars`/`variables`/`classes`) share one `CheckCMDBDataForUnexpandedVars()` helper that also reports the file path. **Design point:** the helper walks each *top-level entry* separately, not the section as a whole — a single walk can only report the offending primitive's immediate property name, which is the metadata key `value` in the CFE-3633 `variables` format and nothing at all for an array element | **done** [`fix/cmdb-name-offending-key`](https://github.com/djbclark/core/tree/fix/cmdb-name-offending-key) | **done** [#8](https://github.com/djbclark/core/issues/8) | **done** — 2-seat panel (grok-4.6, gemini-3.1-pro-high) run on **disposable read-only copies** after gemini was caught editing the tree under review (see the reviewer-contamination note below). Both SHIP WITH CHANGES, unanimous on Q1–Q5. **grok found a real test gap**: test 12 uses a bare `vars` string whose primitive property name *is* the entry name, so folding the walk back into one section walk would still pass it. Added `15-variable-references-metadata-format.cf` and **proved the gap by simulating the regression** — 12 passes under it, 15 fails. Also took grok's `assert(data != NULL)` + dropped NULL guards, the `JsonGetPropertyAsString` naming consistency (5 uses vs my 1), and its correction that "no behaviour change" was false for the log text | — (Jira only now) | **done** [cfengine/core#6315](https://github.com/cfengine/core/pull/6315) — [CFE-4719](https://northerntech.atlassian.net/browse/CFE-4719), comment 159444 | -| B-5b | One bad key silently drops **every** variable on the host; agent then reports no failures | core | **partially done** `8f0076b81`, stacked on B-5a's `958d13949` (branch cut from `fix/cmdb-name-offending-key`, not master). Fixes only the blast-radius half: `CheckCMDBDataForUnexpandedVars()` rejected the whole `vars`/`variables`/`classes` section on the first bad entry, discarding every other entry in it; checked per entry now, matching the file's existing skip-and-continue idiom (same one B-5a/B-21 already used). The "agent reports no promise failures" half is a genuine design question, left open per the original filing | **done** [`fix/cmdb-one-bad-entry-skips-only-that-entry`](https://github.com/djbclark/core/tree/fix/cmdb-one-bad-entry-skips-only-that-entry) | **done** [#9](https://github.com/djbclark/core/issues/9), updated with the fix and scope note | *skipped* — small, mechanical, directly extends an idiom this file already uses elsewhere (B-3/B-5a); discrimination proven by hand instead (new acceptance test fails unpatched, passes patched, full `00_basics/06_host_specific_data` still 14/14) | — (Jira only now) | **done** [CFE-4720](https://northerntech.atlassian.net/browse/CFE-4720) — comment 159453 (2026-08-18) with the fix, branch, and PR — **PR [#6320](https://github.com/cfengine/core/pull/6320)** opened 2026-08-18, stacked on #6315 | +| B-5b | One bad key silently drops **every** variable on the host; agent then reports no failures | core | **replaced** (fork integration branch 2026-11-02, overlay 2026-11-14: the all-or-nothing loader, see the B-5b note below; what follows is the history of `8f0076b81`, which is retired). **partially done** `8f0076b81`, stacked on B-5a's `958d13949` (branch cut from `fix/cmdb-name-offending-key`, not master). Fixes only the blast-radius half: `CheckCMDBDataForUnexpandedVars()` rejected the whole `vars`/`variables`/`classes` section on the first bad entry, discarding every other entry in it; checked per entry now, matching the file's existing skip-and-continue idiom (same one B-5a/B-21 already used). The "agent reports no promise failures" half is a genuine design question, left open per the original filing | **done** [`fix/cmdb-one-bad-entry-skips-only-that-entry`](https://github.com/djbclark/core/tree/fix/cmdb-one-bad-entry-skips-only-that-entry) | **done** [#9](https://github.com/djbclark/core/issues/9), updated with the fix and scope note | *skipped* — small, mechanical, directly extends an idiom this file already uses elsewhere (B-3/B-5a); discrimination proven by hand instead (new acceptance test fails unpatched, passes patched, full `00_basics/06_host_specific_data` still 14/14) | — (Jira only now) | **done** [CFE-4720](https://northerntech.atlassian.net/browse/CFE-4720) — comment 159453 (2026-08-18) with the fix, branch, and PR — **PR [#6320](https://github.com/cfengine/core/pull/6320)** opened 2026-08-18, stacked on #6315 | | B-6 | `eval()` returns `%lf` for integral results, so arithmetic cannot feed any function taking a count | core | **done** `b74e8f4cf`, cut from upstream master `a0bca6aaf`. New `FnReturnRealOrInt()` renders `%ld` when integral and in range. **Range check is the subtle part**: bounds are asymmetric because `LONG_MIN` is a power of two and converts to double exactly, but `(double) LONG_MAX` rounds *up* to `LONG_MAX + 1`, so `<=` would admit 2^63 and make the cast UB. `sum`/`product`/`mean`/`variance` deliberately untouched — all declared `CF_DATA_TYPE_REAL` with policy assigning them to `real =>` vars, whereas `eval` is `CF_DATA_TYPE_STRING` | **done** [`fix/eval-integral-result`](https://github.com/djbclark/core/tree/fix/eval-integral-result) | **done** [#10](https://github.com/djbclark/core/issues/10) | *not panelled* — written by an Opus 5 subagent with its own discrimination proof (`test_eval_integral_result` fails `"3" != "3.000000"` unpatched, passes patched, shared library forcibly relinked). **Known compatibility risk, stated in the PR**: bundled `eval.cf` was itself asserting `300.000000` etc. and its expectations are updated; any policy string-comparing `eval()` output is affected. Belongs in a minor, not a patch, release. Masterfiles/Enterprise consumers NOT surveyed | — (Jira only now) | **done** [cfengine/core#6318](https://github.com/cfengine/core/pull/6318) — [CFE-4721](https://northerntech.atlassian.net/browse/CFE-4721), comment 159447 | | B-7 | Dotted CMDB keys silently become scope paths, with no warning (**warn only** — do not change behaviour) | core | **done** `ee1f9d60c`, cut from upstream master `a0bca6aaf`. New `LogCMDBVariableScope()` called from `GetCMDBVariableRef()`. **Two levels, to avoid log noise on working configs**: VERBOSE for the documented `[namespace:]bundle.variable` form (indistinguishable from deliberate scope qualification), WARNING only for keys that *cannot* be that form — empty scope (`.key`), empty lval (`key.`), or an lval still containing a dot (`com.dotted.key`). The dotted-lval case is empirically grounded: a `vars:` promise splits its promiser identically, so no policy can create a dot-containing variable name. **Also fixed** the three `Installing CMDB variable` verbose messages, which printed `'%s:%s.%s'` and so double-printed the scope (`data:com.com.dotted.key`) — the very address the operator needs | **done** [`fix/cmdb-dotted-key-warning`](https://github.com/djbclark/core/tree/fix/cmdb-dotted-key-warning) | **done** [#11](https://github.com/djbclark/core/issues/11) | *not panelled* — Opus 5 subagent, discrimination proved, **and independently re-verified by the owning session**: warnings fire on exactly the three malformed shapes, `myscope.myvar` is silent (false-positive guard holds), all five variables install at unchanged addresses. Behaviour non-regression proved by byte-identical `--show-vars`/`--show-classes` diffs. `classes` confirmed NOT affected (canonified, not scope-parsed) | — (Jira only now) | **done** [cfengine/core#6317](https://github.com/cfengine/core/pull/6317) — [CFE-4722](https://northerntech.atlassian.net/browse/CFE-4722), comment 159446 | | B-20 | **A JSON `null` value in `host_specific.json` segfaults `cf-promises` and `cf-agent`.** Valid JSON, passes CFEngine's own unresolved-variable check. `JsonPrimitiveToString()` returns NULL for `JSON_PRIMITIVE_TYPE_NULL`; `AddCMDBVariable()` (`cmdb.c:136`) passes it straight in as the value → `RvalNewRewriter` → `xstrdup(NULL)` → `strlen(NULL)`. Agent dies in `GenericAgentDiscoverContext()` **before any promise is evaluated**, so the host falls to failsafe — same severity class as B-10. Found while edge-case-testing B-5a; **confirmed pre-existing on master by rebuilding with unpatched `cmdb.c`** | core | **done** `87490bb8f`. Two guards in `AddCMDBVariable()` plus a `JsonArrayContainsNull()` helper. **Skip-with-error, not reject-the-section** — verified against the file's own idiom, which `continue`s for every per-entry problem and reserves `return false` for section-level structural faults. Six inputs fixed, incl. two silent-corruption cases (`[1, null]` installed `{"1"}`; `[null]` installed an empty slist). Nulls inside data containers deliberately untouched (they render correctly); `classes` already safe because `JsonPrimitiveGetAsString()` returns the literal `"null"` | **done** [`fix/cmdb-null-value-crash`](https://github.com/djbclark/core/tree/fix/cmdb-null-value-crash) | — (Jira only now) | *not panelled* — Opus 5 subagent; discrimination proved with the stale-library trap defeated, and the unpatched rebuild confirmed to genuinely segfault (rc=139) first. Test pins skip-and-continue semantics, not merely absence of a crash | — (Jira only now) | **done** [cfengine/core#6316](https://github.com/cfengine/core/pull/6316) — [CFE-4738](https://northerntech.atlassian.net/browse/CFE-4738), comment 159445 | | B-21 | **The augments loader (`def.json`) has the same null-crash root cause**, at three sites in `libpromises/generic_agent.c`: `:454` (`vars`), `:606` (`variables` — note `{"value": null}` is *safe* there, that path has a `NULL_JSON` guard `cmdb.c` lacks) and `:723` (`classes`, NULL into `CheckContextClassmatch()`). The array paths `:749`/`:792`/`:874` are safe because `JsonIteratorNextValueByType(..., skip_null=true)` filters nulls. Found while fixing B-20 | core | **done** `dd917f09d`, cut from upstream master `a0bca6aaf`. Five guards added in `LoadAugmentsData()`, following B-20's `cmdb.c` idiom exactly (skip-with-error-and-continue, not reject-the-section): the three scalar crash sites (`vars`, `variables` bare form, `classes`) plus the `vars`/`variables` array-truncation sites (`RlistFromContainer()` silently drops nulls, same non-crashing defect class B-20 also fixed). New file-local `JsonArrayContainsNull()` helper, verbatim copy of B-20's. Discrimination proved by hand, not asserted: unpatched rebuild (forced full `libpromises` relink, not just recompile) gave a real SIGSEGV (rc=139) loading the new test's `def.json`; patched rebuild passes it plus all 21 other tests in `00_basics/def.json/`, zero regressions | **done** [`fix/def-json-null-crash`](https://github.com/djbclark/core/tree/fix/def-json-null-crash) | — (Jira only now) | **done** — 1-seat panel (grok-4.6) on a disposable `cp -a` copy; gemini's seat failed twice in this environment (sandbox couldn't see the filesystem at all, unrelated to the code) so this is single-reviewer, weighted per [[panel-reviewer-weighting]]. SHIP: no leaks on the five early-return paths (verified against a memory-allocated/freed table for each), no missed null-crash site left in the function, `{"value": null}` confirmed already-safe pre-existing via `NULL_JSON()`, `--show-vars` survivor ordering matches `SeqSort()`. **grok independently surfaced a sixth site**: the `"inputs"` array has the identical `RlistFromContainer()` truncation defect (not a crash) as the two array sites already fixed; folded into this same commit rather than filed separately, matching B-20's own decision to bundle scalar+array fixes together | — (Jira only now) | **done** [cfengine/core#6319](https://github.com/cfengine/core/pull/6319) — [CFE-4739](https://northerntech.atlassian.net/browse/CFE-4739), comment 159449 | -| B-22 | **Not a bug — `libutils/mustache.c` (916 lines, sole entry point `MustacheRender()`) had zero unit test coverage**, flagged by B-13's review panel as a follow-up. `cfengine/core` has its own `mustache_test.c` + JSON spec fixtures, but they cover only comments/interpolation/sections/inverted/delimiters, none of the CFEngine-specific extensions | libntech | **done** — 17 cmocka cases (~90 assertions): variables (incl. whitespace-padded tags, ints, bools), missing/null/non-object/NULL hash, HTML escaping plus both unescaped forms, dotted names, comments (incl. CRLF), boolean/array/object/inverted sections, the CFEngine extensions `{{.}}`/`{{@}}`/`{{%x}}`/`{{$x}}`/`{{-top-}}`, delimiter changes, standalone-tag line removal, malformed templates, empty input. Wired into `tests/unit/Makefile.am` | **done** [`test/mustache-coverage`](https://github.com/djbclark/libntech/tree/test/mustache-coverage) | — (Jira only now) | **done** — built under `-Werror -Wextra -Wall` and separately under `-fsanitize=address`, both clean; `make check` 40/40 binaries pass | — (Jira only now) | **done** [NorthernTechHQ/libntech#296](https://github.com/NorthernTechHQ/libntech/pull/296) — pure coverage addition, no behavior change | — (test-only, no ticket) | -| B-23 | **Two minor defects in `mustache.c`, found while writing B-22's tests.** `:284,297` use `%20s` (min width) where `%.20s` (truncate) was meant, per the trailing `"..."` — short strings get left-padded, long ones dump the whole remaining template into the log. `IsTagStandalone()` (`:74`) forms `tag_start - 1` when a non-renderable tag starts at offset 0 (exercised by the existing `"{{#b}}yes{{/b}}"` case), pointing one before the input buffer — never dereferenced, so no observed failure under UBSan pointer-overflow or ASan, but UB to construct regardless | libntech | **done** — `%.20s` fix; guard added so the out-of-bounds pointer is never formed | **done** [`fix/mustache-minor-defects`](https://github.com/djbclark/libntech/tree/fix/mustache-minor-defects) — stacked on B-22 | — (Jira only now) | **done** — reverted-then-restored the `%20s` fix and observed the log-padding difference directly (padding present unpatched, absent patched); UB fix has **no reproducing failure mode** — stated plainly in the PR rather than claimed as verified | — (Jira only now) | **done** [NorthernTechHQ/libntech#298](https://github.com/NorthernTechHQ/libntech/pull/298) — stacked on #296 | [CFE-4740](https://northerntech.atlassian.net/browse/CFE-4740) | +| B-22 | **Not a bug — `libutils/mustache.c` (916 lines, sole entry point `MustacheRender()`) had zero unit test coverage**, flagged by B-13's review panel as a follow-up. `cfengine/core` has its own `mustache_test.c` + JSON spec fixtures, but they cover only comments/interpolation/sections/inverted/delimiters, none of the CFEngine-specific extensions | libntech | **done** — 17 cmocka cases (~90 assertions): variables (incl. whitespace-padded tags, ints, bools), missing/null/non-object/NULL hash, HTML escaping plus both unescaped forms, dotted names, comments (incl. CRLF), boolean/array/object/inverted sections, the CFEngine extensions `{{.}}`/`{{@}}`/`{{%x}}`/`{{$x}}`/`{{-top-}}`, delimiter changes, standalone-tag line removal, malformed templates, empty input. Wired into `tests/unit/Makefile.am` | **done** [`test/mustache-coverage`](https://github.com/djbclark/libntech/tree/test/mustache-coverage) | — (Jira only now) | **done** — built under `-Werror -Wextra -Wall` and separately under `-fsanitize=address`, both clean; `make check` 40/40 binaries pass | — (Jira only now) | **done** [NorthernTechHQ/libntech#296](https://github.com/NorthernTechHQ/libntech/pull/296) — pure coverage addition, no behavior change; test-only, no ticket | +| B-23 | **Two minor defects in `mustache.c`, found while writing B-22's tests.** `:284,297` use `%20s` (min width) where `%.20s` (truncate) was meant, per the trailing `"..."` — short strings get left-padded, long ones dump the whole remaining template into the log. `IsTagStandalone()` (`:74`) forms `tag_start - 1` when a non-renderable tag starts at offset 0 (exercised by the existing `"{{#b}}yes{{/b}}"` case), pointing one before the input buffer — never dereferenced, so no observed failure under UBSan pointer-overflow or ASan, but UB to construct regardless | libntech | **done** — `%.20s` fix; guard added so the out-of-bounds pointer is never formed | **done** [`fix/mustache-minor-defects`](https://github.com/djbclark/libntech/tree/fix/mustache-minor-defects) — stacked on B-22 | — (Jira only now) | **done** — reverted-then-restored the `%20s` fix and observed the log-padding difference directly (padding present unpatched, absent patched); UB fix has **no reproducing failure mode** — stated plainly in the PR rather than claimed as verified | — (Jira only now) | **done** [NorthernTechHQ/libntech#298](https://github.com/NorthernTechHQ/libntech/pull/298) — stacked on #296 — [CFE-4740](https://northerntech.atlassian.net/browse/CFE-4740) | +| B-24 | **`JsonParseStrict()` / `JsonParseFileStrict()`: a JSON parse that refuses what `JsonParse()` repairs or ignores.** A repeated object member replaces the earlier one (an invalid first value is never validated) and anything after the document is not looked at. The core CMDB loader (B-5b replaced, R13) reads `host_specific.json` through `JsonParseFileStrict()`, so a file that the strict parse refuses installs nothing. `8a57e55` fixes the file variant: a null byte cut the text off, and the check that compared the lengths could never fire, so `{"vars": {}}not-json` parsed as the valid object | libntech | **done** `dc3fba2` + `8a57e55`, cut from upstream master `3c21479` | **done** [`fix/json-strict-parse`](https://github.com/djbclark/libntech/tree/fix/json-strict-parse) | **done** [fork PR #11](https://github.com/djbclark/libntech/pull/11) | — | — | **not yet offered upstream** (upstream queue item 6, ruling 4 below); in the overlay as libntech `0005`/`0006` (load-bearing for the core loader) and in the fork's `tendcf-integration` | | B-8 | `commands:` promise that exceeded `exec_timeout` is reported **compliant** — `RepairExec()` never returns `ACTION_RESULT_TIMEOUT`, so the promise is judged only on the child's exit status. **This is the actual fail-open**; B-1 only narrows its window. (Said "reported **kept**" until 2026-08-17; the panel's correction is that the default outcome for exit 0 is *repaired*, so the accurate word is **compliant** — 100% aggregate, `promise_repaired` set, `repair_timeout` not set, `PromiseResultIsOK()` true. Defect unchanged, our label was wrong) | core | **done** `326bcdb8d` + `7a32e3969` + `57ac8da22`, plus five acceptance tests in `46be075d4` | **done** [`fix/exec-timeout-promise-result`](https://github.com/djbclark/core/tree/fix/exec-timeout-promise-result) | **done** [#6](https://github.com/djbclark/core/issues/6) | **done** — its own 3-model panel (cursor/gemini/grok, 2026-08-16); grok found a real defect in the fix (the flag was sampled before `cf_pclose`), fixed in `7a32e3969`, and caught the unconditional "and was terminated" wording, fixed in `57ac8da22` | **done** — `security@` 2026-08-17, Gmail id `1a00d22ac0d46c9b`, + correcting follow-up `1a00d44a2758b9ea` carrying the retracted ALARM_PID refutation | **done** [cfengine/core#6299](https://github.com/cfengine/core/pull/6299) — OPEN, MERGEABLE, 2 commits, branch [`fix/exec-timeout-promise-outcome`](https://github.com/djbclark/core/tree/fix/exec-timeout-promise-outcome) cut from master `17eb78e6d`, trailers `Ticket: CFE-4726` + `Changelog:`. Byte-identical tree to `fix/exec-timeout-promise-result`, recommitted in the project's own style (past-tense subjects) with the two review defects squashed into the fix rather than shipped as separate correcting commits. PR body states all three known limits up front | [CFE-4726](https://northerntech.atlassian.net/browse/CFE-4726) | | P-1 | Retain the changes chroot after a `--simulate` run (feature) | core | **done** `f6c06f9e2` (corrected 2026-08-17) | `simulate-keep-chroot` | **done** [#2](https://github.com/djbclark/core/issues/2) | *not done* | — not emailed; went straight to an upstream PR (operator, 2026-08-15). Verified 2026-08-17: only four threads to northern.tech exist and none concerns `--simulate` | **DONE** [cfengine/core#6293](https://github.com/cfengine/core/pull/6293) — open, mergeable, CLA signed | | P-2 | `--simulate-json`: machine-readable rendering of the change set (feature) | core | **done** `b3a6c3da5` (corrected 2026-08-17) | `simulate-json` | **done** [#3](https://github.com/djbclark/core/issues/3) | *not done* | — not emailed; went straight to an upstream PR (operator, 2026-08-15). Verified 2026-08-17: only four threads to northern.tech exist and none concerns `--simulate` | **DONE** [cfengine/core#6294](https://github.com/cfengine/core/pull/6294) — open, mergeable, CLA signed | @@ -363,15 +364,23 @@ anything but whitespace after the document refuse the file before validation. A class that exists already is left as it is and the rest is installed. Strict parsing is the CMDB loader's alone: `readjson` and `def.json` stay lenient. Upstream PRs #6315/#6320 -stay closed unmerged. The overlay manifest's B-5b entry (`overlay/overlay.json`) -still describes and reproduces `8f0076b81`, not this behaviour; it now carries -a `superseded_on_fork` note saying so, and -[`upstream-overlay.md`](upstream-overlay.md) section 7 repeats it. Whether the -overlay keeps, retires or replaces that patch was ruled on 2026-11-14 (Replace: -the overlay will carry the all-or-nothing loader instead of the per-entry skip), -and is done in a later, separate PR because it changes what the overlay -reproduces. The ruling is recorded in full under "Operator rulings of -2026-11-14 on the queue and the fork" below. +stay closed unmerged. + +**B-5b replaced in the overlay, 2026-11-14 (overlay phase B).** From +2026-11-02 to 2026-11-14 the overlay manifest's B-5b entry still reproduced +`8f0076b81` (the per-entry skip) and carried a `superseded_on_fork` note. +Operator ruling 2026-11-14 ("Replace", item 2 below): `overlay/core/0002` is +now the all-or-nothing loader, a squash of the fork's `4443504` onto the +overlay base plus B-5a (`djbclark/core` branch `overlay/cmdb-all-or-nothing`, +commit `4158906f2`, tag `tendcf-pin-4158906`). It also carries B-20 (the +null-value crash fix, `87490bb8f`, load-bearing: the loader keeps its null +guards but refuses the whole file, where B-20 skipped the entry) and B-7 (the dotted-key warning, `ee1f9d60c`, a passenger), as the +fork's final `cmdb.c` contains both. The strict parse comes from two libntech +patches, B-24 (`dc3fba2`, `8a57e55`, fork PR 11). The one change from `4443504` +is the secrets-tracking accessor spelled `EvalContextVariableGet()` at three +call sites because the base has no `EvalContextVariableGetPlaintext()` +([`upstream-overlay.md`](upstream-overlay.md) sections 5 and 7, risk R1). +`8f0076b81` stays retired and is never offered upstream. ### Operator rulings of 2026-11-14 on the queue and the fork @@ -396,9 +405,10 @@ Ruled, one item per line. Quotes are the operator's chosen options, exactly. part only (no strict parse). Upstream is later offered the core all-or-nothing series (queue item 7 of e0d15d93's upstream queue, still gated on D1/D2 and on libntech item 6). `8f0076b81` (fork PR 59) is - retired and never offered upstream. Status: ruled; implementation pending - in a follow-up PR (overlay phase B); queued as build work. The overlay - side is in [`upstream-overlay.md`](upstream-overlay.md) section 7. + retired and never offered upstream. Status: ruled 2026-11-14; implemented in + the overlay phase B PR (the overlay carries libntech `0005`/`0006`, B-24, + so the strict parse is included, not core only). The overlay side is in + [`upstream-overlay.md`](upstream-overlay.md) section 7. 3. **Fork-only assertions in `tests/policy/run.sh`.** Operator ruling, 2026-11-14, B11 round 2 item 2 (adversary D1, Astra r2 B3): "1. Keep Mac, SKIPs". The Mac dev build stays a `run.sh` target and prints an explicit diff --git a/overlay/core/0002-Fixed-one-bad-CMDB-entry-discarding-every-entry-in-i.patch b/overlay/core/0002-Fixed-one-bad-CMDB-entry-discarding-every-entry-in-i.patch deleted file mode 100644 index 2c17675..0000000 --- a/overlay/core/0002-Fixed-one-bad-CMDB-entry-discarding-every-entry-in-i.patch +++ /dev/null @@ -1,320 +0,0 @@ -From 8f0076b814e8a0d69e48baeab8e637926e4d111a Mon Sep 17 00:00:00 2001 -From: Daniel JB Clark -Date: Tue, 18 Aug 2026 18:01:42 -0400 -Subject: [PATCH 2/2] Fixed one bad CMDB entry discarding every entry in its - section - -CheckCMDBDataForUnexpandedVars() rejected a whole 'vars'/'variables'/ -'classes' section on the first entry containing an unresolved variable -reference, discarding every other entry in that section along with it. -Checked per entry instead, inside each section's own install loop, -matching this file's existing skip-and-continue idiom for other -per-entry problems (an invalid variable specification, a missing -'value' field). - -New acceptance test: one good and one bad entry in the same 'vars' -section, asserting the bad entry is still rejected and named, and the -good entry is installed anyway. Discrimination proved: fails on the -unpatched function, passes with the fix; full 00_basics/06 suite still -14/14. - -Changelog: Title -Ticket: CFE-4720 - -diff --git a/libpromises/cmdb.c b/libpromises/cmdb.c -index a874071ab8b19d5d49c065f70b19f9ca3c7d3d51..32c460f92a6f782f36bf44c3ad7fb54b6c36dd9d 100644 ---- a/libpromises/cmdb.c -+++ b/libpromises/cmdb.c -@@ -116,73 +116,66 @@ static bool CheckObjectForUnexpandedVars(JsonElement *object, void *data) - } - - /** -- * Reject CMDB data containing unresolved variable references, naming the entry -- * that was rejected. -+ * Reject a single CMDB entry containing unresolved variable references, -+ * naming the entry that was rejected. - * -- * Each top-level entry is walked separately so that the entry can be named, -- * which is what an operator needs to find the offending line. A single walk of -- * the whole section could only report the immediate property name, and in the -- * 'variables' format (CFE-3633) that is the metadata key 'value' rather than -- * the name of the variable holding it. -+ * Checked per entry, rather than once for the whole section, so that one -+ * offending entry does not take every other entry in the section down with -+ * it -- matching the skip-and-continue idiom the rest of this file already -+ * uses for other per-entry problems (an invalid variable specification, a -+ * missing "value" field). - * -- * @param data the CMDB section to check, which must be a JSON object -+ * @param entry the entry's value, as found under key in its section -+ * @param key the entry's key, as it appears in the CMDB file - * @param section the section's key, as it appears in the CMDB file - * @param file_path the CMDB file the data came from -- * @return whether the data is free of variable references -+ * @return whether the entry is free of variable references - */ --static bool CheckCMDBDataForUnexpandedVars(JsonElement *data, -- const char *section, -- const char *file_path) -+static bool CheckCMDBEntryForUnexpandedVars(JsonElement *entry, -+ const char *key, -+ const char *section, -+ const char *file_path) - { -- assert(data != NULL); -- assert(JsonGetType(data) == JSON_TYPE_OBJECT); -+ assert(entry != NULL); -+ assert(key != NULL); - assert(section != NULL); - assert(file_path != NULL); - -- JsonIterator iter = JsonIteratorInit(data); -- while (JsonIteratorHasMore(&iter)) -+ if (StringContainsUnresolved(key)) - { -- JsonElement *child = JsonIteratorNextValue(&iter); -- const char *entry = JsonGetPropertyAsString(child); -- -- if (StringContainsUnresolved(entry)) -- { -- Log(LOG_LEVEL_ERR, -- "Invalid '%s' CMDB data in '%s', cannot contain variable" -- " references (offending key: '%s')", -- section, file_path, entry); -- return false; -- } -+ Log(LOG_LEVEL_ERR, -+ "Invalid '%s' CMDB data in '%s', cannot contain variable" -+ " references (offending key: '%s')", -+ section, file_path, key); -+ return false; -+ } - -- CMDBUnexpandedVar offender = {NULL, NULL}; -- if (JsonWalk(child, CheckObjectForUnexpandedVars, -- NULL, CheckPrimitiveForUnexpandedVars, &offender)) -- { -- continue; -- } -+ CMDBUnexpandedVar offender = {NULL, NULL}; -+ if (JsonWalk(entry, CheckObjectForUnexpandedVars, -+ NULL, CheckPrimitiveForUnexpandedVars, &offender)) -+ { -+ return true; -+ } - -- /* The walk stops only when one of the two visitors above stops it, -- * and each records what it found before doing so. */ -- assert((offender.value != NULL) || (offender.key != NULL)); -+ /* The walk stops only when one of the two visitors above stops it, and -+ * each records what it found before doing so. */ -+ assert((offender.value != NULL) || (offender.key != NULL)); - -- if (offender.value != NULL) -- { -- Log(LOG_LEVEL_ERR, -- "Invalid '%s' CMDB data in '%s', cannot contain variable" -- " references (offending value in entry '%s': '%s')", -- section, file_path, entry, offender.value); -- } -- else -- { -- Log(LOG_LEVEL_ERR, -- "Invalid '%s' CMDB data in '%s', cannot contain variable" -- " references (offending key in entry '%s': '%s')", -- section, file_path, entry, offender.key); -- } -- return false; -+ if (offender.value != NULL) -+ { -+ Log(LOG_LEVEL_ERR, -+ "Invalid '%s' CMDB data in '%s', cannot contain variable" -+ " references (offending value in entry '%s': '%s')", -+ section, file_path, key, offender.value); - } -- -- return true; -+ else -+ { -+ Log(LOG_LEVEL_ERR, -+ "Invalid '%s' CMDB data in '%s', cannot contain variable" -+ " references (offending key in entry '%s': '%s')", -+ section, file_path, key, offender.key); -+ } -+ return false; - } - - static VarRef *GetCMDBVariableRef(const char *key) -@@ -272,17 +265,17 @@ static bool ReadCMDBVars(EvalContext *ctx, JsonElement *vars, - return false; - } - -- if (!CheckCMDBDataForUnexpandedVars(vars, "vars", file_path)) -- { -- return false; -- } -- - JsonIterator iter = JsonIteratorInit(vars); - while (JsonIteratorHasMore(&iter)) - { - const char *key = JsonIteratorNextKey(&iter); - JsonElement *data = JsonObjectGet(vars, key); - -+ if (!CheckCMDBEntryForUnexpandedVars(data, key, "vars", file_path)) -+ { -+ continue; -+ } -+ - VarRef *ref = GetCMDBVariableRef(key); - if (ref == NULL) - { -@@ -375,15 +368,16 @@ static bool ReadCMDBVariables(EvalContext *ctx, JsonElement *variables, - return false; - } - -- if (!CheckCMDBDataForUnexpandedVars(variables, "variables", file_path)) -- { -- return false; -- } -- - JsonIterator iter = JsonIteratorInit(variables); - while (JsonIteratorHasMore(&iter)) - { - const char *key = JsonIteratorNextKey(&iter); -+ JsonElement *const var_info = JsonObjectGet(variables, key); -+ -+ if (!CheckCMDBEntryForUnexpandedVars(var_info, key, "variables", file_path)) -+ { -+ continue; -+ } - - VarRef *ref = GetCMDBVariableRef(key); - if (ref == NULL) -@@ -391,8 +385,6 @@ static bool ReadCMDBVariables(EvalContext *ctx, JsonElement *variables, - continue; - } - -- JsonElement *const var_info = JsonObjectGet(variables, key); -- - JsonElement *data; - StringSet *tags; - const char *comment = NULL; -@@ -480,16 +472,17 @@ static bool ReadCMDBClasses(EvalContext *ctx, JsonElement *classes, - return false; - } - -- if (!CheckCMDBDataForUnexpandedVars(classes, "classes", file_path)) -- { -- return false; -- } -- - JsonIterator iter = JsonIteratorInit(classes); - while (JsonIteratorHasMore(&iter)) - { - const char *key = JsonIteratorNextKey(&iter); - JsonElement *data = JsonObjectGet(classes, key); -+ -+ if (!CheckCMDBEntryForUnexpandedVars(data, key, "classes", file_path)) -+ { -+ continue; -+ } -+ - if (JsonGetElementType(data) == JSON_ELEMENT_TYPE_PRIMITIVE) - { - const char *expr = JsonPrimitiveGetAsString(data); -diff --git a/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf -new file mode 100644 -index 0000000000000000000000000000000000000000..9cc38dd9975f9c87da01c04a32d73909a7ef7c3f ---- /dev/null -+++ b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf -@@ -0,0 +1,78 @@ -+# test that one entry with an unresolved variable reference does not drop -+# every other entry in the same section -+body common control -+{ -+ inputs => { "../../default.sub.cf" }; -+ bundlesequence => { default("$(this.promise_filename)") }; -+ version => "1.0"; -+} -+ -+####################################################### -+bundle agent init -+{ -+ files: -+ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)." -+ create => "true", -+ perms => mog( -+ "0700", "$(sys.user_data[username])", "$(sys.user_data[username])" -+ ); -+ -+ methods: -+ "prepare_host_specific_data" -+ usebundle => file_copy( -+ "$(this.promise_filename).json", -+ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)host_specific.json" -+ ), -+ handle => "prepare_host_specific_data"; -+ -+ "empty_promises_cf" -+ usebundle => file_make("$(sys.inputdir)/promises.cf", ''); -+} -+ -+####################################################### -+bundle agent check -+{ -+ vars: -+ "error_output" -+ string => execresult( -+ "$(sys.cf_promises) --show-classes -f $(sys.inputdir)/promises.cf", -+ "useshell" -+ ), -+ depends_on => { "prepare_host_specific_data" }; -+ -+ "vars_output" -+ string => execresult( -+ "$(sys.cf_promises) --show-vars -f $(sys.inputdir)/promises.cf | $(G.grep) good_key", -+ "useshell" -+ ), -+ depends_on => { "prepare_host_specific_data" }; -+ -+ classes: -+ # The offending entry is still rejected and named. -+ "offender_named" -+ expression => regcmp( -+ ".*error: Invalid 'vars' CMDB data in '.*host_specific\.json', cannot contain variable references \(offending value in entry 'key_with_unresolved_value': .*", -+ "$(error_output)" -+ ); -+ -+ # Its sibling entry in the same section is installed anyway -- a single -+ # bad entry must not take the rest of the section down with it. -+ "good_entry_survived" -+ expression => regcmp( -+ "data:variables\.good_key\s+a literal value\s+source=cmdb", -+ "$(vars_output)" -+ ); -+ -+ "ok" and => { "offender_named", "good_entry_survived" }; -+ -+ reports: -+ ok:: -+ "$(this.promise_filename) Pass"; -+ -+ !ok:: -+ "$(this.promise_filename) FAIL"; -+ -+ !ok.DEBUG:: -+ "error_output was: '$(error_output)'"; -+ "vars_output was: '$(vars_output)'"; -+} -diff --git a/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf.json b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf.json -new file mode 100644 -index 0000000000000000000000000000000000000000..5f225ddcc8698d6d45fa38adcc4f60fcbdac6dba ---- /dev/null -+++ b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf.json -@@ -0,0 +1,6 @@ -+{ -+ "vars": { -+ "good_key": "a literal value", -+ "key_with_unresolved_value": "$(sys.workdir)" -+ } -+} diff --git a/overlay/core/0002-Made-CMDB-loading-all-or-nothing-one-bad-entry-insta.patch b/overlay/core/0002-Made-CMDB-loading-all-or-nothing-one-bad-entry-insta.patch new file mode 100644 index 0000000..fe02886 --- /dev/null +++ b/overlay/core/0002-Made-CMDB-loading-all-or-nothing-one-bad-entry-insta.patch @@ -0,0 +1,2692 @@ +From 4158906f2a5b3f202af1615fb80357a9a85987cc Mon Sep 17 00:00:00 2001 +From: Daniel JB Clark +Date: Sat, 14 Nov 2026 22:00:37 -0500 +Subject: [PATCH] Made CMDB loading all-or-nothing: one bad entry installs + nothing + +Replaces the per-entry skip of "Fixed one bad CMDB entry discarding +every entry in its section" (8f0076b81). A bad entry in host_specific.json +now rejects the whole file: the loader validates every section first +and installs nothing unless all of it is valid. A skipped entry used to +leave whatever it guarded (an interlock, for example) unguarded. + +The loader keeps the pending classes and variables, installs them only +after every check has passed, and rolls back on a failed insertion. It +parses with JsonParseFileStrict() (libntech), which refuses a repeated +object member and any non-whitespace after the document. Null values +no longer crash it, and a dotted variable key gets a warning. + +This is a squash of the fork's tendcf-integration cmdb loader work +(djbclark/core 4443504) onto the overlay base: libpromises/cmdb.c, +tests/unit/cmdb_test.c and tests/acceptance/00_basics/06_host_specific_data +are taken from that commit, plus the cmdb_test line in +tests/unit/Makefile.am. The one difference is the secrets-tracking +accessor: the base has EvalContextVariableGet(ctx, ref, type_out), so +EvalContextVariableGetPlaintext() is spelled that way here (three call +sites: one in cmdb.c, two in cmdb_test.c). + +It includes the null-value crash fix (B-20, 87490bb8f) and the +dotted-key warning (B-7, ee1f9d60c) because the final cmdb.c contains +them. Requires libntech with JsonParseFileStrict() (fork PR +djbclark/libntech#11). +--- + libpromises/cmdb.c | 910 +++++++++++++++--- + .../06_host_specific_data/07-invalid-data.cf | 10 +- + .../08-class-expression.cf | 9 +- + .../09-class-reg-expression.cf | 9 +- + .../06_host_specific_data/13-null-values.cf | 60 ++ + .../13-null-values.cf.json | 13 + + .../06_host_specific_data/14-dotted-keys.cf | 52 + + .../14-dotted-keys.cf.json | 4 + + ...ariable-references-metadata-format.cf.json | 4 +- + ...e-references-bad-entry-installs-nothing.cf | 87 ++ + ...erences-bad-entry-installs-nothing.cf.json | 6 + + .../17-duplicate-member-installs-nothing.cf | 93 ++ + ...-duplicate-member-installs-nothing.cf.json | 10 + + .../18-trailing-input-installs-nothing.cf | 93 ++ + ...18-trailing-input-installs-nothing.cf.json | 9 + + tests/unit/Makefile.am | 1 + + tests/unit/cmdb_test.c | 779 +++++++++++++++ + 17 files changed, 1983 insertions(+), 166 deletions(-) + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf.json + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf.json + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf.json + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf.json + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf + create mode 100644 tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf.json + create mode 100644 tests/unit/cmdb_test.c + +diff --git a/libpromises/cmdb.c b/libpromises/cmdb.c +index a874071ab..0644e6c26 100644 +--- a/libpromises/cmdb.c ++++ b/libpromises/cmdb.c +@@ -23,13 +23,17 @@ + */ + + #include ++#include /* isspace() */ + #include + #include + #include /* StringSet */ ++#include /* Seq */ ++#include /* CF_MAXVARSIZE */ + #include + #include /* GetDataDir() */ + #include /* VarRef, StringContainsUnresolved() */ + #include /* EvalContext*() */ ++#include /* Class */ + #include /* JoinPaths() */ + + #include +@@ -45,12 +49,36 @@ + #define CMDB_CLASSES_REGULAR_EXPRESSIONS "regular_expressions" + #define CMDB_COMMENT_KEY "comment" + +-JsonElement *ReadJsonFile(const char *filename, LogLevel log_level, size_t size_max) ++#ifdef TEST_UNIT_TEST ++/** ++ * Only in the unit test build (cmdb_test.c includes this file with ++ * TEST_UNIT_TEST defined, as split_process_line_test.c does for ++ * processes_select.c): a variable or class whose CMDB key makes the hook ++ * return true is treated as one that cannot be inserted into the EvalContext, ++ * which validation cannot otherwise produce. It lets a test exercise the ++ * rollback of LoadCMDBData(). Production has no such pointer. ++ */ ++static bool (*CMDBTestFailInsertion)(const char *key) = NULL; ++#define CMDBInsertionForcedToFail(key) \ ++ (CMDBTestFailInsertion != NULL && CMDBTestFailInsertion(key)) ++#else ++#define CMDBInsertionForcedToFail(key) false ++#endif ++ ++/** ++ * @param strict refuse what JsonParseFile() would repair or ignore: a repeated ++ * object member (it replaces the earlier one) and anything but ++ * whitespace after the document (it is not looked at) ++ */ ++static JsonElement *ReadJsonFileMode(const char *filename, LogLevel log_level, ++ size_t size_max, bool strict) + { + assert(filename != NULL); + + JsonElement *doc = NULL; +- JsonParseError err = JsonParseFile(filename, size_max, &doc); ++ JsonParseError err = strict ? ++ JsonParseFileStrict(filename, size_max, &doc) : ++ JsonParseFile(filename, size_max, &doc); + + if (err == JSON_PARSE_ERROR_NO_SUCH_FILE) + { +@@ -67,6 +95,11 @@ JsonElement *ReadJsonFile(const char *filename, LogLevel log_level, size_t size_ + return doc; + } + ++JsonElement *ReadJsonFile(const char *filename, LogLevel log_level, size_t size_max) ++{ ++ return ReadJsonFileMode(filename, log_level, size_max, false); ++} ++ + /* What made the unexpanded-variable walk below stop: exactly one of these is + * set, depending on which visitor stopped it. Both point into the JSON + * document being walked and must not be freed. */ +@@ -116,77 +149,129 @@ static bool CheckObjectForUnexpandedVars(JsonElement *object, void *data) + } + + /** +- * Reject CMDB data containing unresolved variable references, naming the entry +- * that was rejected. ++ * Report that the dots in a CMDB key were taken as a scope separator. + * +- * Each top-level entry is walked separately so that the entry can be named, +- * which is what an operator needs to find the offending line. A single walk of +- * the whole section could only report the immediate property name, and in the +- * 'variables' format (CFE-3633) that is the metadata key 'value' rather than +- * the name of the variable holding it. ++ * VarRefParse() splits a key at its first dot, so the flat-looking key ++ * "com.dotted.key" does not become a variable of that name, it becomes the ++ * variable "dotted.key" in the scope "com". Naming a scope like this is a ++ * supported feature, so a key of the documented "[namespace:]bundle.variable" ++ * form is only reported at the verbose level; warning on every agent run ++ * about a working policy would just be noise. + * +- * @param data the CMDB section to check, which must be a JSON object ++ * The remaining keys cannot be of that form: it has room for exactly one dot, ++ * and neither the bundle name nor the variable name can be empty. A variable ++ * name that still contains a dot is a particularly good hint, since a vars ++ * promise cannot create one (the same splitting happens there), so nothing ++ * but CMDB data can address it. Those keys are much more likely to be flat ++ * keys meant literally, and get a warning. ++ */ ++static void LogCMDBVariableScope(const char *key, const VarRef *ref) ++{ ++ assert(key != NULL); ++ assert(ref != NULL); ++ ++ char *address = VarRefToString(ref, true); ++ if (!StringEqual(ref->scope, "") && !StringEqual(ref->lval, "") && ++ (strchr(ref->lval, '.') == NULL)) ++ { ++ Log(LOG_LEVEL_VERBOSE, "The CMDB key '%s' names the '%s' scope," ++ " the variable is addressed as '%s'", ++ key, ref->scope, address); ++ } ++ else ++ { ++ Log(LOG_LEVEL_WARNING, "The '.' in the CMDB key '%s' was taken as a" ++ " scope separator: the variable is named '%s' in the '%s' scope," ++ " addressed as '%s'", key, ref->lval, ref->scope, address); ++ } ++ free(address); ++} ++ ++/** ++ * Reject a single CMDB entry containing unresolved variable references, ++ * naming the entry that was rejected. ++ * ++ * Checked per entry, rather than once for the whole section, so that the ++ * log names every offending entry, not only the first one in the section ++ * (any one of them makes LoadCMDBData() install nothing from the file). ++ * ++ * @param entry the entry's value, as found under key in its section ++ * @param key the entry's key, as it appears in the CMDB file + * @param section the section's key, as it appears in the CMDB file + * @param file_path the CMDB file the data came from +- * @return whether the data is free of variable references ++ * @return whether the entry is free of variable references + */ +-static bool CheckCMDBDataForUnexpandedVars(JsonElement *data, +- const char *section, +- const char *file_path) ++static bool CheckCMDBEntryForUnexpandedVars(JsonElement *entry, ++ const char *key, ++ const char *section, ++ const char *file_path) + { +- assert(data != NULL); +- assert(JsonGetType(data) == JSON_TYPE_OBJECT); ++ assert(entry != NULL); ++ assert(key != NULL); + assert(section != NULL); + assert(file_path != NULL); + +- JsonIterator iter = JsonIteratorInit(data); +- while (JsonIteratorHasMore(&iter)) ++ if (StringContainsUnresolved(key)) + { +- JsonElement *child = JsonIteratorNextValue(&iter); +- const char *entry = JsonGetPropertyAsString(child); +- +- if (StringContainsUnresolved(entry)) +- { +- Log(LOG_LEVEL_ERR, +- "Invalid '%s' CMDB data in '%s', cannot contain variable" +- " references (offending key: '%s')", +- section, file_path, entry); +- return false; +- } ++ Log(LOG_LEVEL_ERR, ++ "Invalid '%s' CMDB data in '%s', cannot contain variable" ++ " references (offending key: '%s')", ++ section, file_path, key); ++ return false; ++ } + +- CMDBUnexpandedVar offender = {NULL, NULL}; +- if (JsonWalk(child, CheckObjectForUnexpandedVars, +- NULL, CheckPrimitiveForUnexpandedVars, &offender)) +- { +- continue; +- } ++ CMDBUnexpandedVar offender = {NULL, NULL}; ++ if (JsonWalk(entry, CheckObjectForUnexpandedVars, ++ NULL, CheckPrimitiveForUnexpandedVars, &offender)) ++ { ++ return true; ++ } + +- /* The walk stops only when one of the two visitors above stops it, +- * and each records what it found before doing so. */ +- assert((offender.value != NULL) || (offender.key != NULL)); ++ /* The walk stops only when one of the two visitors above stops it, and ++ * each records what it found before doing so. */ ++ assert((offender.value != NULL) || (offender.key != NULL)); + +- if (offender.value != NULL) +- { +- Log(LOG_LEVEL_ERR, +- "Invalid '%s' CMDB data in '%s', cannot contain variable" +- " references (offending value in entry '%s': '%s')", +- section, file_path, entry, offender.value); +- } +- else +- { +- Log(LOG_LEVEL_ERR, +- "Invalid '%s' CMDB data in '%s', cannot contain variable" +- " references (offending key in entry '%s': '%s')", +- section, file_path, entry, offender.key); +- } +- return false; ++ if (offender.value != NULL) ++ { ++ Log(LOG_LEVEL_ERR, ++ "Invalid '%s' CMDB data in '%s', cannot contain variable" ++ " references (offending value in entry '%s': '%s')", ++ section, file_path, key, offender.value); + } +- +- return true; ++ else ++ { ++ Log(LOG_LEVEL_ERR, ++ "Invalid '%s' CMDB data in '%s', cannot contain variable" ++ " references (offending key in entry '%s': '%s')", ++ section, file_path, key, offender.key); ++ } ++ return false; + } + ++/** ++ * Turn a CMDB key into the variable it installs, or NULL (logged) if the key ++ * is not a variable this loader can install. ++ * ++ * The key's syntax is checked here, before VarRefParse(): that function ++ * answers a malformed key with a log message and a usable reference to ++ * something else ('tendcf_interlock[' becomes 'tendcf_interlock'), which ++ * would let an entry silently replace another one. A CMDB key is a variable ++ * name, so it holds no index brackets at all. ++ * ++ * Special scopes are refused too, except 'def': 'this', 'body' and 'edit' name ++ * tables that exist only while a promise, body or bundle is being evaluated, ++ * and 'sys', 'const', 'mon' and 'match' take no namespace, but every CMDB ++ * variable is given one. ++ */ + static VarRef *GetCMDBVariableRef(const char *key) + { ++ if (strpbrk(key, "[]") != NULL) ++ { ++ Log(LOG_LEVEL_ERR, "Invalid variable specification in CMDB data: '%s'" ++ " (a variable name cannot contain '[' or ']')", key); ++ return NULL; ++ } ++ + VarRef *ref = VarRefParse(key); + if (ref->ns == NULL) + { +@@ -207,9 +292,88 @@ static VarRef *GetCMDBVariableRef(const char *key) + { + ref->scope = xstrdup("variables"); + } ++ else ++ { ++ const SpecialScope special = SpecialScopeFromString(ref->scope); ++ if (special != SPECIAL_SCOPE_NONE && special != SPECIAL_SCOPE_DEF) ++ { ++ Log(LOG_LEVEL_ERR, "Invalid variable specification in CMDB data: '%s'" ++ " (the '%s' scope cannot be set from CMDB data)", key, ref->scope); ++ VarRefDestroy(ref); ++ return NULL; ++ } ++ ++ /* The key contained a dot, so the variable did not land in the ++ * default scope. Tell the operator where it went instead. */ ++ LogCMDBVariableScope(key, ref); ++ } + return ref; + } + ++/** Does the given JSON array have a JSON null among its direct children? */ ++static bool JsonArrayContainsNull(const JsonElement *array) ++{ ++ assert(JsonGetType(array) == JSON_TYPE_ARRAY); ++ ++ const size_t length = JsonLength(array); ++ for (size_t i = 0; i < length; i++) ++ { ++ if (JsonGetType(JsonArrayGet(array, i)) == JSON_TYPE_NULL) ++ { ++ return true; ++ } ++ } ++ return false; ++} ++ ++/** ++ * Why a variable cannot be installed, as far as that can be known before ++ * touching the EvalContext: everything AddCMDBVariable() or ++ * EvalContextVariablePutTagsSetWithComment() would reject. ++ * ++ * @return whether the entry's value can be installed (details are logged ++ * otherwise) ++ */ ++static bool CheckCMDBVariableValue(const char *key, const VarRef *ref, ++ const JsonElement *data) ++{ ++ assert(key != NULL); ++ assert(ref != NULL); ++ assert(data != NULL); ++ ++ if (strlen(ref->lval) > CF_MAXVARSIZE) ++ { ++ Log(LOG_LEVEL_ERR, "Invalid variable specification in CMDB data: '%s'" ++ " (the variable name exceeds the maximum length allowed, %d characters)", ++ key, CF_MAXVARSIZE); ++ return false; ++ } ++ ++ if (JsonGetElementType(data) == JSON_ELEMENT_TYPE_PRIMITIVE) ++ { ++ if (JsonGetPrimitiveType(data) == JSON_PRIMITIVE_TYPE_NULL) ++ { ++ /* JsonPrimitiveToString() returns NULL for JSON null and there is ++ * no scalar to install in its place. */ ++ Log(LOG_LEVEL_ERR, "Invalid value in '%s' variable specification in CMDB data" ++ " (null is not a supported value)", key); ++ return false; ++ } ++ } ++ else if ((JsonGetType(data) == JSON_TYPE_ARRAY) && ++ JsonArrayContainsOnlyPrimitives((JsonElement *) data) && ++ JsonArrayContainsNull(data)) ++ { ++ /* RlistFromContainer() skips JSON nulls, so the resulting slist ++ * would silently be shorter than the data it came from. */ ++ Log(LOG_LEVEL_ERR, "Invalid value in '%s' variable specification in CMDB data" ++ " (a list must not contain null)", key); ++ return false; ++ } ++ return true; ++} ++ ++/** Install a variable that CheckCMDBVariableValue() accepted. */ + static bool AddCMDBVariable(EvalContext *ctx, const char *key, const VarRef *ref, + JsonElement *data, StringSet *tags, const char *comment) + { +@@ -219,12 +383,23 @@ static bool AddCMDBVariable(EvalContext *ctx, const char *key, const VarRef *ref + assert(data != NULL); + assert(tags != NULL); + ++ if (CMDBInsertionForcedToFail(key)) ++ { ++ Log(LOG_LEVEL_ERR, "Variable '%s' cannot be added (insertion failure forced by a test)", key); ++ StringSetDestroy(tags); ++ return false; ++ } ++ + bool ret; ++ /* Not '%s:%s.%s' of ref->ns, ref->scope and key: a key that names a scope ++ * already contains that scope, which would then be printed twice. */ ++ char *address = VarRefToString(ref, true); + if (JsonGetElementType(data) == JSON_ELEMENT_TYPE_PRIMITIVE) + { ++ assert(JsonGetPrimitiveType(data) != JSON_PRIMITIVE_TYPE_NULL); + char *value = JsonPrimitiveToString(data); +- Log(LOG_LEVEL_VERBOSE, "Installing CMDB variable '%s:%s.%s=%s'", +- ref->ns, ref->scope, key, value); ++ Log(LOG_LEVEL_VERBOSE, "Installing CMDB variable '%s=%s'", ++ address, value); + ret = EvalContextVariablePutTagsSetWithComment(ctx, ref, value, CF_DATA_TYPE_STRING, + tags, comment); + free(value); +@@ -232,9 +407,9 @@ static bool AddCMDBVariable(EvalContext *ctx, const char *key, const VarRef *ref + else if ((JsonGetType(data) == JSON_TYPE_ARRAY) && + JsonArrayContainsOnlyPrimitives(data)) + { ++ assert(!JsonArrayContainsNull(data)); + // map to slist if the data only has primitives +- Log(LOG_LEVEL_VERBOSE, "Installing CMDB slist variable '%s:%s.%s'", +- ref->ns, ref->scope, key); ++ Log(LOG_LEVEL_VERBOSE, "Installing CMDB slist variable '%s'", address); + Rlist *data_rlist = RlistFromContainer(data); + ret = EvalContextVariablePutTagsSetWithComment(ctx, ref, + data_rlist, CF_DATA_TYPE_STRING_LIST, +@@ -244,12 +419,13 @@ static bool AddCMDBVariable(EvalContext *ctx, const char *key, const VarRef *ref + else + { + // install as a data container +- Log(LOG_LEVEL_VERBOSE, "Installing CMDB data container variable '%s:%s.%s'", +- ref->ns, ref->scope, key); ++ Log(LOG_LEVEL_VERBOSE, "Installing CMDB data container variable '%s'", ++ address); + ret = EvalContextVariablePutTagsSetWithComment(ctx, ref, + data, CF_DATA_TYPE_CONTAINER, + tags, comment); + } ++ free(address); + if (!ret) + { + /* On success, EvalContextVariablePutTagsSet() consumes the tags set, +@@ -259,8 +435,147 @@ static bool AddCMDBVariable(EvalContext *ctx, const char *key, const VarRef *ref + return ret; + } + +-static bool ReadCMDBVars(EvalContext *ctx, JsonElement *vars, +- const char *file_path) ++/* ++ * The CMDB file is loaded in two steps. First every entry of every section is ++ * validated and prepared, without touching the EvalContext; only if all of ++ * them are fine is anything installed. A host-specific file is one document ++ * written by one author, so a single bad entry means the file as a whole ++ * cannot be trusted, and installing only the entries around it could leave ++ * policy running without data it expects to guard it (see LoadCMDBData()). ++ */ ++typedef struct ++{ ++ const char *key; /* borrowed from the JSON document */ ++ VarRef *ref; /* owned */ ++ JsonElement *data; /* borrowed from the JSON document */ ++ StringSet *tags; /* owned until installed */ ++ const char *comment; /* borrowed from the JSON document, or NULL */ ++} CMDBPendingVariable; ++ ++typedef struct ++{ ++ const char *key; /* borrowed from the JSON document */ ++ StringSet *tags; /* owned until installed */ ++ const char *comment; /* borrowed from the JSON document, or NULL */ ++ bool installed; /* set by InstallCMDBPending() */ ++} CMDBPendingClass; ++ ++typedef struct ++{ ++ Seq *variables; /* CMDBPendingVariable *, in install order */ ++ Seq *classes; /* CMDBPendingClass *, in install order */ ++} CMDBPending; ++ ++static void CMDBPendingVariableDestroy(void *item) ++{ ++ CMDBPendingVariable *var = item; ++ VarRefDestroy(var->ref); ++ StringSetDestroy(var->tags); ++ free(var); ++} ++ ++static void CMDBPendingClassDestroy(void *item) ++{ ++ CMDBPendingClass *cls = item; ++ StringSetDestroy(cls->tags); ++ free(cls); ++} ++ ++static CMDBPending CMDBPendingNew(void) ++{ ++ return (CMDBPending) { ++ .variables = SeqNew(16, CMDBPendingVariableDestroy), ++ .classes = SeqNew(16, CMDBPendingClassDestroy), ++ }; ++} ++ ++static void CMDBPendingDestroy(CMDBPending *pending) ++{ ++ SeqDestroy(pending->variables); ++ SeqDestroy(pending->classes); ++} ++ ++/** ++ * Refuse two entries that name the same variable (also across sections and ++ * spellings: 'x', 'data:variables.x' and 'variables.x' are one variable): the ++ * later would replace the earlier one, so an entry could silently take the ++ * place of the one a policy relies on. ++ * ++ * @return false (logged) if any variable is named more than once ++ */ ++static bool CMDBPendingCheckNoDuplicates(const CMDBPending *pending) ++{ ++ bool valid = true; ++ StringSet *seen = StringSetNew(); ++ for (size_t i = 0; i < SeqLength(pending->variables); i++) ++ { ++ const CMDBPendingVariable *var = SeqAt(pending->variables, i); ++ char *address = VarRefToString(var->ref, true); ++ if (StringSetContains(seen, address)) ++ { ++ Log(LOG_LEVEL_ERR, ++ "Invalid variable specification in CMDB data: '%s' names the variable '%s'" ++ " that another entry names too", var->key, address); ++ valid = false; ++ free(address); ++ } ++ else ++ { ++ StringSetAdd(seen, address); /* takes ownership */ ++ } ++ } ++ StringSetDestroy(seen); ++ return valid; ++} ++ ++/** ++ * Is rules an array that is empty or holds just the string 'allowed'? The ++ * element's type is checked before its value is read. ++ */ ++static bool CheckCMDBRuleArray(const JsonElement *rules, const char *allowed) ++{ ++ if (JsonGetType(rules) != JSON_TYPE_ARRAY || JsonLength(rules) > 1) ++ { ++ return false; ++ } ++ if (JsonLength(rules) == 0) ++ { ++ return true; ++ } ++ const JsonElement *rule = JsonArrayGet(rules, 0); ++ return (JsonGetType(rule) == JSON_TYPE_STRING && ++ StringEqual(JsonPrimitiveGetAsString(rule), allowed)); ++} ++ ++/** Takes ownership of ref and tags. */ ++static void CMDBPendingAddVariable(CMDBPending *pending, const char *key, ++ VarRef *ref, JsonElement *data, ++ StringSet *tags, const char *comment) ++{ ++ CMDBPendingVariable *var = xmalloc(sizeof(*var)); ++ *var = (CMDBPendingVariable) { ++ .key = key, .ref = ref, .data = data, .tags = tags, .comment = comment, ++ }; ++ SeqAppend(pending->variables, var); ++} ++ ++/** Takes ownership of tags. */ ++static void CMDBPendingAddClass(CMDBPending *pending, const char *key, ++ StringSet *tags, const char *comment) ++{ ++ CMDBPendingClass *cls = xmalloc(sizeof(*cls)); ++ *cls = (CMDBPendingClass) { .key = key, .tags = tags, .comment = comment }; ++ SeqAppend(pending->classes, cls); ++} ++ ++/** ++ * Validate the 'vars' section. ++ * ++ * @return false if the section or any entry in it is invalid. Every entry is ++ * looked at regardless, so that all problems are logged in one go. ++ */ ++static bool PrepareCMDBVars(JsonElement *vars, const char *file_path, ++ CMDBPending *pending) + { + assert(vars != NULL); + +@@ -272,40 +587,49 @@ static bool ReadCMDBVars(EvalContext *ctx, JsonElement *vars, + return false; + } + +- if (!CheckCMDBDataForUnexpandedVars(vars, "vars", file_path)) +- { +- return false; +- } +- ++ bool all_valid = true; + JsonIterator iter = JsonIteratorInit(vars); + while (JsonIteratorHasMore(&iter)) + { + const char *key = JsonIteratorNextKey(&iter); + JsonElement *data = JsonObjectGet(vars, key); + ++ if (!CheckCMDBEntryForUnexpandedVars(data, key, "vars", file_path)) ++ { ++ all_valid = false; ++ continue; ++ } ++ + VarRef *ref = GetCMDBVariableRef(key); + if (ref == NULL) + { ++ all_valid = false; + continue; + } + +- StringSet *tags = StringSetNew(); +- StringSetAdd(tags, xstrdup(CMDB_SOURCE_TAG)); +- bool ret = AddCMDBVariable(ctx, key, ref, data, tags, NULL); +- VarRefDestroy(ref); +- if (!ret) ++ if (!CheckCMDBVariableValue(key, ref, data)) + { +- /* Details should have been logged already. */ +- Log(LOG_LEVEL_ERR, "Failed to add CMDB variable '%s'", key); ++ VarRefDestroy(ref); ++ all_valid = false; ++ continue; + } ++ ++ StringSet *tags = StringSetNew(); ++ StringSetAdd(tags, xstrdup(CMDB_SOURCE_TAG)); ++ CMDBPendingAddVariable(pending, key, ref, data, tags, NULL); + } +- return true; ++ return all_valid; + } + ++/** ++ * Read the tags of an entry; *valid is cleared if they are malformed. The ++ * returned set is never NULL and always has default_tag in it. ++ */ + static StringSet *GetTagsFromJsonTags(const char *item_type, + const char *key, + const JsonElement *json_tags, +- const char *default_tag) ++ const char *default_tag, ++ bool *valid) + { + StringSet *tags = NULL; + if (JSON_NOT_NULL(json_tags)) +@@ -317,6 +641,7 @@ static StringSet *GetTagsFromJsonTags(const char *item_type, + "Invalid json_tags information for %s '%s' in CMDB data:" + " must be a JSON array of strings", + item_type, key); ++ *valid = false; + } + else + { +@@ -327,6 +652,7 @@ static StringSet *GetTagsFromJsonTags(const char *item_type, + "Invalid json_tags information %s '%s' in CMDB data:" + " must be a JSON array of strings", + item_type, key); ++ *valid = false; + } + } + } +@@ -339,8 +665,10 @@ static StringSet *GetTagsFromJsonTags(const char *item_type, + return tags; + } + ++/** Read the comment of an entry; *valid is cleared if it is not a string. */ + static inline const char *GetCMDBComment(const char *item_type, const char *identifier, +- const JsonElement *json_object) ++ const JsonElement *json_object, ++ bool *valid) + { + assert(JsonGetType(json_object) == JSON_TYPE_OBJECT); + +@@ -355,15 +683,19 @@ static inline const char *GetCMDBComment(const char *item_type, const char *iden + Log(LOG_LEVEL_ERR, + "Invalid type of the 'comment' field for the '%s' %s in CMDB data, must be a string", + identifier, item_type); ++ *valid = false; + return NULL; + } + + return JsonPrimitiveGetAsString(json_comment); + } + +-/** Uses the new format allowing metadata (CFE-3633) */ +-static bool ReadCMDBVariables(EvalContext *ctx, JsonElement *variables, +- const char *file_path) ++/** ++ * Validate the 'variables' section, which uses the new format allowing ++ * metadata (CFE-3633). Same contract as PrepareCMDBVars(). ++ */ ++static bool PrepareCMDBVariables(JsonElement *variables, const char *file_path, ++ CMDBPending *pending) + { + assert(variables != NULL); + +@@ -375,27 +707,30 @@ static bool ReadCMDBVariables(EvalContext *ctx, JsonElement *variables, + return false; + } + +- if (!CheckCMDBDataForUnexpandedVars(variables, "variables", file_path)) +- { +- return false; +- } +- ++ bool all_valid = true; + JsonIterator iter = JsonIteratorInit(variables); + while (JsonIteratorHasMore(&iter)) + { + const char *key = JsonIteratorNextKey(&iter); ++ JsonElement *const var_info = JsonObjectGet(variables, key); ++ ++ if (!CheckCMDBEntryForUnexpandedVars(var_info, key, "variables", file_path)) ++ { ++ all_valid = false; ++ continue; ++ } + + VarRef *ref = GetCMDBVariableRef(key); + if (ref == NULL) + { ++ all_valid = false; + continue; + } + +- JsonElement *const var_info = JsonObjectGet(variables, key); +- + JsonElement *data; + StringSet *tags; + const char *comment = NULL; ++ bool entry_valid = true; + + if (JsonGetType(var_info) == JSON_TYPE_OBJECT) + { +@@ -405,30 +740,75 @@ static bool ReadCMDBVariables(EvalContext *ctx, JsonElement *variables, + { + Log(LOG_LEVEL_ERR, "Missing value in '%s' variable specification in CMDB data (value field is required)", key); + VarRefDestroy(ref); ++ all_valid = false; + continue; + } + + JsonElement *json_tags = JsonObjectGet(var_info, CMDB_VARIABLES_TAGS); +- tags = GetTagsFromJsonTags("variable", key, json_tags, CMDB_SOURCE_TAG); +- comment = GetCMDBComment("variable", key, var_info); ++ tags = GetTagsFromJsonTags("variable", key, json_tags, CMDB_SOURCE_TAG, &entry_valid); ++ comment = GetCMDBComment("variable", key, var_info, &entry_valid); + } + else + { + // Just a bare value, like in "vars", no metadata + data = var_info; +- tags = GetTagsFromJsonTags("variable", key, NULL, CMDB_SOURCE_TAG); ++ tags = GetTagsFromJsonTags("variable", key, NULL, CMDB_SOURCE_TAG, &entry_valid); + } + + assert(tags != NULL); + assert(data != NULL); + +- bool ret = AddCMDBVariable(ctx, key, ref, data, tags, comment); +- VarRefDestroy(ref); +- if (!ret) ++ if (!entry_valid || !CheckCMDBVariableValue(key, ref, data)) + { +- /* Details should have been logged already. */ +- Log(LOG_LEVEL_ERR, "Failed to add CMDB variable '%s'", key); ++ StringSetDestroy(tags); ++ VarRefDestroy(ref); ++ all_valid = false; ++ continue; + } ++ ++ CMDBPendingAddVariable(pending, key, ref, data, tags, comment); ++ } ++ return all_valid; ++} ++ ++/** Split a class key the way AddCMDBClass() does: "ns:name" or "name". */ ++static const char *CMDBClassNameOfKey(const char *key) ++{ ++ const char *sep = strchr(key, ':'); ++ return (sep == NULL) ? key : (sep + 1); ++} ++ ++/** ++ * Does EvalContextClassPutTagsSet() have a name it can work with? ++ * ++ * That function chops trailing whitespace off a copy of the name and refuses ++ * the class if nothing is left (in the default namespace), yet files the class ++ * under the name as given, whitespace canonified to '_'. A name with ++ * whitespace at either end is thus either refused there or filed under a name ++ * its author did not write, so it is refused here. ++ */ ++static bool CheckCMDBClassName(const char *key) ++{ ++ const char *name = CMDBClassNameOfKey(key); ++ if (name[0] == '\0') ++ { ++ Log(LOG_LEVEL_ERR, "Invalid CMDB class '%s': the class name is empty", key); ++ return false; ++ } ++ if (isspace((unsigned char) name[0]) || ++ isspace((unsigned char) name[strlen(name) - 1])) ++ { ++ Log(LOG_LEVEL_ERR, ++ "Invalid CMDB class '%s': the class name starts or ends with whitespace", ++ key); ++ return false; ++ } ++ if (strlen(name) >= CF_MAXVARSIZE) ++ { ++ Log(LOG_LEVEL_ERR, ++ "Invalid CMDB class '%s': the class name is equal or longer than %d characters", ++ key, CF_MAXVARSIZE); ++ return false; + } + return true; + } +@@ -439,6 +819,13 @@ static bool AddCMDBClass(EvalContext *ctx, const char *key, StringSet *tags, con + assert(key != NULL); + assert(tags != NULL); + ++ if (CMDBInsertionForcedToFail(key)) ++ { ++ Log(LOG_LEVEL_ERR, "Class '%s' cannot be added (insertion failure forced by a test)", key); ++ StringSetDestroy(tags); ++ return false; ++ } ++ + bool ret; + Log(LOG_LEVEL_VERBOSE, "Installing CMDB class '%s'", key); + +@@ -467,8 +854,11 @@ static bool AddCMDBClass(EvalContext *ctx, const char *key, StringSet *tags, con + return ret; + } + +-static bool ReadCMDBClasses(EvalContext *ctx, JsonElement *classes, +- const char *file_path) ++/** ++ * Validate the 'classes' section. Same contract as PrepareCMDBVars(). ++ */ ++static bool PrepareCMDBClasses(JsonElement *classes, const char *file_path, ++ CMDBPending *pending) + { + assert(classes != NULL); + +@@ -480,16 +870,23 @@ static bool ReadCMDBClasses(EvalContext *ctx, JsonElement *classes, + return false; + } + +- if (!CheckCMDBDataForUnexpandedVars(classes, "classes", file_path)) +- { +- return false; +- } +- ++ bool all_valid = true; + JsonIterator iter = JsonIteratorInit(classes); + while (JsonIteratorHasMore(&iter)) + { + const char *key = JsonIteratorNextKey(&iter); + JsonElement *data = JsonObjectGet(classes, key); ++ ++ if (!CheckCMDBEntryForUnexpandedVars(data, key, "classes", file_path) || ++ !CheckCMDBClassName(key)) ++ { ++ all_valid = false; ++ continue; ++ } ++ ++ StringSet *tags = NULL; ++ const char *comment = NULL; ++ + if (JsonGetElementType(data) == JSON_ELEMENT_TYPE_PRIMITIVE) + { + const char *expr = JsonPrimitiveGetAsString(data); +@@ -497,38 +894,33 @@ static bool ReadCMDBClasses(EvalContext *ctx, JsonElement *classes, + { + Log(LOG_LEVEL_ERR, + "Invalid class specification '%s' in CMDB data, only \"any::\" allowed", expr); ++ all_valid = false; + continue; + } + +- StringSet *default_tags = StringSetNew(); +- StringSetAdd(default_tags, xstrdup(CMDB_SOURCE_TAG)); +- bool ret = AddCMDBClass(ctx, key, default_tags, NULL); +- if (!ret) +- { +- /* It does not have to be and error, it could be a result of +- * the class already existing. Anyways, details about +- * potential errors should have been logged already. */ +- Log(LOG_LEVEL_DEBUG, "Did not add CMDB class '%s'", key); +- } ++ tags = StringSetNew(); ++ StringSetAdd(tags, xstrdup(CMDB_SOURCE_TAG)); + } + else if (JsonGetContainerType(data) == JSON_CONTAINER_TYPE_ARRAY && + JsonArrayContainsOnlyPrimitives(data)) + { ++ bool array_valid = true; + switch (JsonLength(data)) + { + case 0: + Log(LOG_LEVEL_ERR, + "Empty class specification '[]' in CMDB data not allowed, only '[\"any::\"]'"); +- continue;; ++ array_valid = false; ++ break; + case 1: + if (!StringEqual(JsonPrimitiveGetAsString(JsonArrayGet(data, 0)), "any::")) + { + Log(LOG_LEVEL_ERR, + "Invalid class specification '[\"%s\"]' in CMDB data, only '[\"any::\"]' allowed", + JsonPrimitiveGetAsString(JsonArrayGet(data, 0))); +- continue; ++ array_valid = false; + } +- // All good :) ++ // else all good :) + break; + default: + { +@@ -539,18 +931,17 @@ static bool ReadCMDBClasses(EvalContext *ctx, JsonElement *classes, + StringWriterData(str)); + WriterClose(str); + } +- continue; ++ array_valid = false; ++ break; + } +- StringSet *default_tags = StringSetNew(); +- StringSetAdd(default_tags, xstrdup(CMDB_SOURCE_TAG)); +- bool ret = AddCMDBClass(ctx, key, default_tags, NULL); +- if (!ret) ++ if (!array_valid) + { +- /* It does not have to be and error, it could be a result of +- * the class already existing. Anyways, details about +- * potential errors should have been logged already. */ +- Log(LOG_LEVEL_DEBUG, "Did not add CMDB class '%s'", key); ++ all_valid = false; ++ continue; + } ++ ++ tags = StringSetNew(); ++ StringSetAdd(tags, xstrdup(CMDB_SOURCE_TAG)); + } + else if (JsonGetContainerType(data) == JSON_CONTAINER_TYPE_OBJECT) + { +@@ -559,47 +950,221 @@ static bool ReadCMDBClasses(EvalContext *ctx, JsonElement *classes, + const JsonElement *json_tags = JsonObjectGet(data, CMDB_CLASSES_TAGS); + + if (JSON_NOT_NULL(class_exprs) && +- (JsonGetType(class_exprs) != JSON_TYPE_ARRAY || +- JsonLength(class_exprs) > 1 || +- (JsonLength(class_exprs) == 1 && +- !StringEqual(JsonPrimitiveGetAsString(JsonArrayGet(class_exprs, 0)), "any::")))) ++ !CheckCMDBRuleArray(class_exprs, "any::")) + { + Log(LOG_LEVEL_ERR, + "Invalid class expression rules for class '%s' in CMDB data," + " only '[]' or '[\"any::\"]' allowed", key); ++ all_valid = false; + continue; + } + if (JSON_NOT_NULL(reg_exprs) && +- (JsonGetType(reg_exprs) != JSON_TYPE_ARRAY || +- JsonLength(reg_exprs) > 1 || +- (JsonLength(reg_exprs) == 1 && +- !StringEqual(JsonPrimitiveGetAsString(JsonArrayGet(reg_exprs, 0)), "any")))) ++ !CheckCMDBRuleArray(reg_exprs, "any")) + { + Log(LOG_LEVEL_ERR, + "Invalid regular expression rules for class '%s' in CMDB data," + " only '[]' or '[\"any\"]' allowed", key); ++ all_valid = false; + continue; + } + +- StringSet *tags = GetTagsFromJsonTags("class", key, json_tags, CMDB_SOURCE_TAG); +- const char *comment = GetCMDBComment("class", key, data); +- bool ret = AddCMDBClass(ctx, key, tags, comment); +- if (!ret) ++ bool entry_valid = true; ++ tags = GetTagsFromJsonTags("class", key, json_tags, CMDB_SOURCE_TAG, &entry_valid); ++ comment = GetCMDBComment("class", key, data, &entry_valid); ++ if (!entry_valid) + { +- /* It does not have to be and error, it could be a result of +- * the class already existing. Anyways, details about +- * potential errors should have been logged already. */ +- Log(LOG_LEVEL_DEBUG, "Did not add CMDB class '%s'", key); ++ StringSetDestroy(tags); ++ all_valid = false; ++ continue; + } + } + else + { + Log(LOG_LEVEL_ERR, "Invalid CMDB class data for class '%s'", key); ++ all_valid = false; ++ continue; ++ } ++ ++ CMDBPendingAddClass(pending, key, tags, comment); ++ } ++ return all_valid; ++} ++ ++/** ++ * The namespace and name EvalContextClassPutTagsSet() files a CMDB class ++ * under: the key split the way AddCMDBClass() splits it, the name canonified ++ * the way ClassInit() does. Both results are owned by the caller. ++ */ ++static void CMDBClassIdentity(const char *key, char **ns, char **name) ++{ ++ const char *sep = strchr(key, ':'); ++ *ns = (sep != NULL) ? xstrndup(key, sep - key) : xstrdup(CMDB_NAMESPACE); ++ /* Not CanonifyName(): it returns a static buffer */ ++ *name = xstrdup(CMDBClassNameOfKey(key)); ++ CanonifyNameInPlace(*name); ++} ++ ++/** ++ * Is the class there already? EvalContextClassPutTagsSet() decides that by the ++ * name as given, so 'pre-class' is not found where 'pre_class' is, and the ++ * existing class would be replaced; ask by the canonical identity instead. ++ */ ++static bool CMDBClassExists(const EvalContext *ctx, const char *key) ++{ ++ char *ns; ++ char *name; ++ CMDBClassIdentity(key, &ns, &name); ++ const Class *existing = EvalContextClassGet(ctx, ns, name); ++ free(ns); ++ free(name); ++ return (existing != NULL && existing->scope == CONTEXT_SCOPE_NAMESPACE); ++} ++ ++/** ++ * Refuse a variable that exists already: installing it would replace the ++ * existing one, and RollbackCMDBPending() could not put that one back. ++ * Nothing is there to be replaced in practice, as the loader runs on a ++ * cleared context, before augments. ++ * ++ * @return false (logged) if any variable exists already ++ */ ++static bool CMDBPendingCheckNoExistingVariables(const EvalContext *ctx, ++ const CMDBPending *pending) ++{ ++ bool valid = true; ++ for (size_t i = 0; i < SeqLength(pending->variables); i++) ++ { ++ const CMDBPendingVariable *var = SeqAt(pending->variables, i); ++ /* The value of an empty list is NULL, so go by the type */ ++ DataType type = CF_DATA_TYPE_NONE; ++ EvalContextVariableGet(ctx, var->ref, &type); ++ if (type != CF_DATA_TYPE_NONE) ++ { ++ char *address = VarRefToString(var->ref, true); ++ Log(LOG_LEVEL_ERR, ++ "Invalid variable specification in CMDB data: '%s' names the" ++ " variable '%s' that exists already", var->key, address); ++ free(address); ++ valid = false; ++ } ++ } ++ return valid; ++} ++ ++/** ++ * Take back what InstallCMDBPending() installed so far: the first ++ * variables_installed variables and every class marked installed. A class ++ * that was already there was not installed by this load and is left alone. ++ * ++ * Removing is restoring only because nothing it removes was there before: ++ * LoadCMDBData() refuses a variable that exists already, and ++ * InstallCMDBPending() does not touch a class that does (by its canonical ++ * identity, so an alias of one is left alone too). ++ */ ++static void RollbackCMDBPending(EvalContext *ctx, CMDBPending *pending, ++ size_t variables_installed) ++{ ++ for (size_t i = 0; i < variables_installed; i++) ++ { ++ const CMDBPendingVariable *var = SeqAt(pending->variables, i); ++ EvalContextVariableRemove(ctx, var->ref); ++ } ++ ++ for (size_t i = 0; i < SeqLength(pending->classes); i++) ++ { ++ const CMDBPendingClass *cls = SeqAt(pending->classes, i); ++ if (!cls->installed) ++ { ++ continue; ++ } ++ ++ char *ns; ++ char *name; ++ CMDBClassIdentity(cls->key, &ns, &name); ++ EvalContextClassRemove(ctx, ns, name); ++ free(ns); ++ free(name); ++ } ++} ++ ++/** ++ * Install what PrepareCMDB*() accepted. Everything that can make an ++ * installation fail has been checked by then, except for one case: a class ++ * that already exists is not an error and is left as it is. ++ * ++ * All or nothing in its own right, should the validation have a gap: at the ++ * first class or variable that cannot be installed, everything this call ++ * installed is removed again and false is returned. ++ * ++ * @return false if a class or variable could not be installed after all ++ * (details are logged); nothing is left installed then ++ */ ++static bool InstallCMDBPending(EvalContext *ctx, CMDBPending *pending) ++{ ++ /* Classes first: installing them before the variables lets a failing ++ * variable exercise the rollback of both. */ ++ for (size_t i = 0; i < SeqLength(pending->classes); i++) ++ { ++ CMDBPendingClass *cls = SeqAt(pending->classes, i); ++ if (CMDBClassExists(ctx, cls->key)) ++ { ++ /* Not an error, and not ours to replace or to take back */ ++ Log(LOG_LEVEL_VERBOSE, ++ "Not adding CMDB class '%s', it exists already", cls->key); ++ continue; ++ } ++ ++ cls->installed = AddCMDBClass(ctx, cls->key, cls->tags, cls->comment); ++ cls->tags = NULL; /* consumed or destroyed by AddCMDBClass() */ ++ if (!cls->installed) ++ { ++ /* Details should have been logged already. */ ++ Log(LOG_LEVEL_ERR, ++ "Failed to add CMDB class '%s', removing what was installed from the file", ++ cls->key); ++ RollbackCMDBPending(ctx, pending, 0); ++ return false; ++ } ++ } ++ ++ for (size_t i = 0; i < SeqLength(pending->variables); i++) ++ { ++ CMDBPendingVariable *var = SeqAt(pending->variables, i); ++ bool ret = AddCMDBVariable(ctx, var->key, var->ref, var->data, ++ var->tags, var->comment); ++ var->tags = NULL; /* consumed or destroyed by AddCMDBVariable() */ ++ if (!ret) ++ { ++ /* Details should have been logged already. */ ++ Log(LOG_LEVEL_ERR, ++ "Failed to add CMDB variable '%s', removing what was installed from the file", ++ var->key); ++ RollbackCMDBPending(ctx, pending, i); ++ return false; + } + } + return true; + } + ++/** ++ * Load host_specific.json into the EvalContext, all or nothing. ++ * ++ * If the file is there but anything in it is unusable (unreadable, not ++ * valid JSON, a repeated member in any object, anything but whitespace after ++ * the document, not an object, a section of the wrong type, or any single ++ * entry of 'vars', 'variables' or 'classes' that cannot be validated or ++ * installed), nothing at all is installed from it, every problem found is ++ * logged, and false is returned. A policy that guards an action by data it ++ * reads from this file (an interlock that blocks a service, say) must not ++ * see the file half-loaded: the guard's entry could be exactly the one ++ * dropped, and the action would then run unguarded. ++ * ++ * An absent file is not an error. A top-level key other than 'vars', ++ * 'variables' and 'classes' makes the file invalid, as does a section whose ++ * value is not a JSON object (JSON null included), and two entries that ++ * name the same variable (one would silently replace the other), or a ++ * variable that exists already. A class that exists already is left as it is. ++ */ + bool LoadCMDBData(EvalContext *ctx) + { + char file_path[PATH_MAX] = {0}; +@@ -617,10 +1182,13 @@ bool LoadCMDBData(EvalContext *ctx) + return false; + } + +- JsonElement *data = ReadJsonFile(file_path, LOG_LEVEL_ERR, HOST_SPECIFIC_DATA_MAX_SIZE); ++ /* Strict: a repeated member would have replaced the earlier one, and ++ * trailing input would have been ignored, before any validation saw it. */ ++ JsonElement *data = ReadJsonFileMode(file_path, LOG_LEVEL_ERR, ++ HOST_SPECIFIC_DATA_MAX_SIZE, true); + if (data == NULL) + { +- /* Details are logged by ReadJsonFile() */ ++ /* Details are logged by ReadJsonFileMode() */ + return false; + } + if (JsonGetType(data) != JSON_TYPE_OBJECT) +@@ -630,39 +1198,73 @@ bool LoadCMDBData(EvalContext *ctx) + return false; + } + +- Log(LOG_LEVEL_VERBOSE, "Loaded CMDB data file '%s', installing contents", file_path); ++ Log(LOG_LEVEL_VERBOSE, "Loaded CMDB data file '%s', validating contents", file_path); + ++ bool unknown_key_found = false; + JsonIterator iter = JsonIteratorInit(data); + while (JsonIteratorHasMore(&iter)) + { + const char *key = JsonIteratorNextKey(&iter); +- /* Only vars and classes allowed in CMDB data */ ++ /* Only vars, variables and classes allowed in CMDB data. A section ++ * this loader does not know is data it cannot install, which may be ++ * the very data a policy relies on (a misspelled 'variables' would ++ * drop an interlock and keep its services), so the file is refused. */ + if (!IsStrIn(key, (const char*[4]){"vars", "classes", "variables", NULL})) + { +- Log(LOG_LEVEL_WARNING, "Invalid key '%s' in the CMDB data file '%s', skipping it", ++ Log(LOG_LEVEL_ERR, "Invalid key '%s' in the CMDB data file '%s'" ++ " (only 'vars', 'variables' and 'classes' are allowed)", + key, file_path); ++ unknown_key_found = true; + } + } + +- bool success = true; ++ /* Step 1: validate everything, install nothing. Every section is ++ * looked at even after a failure, so the log names every bad entry. ++ * A section is there if its key is, whatever it holds: a JSON null is ++ * not "absent" and PrepareCMDB*() refuse it as not being an object. */ ++ CMDBPending pending = CMDBPendingNew(); ++ bool valid = unknown_key_found ? false : true; + JsonElement *vars = JsonObjectGet(data, "vars"); +- if (JSON_NOT_NULL(vars) && !ReadCMDBVars(ctx, vars, file_path)) ++ if (vars != NULL && !PrepareCMDBVars(vars, file_path, &pending)) + { +- success = false; ++ valid = false; + } + /* Uses the new format allowing metadata (CFE-3633) */ + JsonElement *variables = JsonObjectGet(data, "variables"); +- if (JSON_NOT_NULL(variables) && +- !ReadCMDBVariables(ctx, variables, file_path)) ++ if (variables != NULL && ++ !PrepareCMDBVariables(variables, file_path, &pending)) + { +- success = false; ++ valid = false; + } + JsonElement *classes = JsonObjectGet(data, "classes"); +- if (JSON_NOT_NULL(classes) && !ReadCMDBClasses(ctx, classes, file_path)) ++ if (classes != NULL && !PrepareCMDBClasses(classes, file_path, &pending)) ++ { ++ valid = false; ++ } ++ if (!CMDBPendingCheckNoDuplicates(&pending)) + { +- success = false; ++ valid = false; ++ } ++ if (!CMDBPendingCheckNoExistingVariables(ctx, &pending)) ++ { ++ valid = false; ++ } ++ ++ /* Step 2: install, but only if the whole file is good. */ ++ bool success = false; ++ if (valid) ++ { ++ Log(LOG_LEVEL_VERBOSE, "Installing contents of CMDB data file '%s'", file_path); ++ success = InstallCMDBPending(ctx, &pending); ++ } ++ else ++ { ++ Log(LOG_LEVEL_ERR, ++ "CMDB data file '%s' has invalid entries (see above), installing none of its contents", ++ file_path); + } + ++ CMDBPendingDestroy(&pending); + JsonDestroy(data); + return success; + } +diff --git a/tests/acceptance/00_basics/06_host_specific_data/07-invalid-data.cf b/tests/acceptance/00_basics/06_host_specific_data/07-invalid-data.cf +index c7ef877b7..f0b55ed48 100644 +--- a/tests/acceptance/00_basics/06_host_specific_data/07-invalid-data.cf ++++ b/tests/acceptance/00_basics/06_host_specific_data/07-invalid-data.cf +@@ -1,4 +1,6 @@ +-# basic test of the host_specific.json functionality: 'classes' object with a tag and a comment ++# test of the host_specific.json functionality: an unknown top-level key is an ++# error and the whole file is rejected, so the valid 'classes' next to it is not ++# installed either + body common control + { + inputs => { "../../default.sub.cf" }; +@@ -38,8 +40,10 @@ bundle agent check + methods: + "" + usebundle => dcs_passif_output( +- "\s*warning: Invalid key 'clses'.*\s+ns1:test_class\s+.*", +- "", ++ "\s*error: Invalid key 'clses' in the CMDB data file.*installing none of its contents.*", ++ # the unknown key makes the loader install nothing, so the valid ++ # class next to it is not defined ++ ".*\s+ns1:test_class\s+.*", + $(command), + $(this.promise_filename) + ); +diff --git a/tests/acceptance/00_basics/06_host_specific_data/08-class-expression.cf b/tests/acceptance/00_basics/06_host_specific_data/08-class-expression.cf +index afe07efbb..837d14dbf 100644 +--- a/tests/acceptance/00_basics/06_host_specific_data/08-class-expression.cf ++++ b/tests/acceptance/00_basics/06_host_specific_data/08-class-expression.cf +@@ -1,4 +1,5 @@ +-# basic test of the host_specific.json functionality: 'classes' object with a tag and a comment ++# test of the host_specific.json functionality: an invalid 'class_expressions' ++# rule is reported and the whole file is rejected (all or nothing) + body common control + { + inputs => { "../../default.sub.cf" }; +@@ -38,8 +39,10 @@ bundle agent check + methods: + "" + usebundle => dcs_passif_output( +- "\s*error: Invalid class expression rules for class 'ns1:test_class1'.*\s+ns2:test_class2\s+.*", +- "", ++ "\s*error: Invalid class expression rules for class 'ns1:test_class1'.*", ++ # the invalid entry makes the loader install nothing, so the valid ++ # class next to it is not defined either ++ ".*\s+ns2:test_class2\s+.*", + $(command), + $(this.promise_filename) + ); +diff --git a/tests/acceptance/00_basics/06_host_specific_data/09-class-reg-expression.cf b/tests/acceptance/00_basics/06_host_specific_data/09-class-reg-expression.cf +index 8d056ca50..ebf7d7bfa 100644 +--- a/tests/acceptance/00_basics/06_host_specific_data/09-class-reg-expression.cf ++++ b/tests/acceptance/00_basics/06_host_specific_data/09-class-reg-expression.cf +@@ -1,4 +1,5 @@ +-# basic test of the host_specific.json functionality: 'classes' object with a tag and a comment ++# test of the host_specific.json functionality: an invalid 'regular_expressions' ++# rule is reported and the whole file is rejected (all or nothing) + body common control + { + inputs => { "../../default.sub.cf" }; +@@ -38,8 +39,10 @@ bundle agent check + methods: + "" + usebundle => dcs_passif_output( +- "\s*error: Invalid regular expression rules for class 'ns1:test_class1'.*\s+ns2:test_class2\s+.*", +- "", ++ "\s*error: Invalid regular expression rules for class 'ns1:test_class1'.*", ++ # the invalid entry makes the loader install nothing, so the valid ++ # class next to it is not defined either ++ ".*\s+ns2:test_class2\s+.*", + $(command), + $(this.promise_filename) + ); +diff --git a/tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf b/tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf +new file mode 100644 +index 000000000..6e92059e7 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf +@@ -0,0 +1,60 @@ ++# test of the host_specific.json functionality: JSON null values are rejected ++# (used to crash), every one is reported, and since one invalid entry makes ++# the loader install nothing, the good entries are not installed either ++body common control ++{ ++ inputs => { "../../default.sub.cf" }; ++ bundlesequence => { default("$(this.promise_filename)") }; ++ version => "1.0"; ++} ++ ++####################################################### ++bundle agent init ++{ ++ files: ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)." ++ create => "true", ++ perms => mog( ++ "0700", "$(sys.user_data[username])", "$(sys.user_data[username])" ++ ); ++ ++ methods: ++ "prepare_host_specific_data" ++ usebundle => file_copy( ++ "$(this.promise_filename).json", ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)host_specific.json" ++ ), ++ handle => "prepare_host_specific_data"; ++ ++ "empty_promises_cf" ++ usebundle => file_make("$(sys.inputdir)/promises.cf", ''); ++} ++ ++####################################################### ++bundle agent check ++{ ++ vars: ++ "command" ++ string => "$(sys.cf_promises) --show-vars -f $(sys.inputdir)/promises.cf | $(G.grep) -E 'CMDB data|value_from_'"; ++ ++ # The offending entries are reported in the order they appear in the CMDB ++ # data, 'vars' first, then 'variables', and the file is rejected as a ++ # whole. ++ "expected" ++ string => concat( ++ "\s*error: Invalid value in 'null_scalar'.*null is not a supported value.*", ++ "\s+error: Invalid value in 'null_list'.*a list must not contain null.*", ++ "\s+error: Invalid value in 'null_value'.*null is not a supported value.*", ++ "\s+error: Invalid value in 'null_bare'.*null is not a supported value.*", ++ "\s+error: Invalid value in 'null_value_list'.*a list must not contain null.*", ++ "\s+error: CMDB data file '.*host_specific\.json' has invalid entries.*installing none of its contents.*" ++ ); ++ ++ methods: ++ # value_from_vars and value_from_variables are the good entries' values: ++ # none of them may be installed. ++ "" ++ usebundle => dcs_passif_output( ++ $(expected), ".*value_from_.*", $(command), $(this.promise_filename) ++ ); ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf.json b/tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf.json +new file mode 100644 +index 000000000..f8632e347 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf.json +@@ -0,0 +1,13 @@ ++{ ++ "vars": { ++ "null_scalar": null, ++ "null_list": ["a", null], ++ "good_vars": "value_from_vars" ++ }, ++ "variables": { ++ "null_value": { "value": null }, ++ "null_bare": null, ++ "null_value_list": { "value": ["a", null] }, ++ "good_variables": { "value": "value_from_variables" } ++ } ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf b/tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf +new file mode 100644 +index 000000000..f1e88bd45 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf +@@ -0,0 +1,52 @@ ++# test of the host_specific.json functionality: keys with more dots than the ++# "[namespace:]bundle.variable" form allows are warned about, plain scope ++# qualification is not ++body common control ++{ ++ inputs => { "../../default.sub.cf" }; ++ bundlesequence => { default("$(this.promise_filename)") }; ++ version => "1.0"; ++} ++ ++####################################################### ++bundle agent init ++{ ++ files: ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)." ++ create => "true", ++ perms => mog( ++ "0700", "$(sys.user_data[username])", "$(sys.user_data[username])" ++ ); ++ ++ methods: ++ "prepare_host_specific_data" ++ usebundle => file_copy( ++ "$(this.promise_filename).json", ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)host_specific.json" ++ ), ++ handle => "prepare_host_specific_data"; ++ ++ "empty_promises_cf" ++ usebundle => file_make("$(sys.inputdir)/promises.cf", ''); ++} ++ ++####################################################### ++bundle agent check ++{ ++ vars: ++ "command" ++ string => "$(sys.cf_promises) --show-vars -f $(sys.inputdir)/promises.cf"; ++ ++ methods: ++ "" ++ usebundle => dcs_passif_output( ++ # both dotted keys are warned about, and both variables are still ++ # installed at the address the dots ask for ++ ".*warning: The '.' in the CMDB key 'com\.dotted\.key' was taken as a scope separator: the variable is named 'dotted\.key' in the 'com' scope, addressed as 'data:com\.dotted\.key'.*warning: The '.' in the CMDB key 'vcom\.vdotted\.vkey' was taken as a scope separator.*data:com\.dotted\.key\s+dotted_value\s.*data:myscope\.myvar\s+scoped_value\s.*data:vcom\.vdotted\.vkey\s+dotted_value2\s.*", ++ # a plain "bundle.variable" key is a supported way to pick a scope, ++ # it must not be warned about ++ ".*warning:[^\n]*'myscope\.myvar'.*", ++ $(command), ++ $(this.promise_filename) ++ ); ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf.json b/tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf.json +new file mode 100644 +index 000000000..b61b5e31d +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf.json +@@ -0,0 +1,4 @@ ++{ ++ "vars": { "com.dotted.key": "dotted_value", "myscope.myvar": "scoped_value" }, ++ "variables": { "vcom.vdotted.vkey": { "value": "dotted_value2" } } ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/15-variable-references-metadata-format.cf.json b/tests/acceptance/00_basics/06_host_specific_data/15-variable-references-metadata-format.cf.json +index 1c3f1a47a..a3fe6a85b 100644 +--- a/tests/acceptance/00_basics/06_host_specific_data/15-variable-references-metadata-format.cf.json ++++ b/tests/acceptance/00_basics/06_host_specific_data/15-variable-references-metadata-format.cf.json +@@ -1,8 +1,6 @@ + { + "variables": { +- "good_key": { +- "value": "a literal value" +- }, ++ "good_key": { "value": "a literal value" }, + "key_with_unresolved_value": { + "value": "$(sys.workdir)", + "comment": "the offending entry must be named by this key, not by 'value'", +diff --git a/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf +new file mode 100644 +index 000000000..46b16d910 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf +@@ -0,0 +1,87 @@ ++# test that one entry with an unresolved variable reference makes the CMDB ++# loader install nothing from host_specific.json: the file is all or nothing, ++# so no entry can be left behind without the entry that was meant to guard it ++body common control ++{ ++ inputs => { "../../default.sub.cf" }; ++ bundlesequence => { default("$(this.promise_filename)") }; ++ version => "1.0"; ++} ++ ++####################################################### ++bundle agent init ++{ ++ files: ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)." ++ create => "true", ++ perms => mog( ++ "0700", "$(sys.user_data[username])", "$(sys.user_data[username])" ++ ); ++ ++ methods: ++ "prepare_host_specific_data" ++ usebundle => file_copy( ++ "$(this.promise_filename).json", ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)host_specific.json" ++ ), ++ handle => "prepare_host_specific_data"; ++ ++ "empty_promises_cf" ++ usebundle => file_make("$(sys.inputdir)/promises.cf", ''); ++} ++ ++####################################################### ++bundle agent check ++{ ++ vars: ++ "error_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-classes -f $(sys.inputdir)/promises.cf", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ "vars_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-vars -f $(sys.inputdir)/promises.cf | $(G.grep) good_key", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ classes: ++ # The offending entry is rejected and named. ++ "offender_named" ++ expression => regcmp( ++ ".*error: Invalid 'vars' CMDB data in '.*host_specific\.json', cannot contain variable references \(offending value in entry 'key_with_unresolved_value': .*", ++ "$(error_output)" ++ ); ++ ++ # The loader says that it installed none of the file. ++ "nothing_installed_reported" ++ expression => regcmp( ++ ".*error: CMDB data file '.*host_specific\.json' has invalid entries \(see above\), installing none of its contents.*", ++ "$(error_output)" ++ ); ++ ++ # Its sibling entry in the same section is NOT installed: a single bad ++ # entry takes the whole file down, so that policy never runs with only ++ # part of the data it was written against. ++ "good_entry_absent" ++ not => regcmp( ++ ".*good_key.*", ++ "$(vars_output)" ++ ); ++ ++ "ok" and => { "offender_named", "nothing_installed_reported", "good_entry_absent" }; ++ ++ reports: ++ ok:: ++ "$(this.promise_filename) Pass"; ++ ++ !ok:: ++ "$(this.promise_filename) FAIL"; ++ ++ !ok.DEBUG:: ++ "error_output was: '$(error_output)'"; ++ "vars_output was: '$(vars_output)'"; ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf.json b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf.json +new file mode 100644 +index 000000000..5f225ddcc +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf.json +@@ -0,0 +1,6 @@ ++{ ++ "vars": { ++ "good_key": "a literal value", ++ "key_with_unresolved_value": "$(sys.workdir)" ++ } ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf b/tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf +new file mode 100644 +index 000000000..824c7b05f +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf +@@ -0,0 +1,93 @@ ++# test that a repeated object member makes the CMDB loader install nothing from ++# host_specific.json: the parser used to keep only the last of the two, so the ++# file looked valid and the discarded entry was never checked ++body common control ++{ ++ inputs => { "../../default.sub.cf" }; ++ bundlesequence => { default("$(this.promise_filename)") }; ++ version => "1.0"; ++} ++ ++####################################################### ++bundle agent init ++{ ++ files: ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)." ++ create => "true", ++ perms => mog( ++ "0700", "$(sys.user_data[username])", "$(sys.user_data[username])" ++ ); ++ ++ methods: ++ "prepare_host_specific_data" ++ usebundle => file_copy( ++ "$(this.promise_filename).json", ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)host_specific.json" ++ ), ++ handle => "prepare_host_specific_data"; ++ ++ "empty_promises_cf" ++ usebundle => file_make("$(sys.inputdir)/promises.cf", ''); ++} ++ ++####################################################### ++bundle agent check ++{ ++ vars: ++ "error_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-classes -f $(sys.inputdir)/promises.cf", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ "vars_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-vars -f $(sys.inputdir)/promises.cf | $(G.grep) good_key", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ "classes_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-classes -f $(sys.inputdir)/promises.cf | $(G.grep) cmdb_good_class", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ classes: ++ # The file is refused by the parser, and the loader says so. ++ "refusal_reported" ++ expression => regcmp( ++ ".*error: Could not parse JSON file .*host_specific\.json: Unwilling to parse json data with a duplicate object member.*", ++ "$(error_output)" ++ ); ++ ++ # Nothing is installed from it: neither the variable nor the class next ++ # to the offending text. ++ "good_entry_absent" ++ not => regcmp( ++ ".*good_key.*", ++ "$(vars_output)" ++ ); ++ ++ "good_class_absent" ++ not => regcmp( ++ ".*cmdb_good_class.*", ++ "$(classes_output)" ++ ); ++ ++ "ok" and => { "refusal_reported", "good_entry_absent", "good_class_absent" }; ++ ++ reports: ++ ok:: ++ "$(this.promise_filename) Pass"; ++ ++ !ok:: ++ "$(this.promise_filename) FAIL"; ++ ++ !ok.DEBUG:: ++ "error_output was: '$(error_output)'"; ++ "vars_output was: '$(vars_output)'"; ++ "classes_output was: '$(classes_output)'"; ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf.json b/tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf.json +new file mode 100644 +index 000000000..93c902882 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf.json +@@ -0,0 +1,10 @@ ++{ ++ "vars": { ++ "good_key": "a literal value", ++ "repeated_key": "first", ++ "repeated_key": "second" ++ }, ++ "classes": { ++ "ns1:cmdb_good_class": "any::" ++ } ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf b/tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf +new file mode 100644 +index 000000000..4f1c2d670 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf +@@ -0,0 +1,93 @@ ++# test that input after the document makes the CMDB loader install nothing from ++# host_specific.json: the parser used to stop after the first complete object ++# and never look at what followed ++body common control ++{ ++ inputs => { "../../default.sub.cf" }; ++ bundlesequence => { default("$(this.promise_filename)") }; ++ version => "1.0"; ++} ++ ++####################################################### ++bundle agent init ++{ ++ files: ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)." ++ create => "true", ++ perms => mog( ++ "0700", "$(sys.user_data[username])", "$(sys.user_data[username])" ++ ); ++ ++ methods: ++ "prepare_host_specific_data" ++ usebundle => file_copy( ++ "$(this.promise_filename).json", ++ "$(sys.workdir)$(const.dirsep)data$(const.dirsep)host_specific.json" ++ ), ++ handle => "prepare_host_specific_data"; ++ ++ "empty_promises_cf" ++ usebundle => file_make("$(sys.inputdir)/promises.cf", ''); ++} ++ ++####################################################### ++bundle agent check ++{ ++ vars: ++ "error_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-classes -f $(sys.inputdir)/promises.cf", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ "vars_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-vars -f $(sys.inputdir)/promises.cf | $(G.grep) good_key", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ "classes_output" ++ string => execresult( ++ "$(sys.cf_promises) --show-classes -f $(sys.inputdir)/promises.cf | $(G.grep) cmdb_good_class", ++ "useshell" ++ ), ++ depends_on => { "prepare_host_specific_data" }; ++ ++ classes: ++ # The file is refused by the parser, and the loader says so. ++ "refusal_reported" ++ expression => regcmp( ++ ".*error: Could not parse JSON file .*host_specific\.json: Unwilling to parse json data with trailing non-whitespace characters.*", ++ "$(error_output)" ++ ); ++ ++ # Nothing is installed from it: neither the variable nor the class next ++ # to the offending text. ++ "good_entry_absent" ++ not => regcmp( ++ ".*good_key.*", ++ "$(vars_output)" ++ ); ++ ++ "good_class_absent" ++ not => regcmp( ++ ".*cmdb_good_class.*", ++ "$(classes_output)" ++ ); ++ ++ "ok" and => { "refusal_reported", "good_entry_absent", "good_class_absent" }; ++ ++ reports: ++ ok:: ++ "$(this.promise_filename) Pass"; ++ ++ !ok:: ++ "$(this.promise_filename) FAIL"; ++ ++ !ok.DEBUG:: ++ "error_output was: '$(error_output)'"; ++ "vars_output was: '$(vars_output)'"; ++ "classes_output was: '$(classes_output)'"; ++} +diff --git a/tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf.json b/tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf.json +new file mode 100644 +index 000000000..550169977 +--- /dev/null ++++ b/tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf.json +@@ -0,0 +1,9 @@ ++{ ++ "vars": { ++ "good_key": "a literal value" ++ }, ++ "classes": { ++ "ns1:cmdb_good_class": "any::" ++ } ++} ++{ "vars": { "other_key": "never looked at" } } +diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am +index 024e25f78..784872a19 100644 +--- a/tests/unit/Makefile.am ++++ b/tests/unit/Makefile.am +@@ -99,6 +99,7 @@ check_PROGRAMS = \ + set_domainname_test \ + evalfunction_test \ + eval_context_test \ ++ cmdb_test \ + regex_test \ + lastseen_test \ + lastseen_migration_test \ +diff --git a/tests/unit/cmdb_test.c b/tests/unit/cmdb_test.c +new file mode 100644 +index 000000000..2b9984f44 +--- /dev/null ++++ b/tests/unit/cmdb_test.c +@@ -0,0 +1,779 @@ ++#include ++ ++/* Includes cmdb.c itself, with its test hook (not in the production library) */ ++#define TEST_UNIT_TEST ++#include ++ ++#include ++#include /* Class */ ++#include /* StringSetContains */ ++#include /* StringEqual */ ++#include /* VarRefParse */ ++#include ++#include /* xsnprintf */ ++ ++/* LoadCMDBData() is all-or-nothing: if any entry of host_specific.json cannot ++ * be validated or installed, none of the file is installed. */ ++ ++char CFWORKDIR[CF_BUFSIZE]; ++ ++void tests_setup(void) ++{ ++ static char env[] = /* Needs to be static for putenv() */ ++ "CFENGINE_TEST_OVERRIDE_WORKDIR=/tmp/CFENGINE_cmdb_test.XXXXXX"; ++ char *workdir = strchr(env, '='); ++ assert(workdir && workdir[1] == '/'); ++ workdir++; ++ ++ mkdtemp(workdir); ++ strlcpy(CFWORKDIR, workdir, CF_BUFSIZE); ++ putenv(env); ++ ++ mkdir(GetDataDir(), 0700); ++} ++ ++void tests_teardown(void) ++{ ++ char cmd[CF_BUFSIZE]; ++ xsnprintf(cmd, CF_BUFSIZE, "rm -rf '%s'", CFWORKDIR); ++ system(cmd); ++} ++ ++static void CMDBFilePath(char *path, size_t size) ++{ ++ xsnprintf(path, size, "%s/host_specific.json", GetDataDir()); ++} ++ ++static void WriteCMDBFile(const char *json) ++{ ++ char path[CF_BUFSIZE]; ++ CMDBFilePath(path, sizeof(path)); ++ ++ FILE *f = fopen(path, "w"); ++ assert_true(f != NULL); ++ assert_true(fputs(json, f) >= 0); ++ assert_int_equal(fclose(f), 0); ++} ++ ++static void RemoveCMDBFile(void) ++{ ++ char path[CF_BUFSIZE]; ++ CMDBFilePath(path, sizeof(path)); ++ unlink(path); ++} ++ ++/* The variable's scalar value, or NULL if it is not installed. */ ++static const char *InstalledScalar(const EvalContext *ctx, const char *ref_str) ++{ ++ VarRef *ref = VarRefParse(ref_str); ++ DataType type; ++ const void *value = EvalContextVariableGet(ctx, ref, &type); ++ VarRefDestroy(ref); ++ if (value == NULL) ++ { ++ return NULL; ++ } ++ assert_int_equal(type, CF_DATA_TYPE_STRING); ++ return value; ++} ++ ++static bool VariableIsInstalled(const EvalContext *ctx, const char *ref_str) ++{ ++ VarRef *ref = VarRefParse(ref_str); ++ DataType type; ++ const bool found = ++ (EvalContextVariableGet(ctx, ref, &type) != NULL); ++ VarRefDestroy(ref); ++ return found; ++} ++ ++static bool ClassIsInstalled(const EvalContext *ctx, const char *name) ++{ ++ return (EvalContextClassGet(ctx, "data", name) != NULL); ++} ++ ++static void test_missing_file_is_not_an_error(void) ++{ ++ RemoveCMDBFile(); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(LoadCMDBData(ctx)); ++ EvalContextDestroy(ctx); ++} ++ ++static void test_all_good_file_installs_everything(void) ++{ ++ WriteCMDBFile( ++ "{\n" ++ " \"vars\": {\n" ++ " \"good_var\": \"from_vars\",\n" ++ " \"good_list\": [\"a\", \"b\"],\n" ++ " \"good_data\": { \"k\": \"v\" }\n" ++ " },\n" ++ " \"variables\": {\n" ++ " \"good_variable\": { \"value\": \"from_variables\",\n" ++ " \"tags\": [\"t1\"], \"comment\": \"c\" },\n" ++ " \"good_bare\": \"bare\"\n" ++ " },\n" ++ " \"classes\": {\n" ++ " \"good_class\": \"any::\",\n" ++ " \"ns1:good_ns_class\": { \"class_expressions\": [] }\n" ++ " }\n" ++ "}\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(LoadCMDBData(ctx)); ++ ++ assert_string_equal(InstalledScalar(ctx, "data:variables.good_var"), ++ "from_vars"); ++ assert_true(VariableIsInstalled(ctx, "data:variables.good_list")); ++ assert_true(VariableIsInstalled(ctx, "data:variables.good_data")); ++ assert_string_equal(InstalledScalar(ctx, "data:variables.good_variable"), ++ "from_variables"); ++ assert_string_equal(InstalledScalar(ctx, "data:variables.good_bare"), ++ "bare"); ++ assert_true(ClassIsInstalled(ctx, "good_class")); ++ assert_true(EvalContextClassGet(ctx, "ns1", "good_ns_class") != NULL); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_bad_var_entry_installs_nothing(void) ++{ ++ /* A JSON null has no value to install; the good entries on both sides of ++ * it, and the ones in the other sections, must not be installed either. */ ++ WriteCMDBFile( ++ "{\n" ++ " \"vars\": {\n" ++ " \"first_good_var\": \"x\",\n" ++ " \"bad_var\": null,\n" ++ " \"last_good_var\": \"y\"\n" ++ " },\n" ++ " \"variables\": { \"good_variable\": { \"value\": \"z\" } },\n" ++ " \"classes\": { \"good_class\": \"any::\" }\n" ++ "}\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_false(LoadCMDBData(ctx)); ++ ++ assert_false(VariableIsInstalled(ctx, "data:variables.first_good_var")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.bad_var")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.last_good_var")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_variable")); ++ assert_false(ClassIsInstalled(ctx, "good_class")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_bad_variables_entry_installs_nothing(void) ++{ ++ /* An entry with an unresolved variable reference, in the 'variables' ++ * section, next to good entries in 'vars' and 'variables'. */ ++ WriteCMDBFile( ++ "{\n" ++ " \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": {\n" ++ " \"good_variable\": { \"value\": \"y\" },\n" ++ " \"unresolved\": { \"value\": \"$(sys.workdir)\" }\n" ++ " }\n" ++ "}\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_false(LoadCMDBData(ctx)); ++ ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_var")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_variable")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.unresolved")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_bad_class_entry_installs_nothing(void) ++{ ++ /* Only "any::" is a valid class specification. */ ++ WriteCMDBFile( ++ "{\n" ++ " \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": {\n" ++ " \"first_good_class\": \"any::\",\n" ++ " \"bad_class\": [\"linux::\"],\n" ++ " \"last_good_class\": [\"any::\"]\n" ++ " }\n" ++ "}\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_false(LoadCMDBData(ctx)); ++ ++ assert_false(ClassIsInstalled(ctx, "first_good_class")); ++ assert_false(ClassIsInstalled(ctx, "bad_class")); ++ assert_false(ClassIsInstalled(ctx, "last_good_class")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_var")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_bad_metadata_installs_nothing(void) ++{ ++ /* Malformed tags make the entry invalid, not just tag-less. */ ++ WriteCMDBFile( ++ "{\n" ++ " \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": {\n" ++ " \"bad_tags\": { \"value\": \"y\", \"tags\": \"not_an_array\" }\n" ++ " }\n" ++ "}\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_false(LoadCMDBData(ctx)); ++ ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_var")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.bad_tags")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_section_of_wrong_type_installs_nothing(void) ++{ ++ WriteCMDBFile( ++ "{\n" ++ " \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": [ \"any::\" ]\n" ++ "}\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_false(LoadCMDBData(ctx)); ++ ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_var")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_reload_does_not_keep_earlier_data(void) ++{ ++ /* What cf-execd, cf-serverd and cf-reactor do on a policy reload: clear ++ * the context, then load again. A file that has turned bad since must ++ * leave no variables behind, neither old nor new. */ ++ WriteCMDBFile("{ \"vars\": { \"good_var\": \"old\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(LoadCMDBData(ctx)); ++ assert_string_equal(InstalledScalar(ctx, "data:variables.good_var"), "old"); ++ ++ WriteCMDBFile("{ \"vars\": { \"good_var\": \"new\", \"bad\": null } }\n"); ++ EvalContextClear(ctx); ++ assert_false(LoadCMDBData(ctx)); ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_var")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++/* Load json, expect failure, and expect none of the named variables (full ++ * references) and classes (in the 'data' namespace) to be installed. */ ++static void assert_rejected_and_nothing_installed(const char *json, ++ const char *const *variables, ++ const char *const *classes) ++{ ++ WriteCMDBFile(json); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_false(LoadCMDBData(ctx)); ++ ++ for (size_t i = 0; variables != NULL && variables[i] != NULL; i++) ++ { ++ if (VariableIsInstalled(ctx, variables[i])) ++ { ++ print_error("variable '%s' is installed by a rejected file\n", ++ variables[i]); ++ } ++ assert_false(VariableIsInstalled(ctx, variables[i])); ++ } ++ for (size_t i = 0; classes != NULL && classes[i] != NULL; i++) ++ { ++ if (ClassIsInstalled(ctx, classes[i])) ++ { ++ print_error("class '%s' is installed by a rejected file\n", ++ classes[i]); ++ } ++ assert_false(ClassIsInstalled(ctx, classes[i])); ++ } ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_unbalanced_index_key_installs_nothing(void) ++{ ++ /* VarRefParse() answers 'tendcf_interlock[' with a reference to ++ * 'tendcf_interlock', which would replace the valid interlock container ++ * and leave the service container installed. */ ++ const char *const absent[] = { ++ "data:variables.tendcf_interlock", ++ "data:variables.tendcf_service", ++ NULL ++ }; ++ ++ /* in 'vars' */ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": {\n" ++ " \"tendcf_interlock\": { \"demo\": { \"bundle\": \"demo\" } },\n" ++ " \"tendcf_interlock[\": [],\n" ++ " \"tendcf_service\": { \"svc\": { \"bundle\": \"demo\" } } } }\n", ++ absent, NULL); ++ ++ /* in 'variables' */ ++ assert_rejected_and_nothing_installed( ++ "{ \"variables\": {\n" ++ " \"tendcf_interlock\": { \"value\": { \"demo\": { \"bundle\": \"demo\" } } },\n" ++ " \"tendcf_interlock[\": { \"value\": [] },\n" ++ " \"tendcf_service\": { \"value\": { \"svc\": { \"bundle\": \"demo\" } } } } }\n", ++ absent, NULL); ++ ++ /* the interlock in 'variables', the service in 'vars' */ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": {\n" ++ " \"tendcf_service\": { \"svc\": { \"bundle\": \"demo\" } } },\n" ++ " \"variables\": {\n" ++ " \"tendcf_interlock\": { \"value\": { \"demo\": { \"bundle\": \"demo\" } } },\n" ++ " \"tendcf_interlock[\": { \"value\": [] } } }\n", ++ absent, NULL); ++ ++ /* a stray closing bracket */ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": { \"good\": \"x\", \"bad]\": \"y\" } }\n", ++ (const char *const[]) { "data:variables.good", NULL }, NULL); ++} ++ ++static void test_balanced_index_key_installs_nothing(void) ++{ ++ /* An index would make the entry an element of some other variable. */ ++ const char *const absent[] = { "data:variables.good", NULL }; ++ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": { \"good\": \"x\", \"array[key]\": \"y\" } }\n", ++ absent, NULL); ++ assert_rejected_and_nothing_installed( ++ "{ \"variables\": { \"good\": \"x\", \"array[key]\": { \"value\": \"y\" } } }\n", ++ absent, NULL); ++} ++ ++static void test_frame_dependent_scopes_install_nothing(void) ++{ ++ /* These scopes name tables that exist only while a bundle, body or ++ * promise is evaluated (or take no namespace); installing into them ++ * during discovery crashes the agent, after earlier entries went in. */ ++ const char *const keys[] = { ++ "default:body.bad", "body.bad", "edit.bad", "default:edit.bad", ++ "this.v", "default:this.v", "sys.bad", "default:sys.bad", "const.bad", ++ "mon.bad", "match.bad", NULL ++ }; ++ const char *const absent[] = { "data:variables.good", NULL }; ++ const char *const absent_class[] = { "good_class", NULL }; ++ ++ for (size_t i = 0; keys[i] != NULL; i++) ++ { ++ char json[CF_BUFSIZE]; ++ ++ xsnprintf(json, sizeof(json), ++ "{ \"vars\": { \"good\": \"x\", \"%s\": \"y\" }," ++ " \"classes\": { \"good_class\": \"any::\" } }\n", keys[i]); ++ assert_rejected_and_nothing_installed(json, absent, absent_class); ++ ++ xsnprintf(json, sizeof(json), ++ "{ \"variables\": { \"good\": \"x\", \"%s\": { \"value\": \"y\" } }," ++ " \"classes\": { \"good_class\": \"any::\" } }\n", keys[i]); ++ assert_rejected_and_nothing_installed(json, absent, absent_class); ++ } ++} ++ ++static void test_def_scope_is_installed(void) ++{ ++ /* 'def' is no more special than a bundle name (CFE-3668) and works. */ ++ WriteCMDBFile("{ \"vars\": { \"def.mine\": \"x\", \"mybundle.mine\": \"y\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(LoadCMDBData(ctx)); ++ assert_string_equal(InstalledScalar(ctx, "data:def.mine"), "x"); ++ assert_string_equal(InstalledScalar(ctx, "data:mybundle.mine"), "y"); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_null_section_installs_nothing(void) ++{ ++ /* A section that is present as JSON null is not an absent section. */ ++ const char *const absent[] = { ++ "data:variables.good_var", "data:variables.good_variable", NULL ++ }; ++ const char *const absent_class[] = { "good_class", NULL }; ++ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": null,\n" ++ " \"variables\": { \"good_variable\": { \"value\": \"y\" } },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ absent, absent_class); ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": null,\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ absent, absent_class); ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": { \"good_variable\": { \"value\": \"y\" } },\n" ++ " \"classes\": null }\n", ++ absent, absent_class); ++} ++ ++static void test_rule_array_of_wrong_type_installs_nothing(void) ++{ ++ /* The sole element must be a string; an object or array there used to ++ * reach an assertion that aborts the agent. */ ++ const char *const absent[] = { "data:variables.good_var", NULL }; ++ const char *const absent_class[] = { "good_class", "bad_class", NULL }; ++ const char *const specs[] = { ++ "{ \"class_expressions\": [ {} ] }", ++ "{ \"class_expressions\": [ [] ] }", ++ "{ \"class_expressions\": [ 1 ] }", ++ "{ \"class_expressions\": [ null ] }", ++ "{ \"regular_expressions\": [ {} ] }", ++ "{ \"regular_expressions\": [ [] ] }", ++ "{ \"regular_expressions\": [ true ] }", ++ NULL ++ }; ++ ++ for (size_t i = 0; specs[i] != NULL; i++) ++ { ++ char json[CF_BUFSIZE]; ++ xsnprintf(json, sizeof(json), ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"good_class\": \"any::\", \"bad_class\": %s } }\n", ++ specs[i]); ++ assert_rejected_and_nothing_installed(json, absent, absent_class); ++ } ++} ++ ++static void test_unknown_top_level_key_installs_nothing(void) ++{ ++ /* A misspelled section would otherwise drop its data and keep the rest. */ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"varables\": { \"tendcf_interlock\": { \"value\": \"y\" } },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ (const char *const[]) { "data:variables.good_var", NULL }, ++ (const char *const[]) { "good_class", NULL }); ++} ++ ++static void test_duplicate_variable_installs_nothing(void) ++{ ++ /* One spelling replacing another could swap out a valid entry. */ ++ const char *const absent[] = { "data:variables.dup", "data:variables.good", NULL }; ++ ++ assert_rejected_and_nothing_installed( ++ "{ \"vars\": { \"good\": \"x\", \"dup\": \"first\" },\n" ++ " \"variables\": { \"data:variables.dup\": { \"value\": \"second\" } } }\n", ++ absent, NULL); ++} ++ ++static bool FailInsertionOfFailMe(const char *key) ++{ ++ return StringEqual(key, "fail_me"); ++} ++ ++static void test_failed_insertion_rolls_back_everything(void) ++{ ++ /* Validation accepts the file, then one variable cannot be inserted ++ * after the others went in (forced through the test seam, as no real ++ * input gets there). Whatever this load installed must be taken back, ++ * and a class that was there before it must be left alone. */ ++ WriteCMDBFile( ++ "{ \"vars\": { \"first\": \"1\", \"fail_me\": \"2\", \"last\": \"3\" },\n" ++ " \"variables\": { \"in_variables\": { \"value\": \"4\" } },\n" ++ " \"classes\": { \"new_class\": \"any::\", \"ns1:new_ns_class\": \"any::\"," ++ " \"pre_class\": \"any::\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(EvalContextClassPutSoftNS(ctx, "data", "pre_class", ++ CONTEXT_SCOPE_NAMESPACE, "")); ++ ++ CMDBTestFailInsertion = FailInsertionOfFailMe; ++ const bool loaded = LoadCMDBData(ctx); ++ CMDBTestFailInsertion = NULL; ++ assert_false(loaded); ++ ++ assert_false(VariableIsInstalled(ctx, "data:variables.first")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.fail_me")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.last")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.in_variables")); ++ assert_false(ClassIsInstalled(ctx, "new_class")); ++ assert_true(EvalContextClassGet(ctx, "ns1", "new_ns_class") == NULL); ++ assert_true(ClassIsInstalled(ctx, "pre_class")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++/* Is there a class of that name in the 'data' namespace, put there by ++ * something else than the CMDB (it has pre_tag, not the CMDB's tag)? */ ++static bool ClassIsUntouched(const EvalContext *ctx, const char *name) ++{ ++ const Class *cls = EvalContextClassGet(ctx, "data", name); ++ return (cls != NULL && ++ StringSetContains(cls->tags, "pre_tag") && ++ !StringSetContains(cls->tags, CMDB_SOURCE_TAG)); ++} ++ ++static void test_whitespace_class_name_installs_nothing(void) ++{ ++ /* EvalContextClassPutTagsSet() refuses a default-namespace class whose ++ * name is only whitespace, and files "x " as "x_". The good variable and ++ * class beside it must not be installed either. */ ++ const char *const names[] = { ++ " ", "\\t", "default: ", "ns1: ", "trailing ", " leading", "tab\\t", ++ NULL ++ }; ++ const char *const absent[] = { "data:variables.good_var", NULL }; ++ const char *const absent_class[] = { ++ "good_class", "trailing_", "_leading", "tab_", NULL ++ }; ++ ++ for (size_t i = 0; names[i] != NULL; i++) ++ { ++ char json[CF_BUFSIZE]; ++ xsnprintf(json, sizeof(json), ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"good_class\": \"any::\", \"%s\": \"any::\" } }\n", ++ names[i]); ++ assert_rejected_and_nothing_installed(json, absent, absent_class); ++ } ++} ++ ++static void test_existing_class_is_not_an_error(void) ++{ ++ /* A class that is there already stays as it is, and the rest of the file ++ * is installed; so does one that differs only in how it is spelled. */ ++ WriteCMDBFile( ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"pre_class\": \"any::\", \"pre-aliased\": \"any::\"," ++ " \"new_class\": \"any::\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(EvalContextClassPutSoftNS(ctx, "data", "pre_class", ++ CONTEXT_SCOPE_NAMESPACE, "pre_tag")); ++ assert_true(EvalContextClassPutSoftNS(ctx, "data", "pre_aliased", ++ CONTEXT_SCOPE_NAMESPACE, "pre_tag")); ++ assert_true(LoadCMDBData(ctx)); ++ ++ assert_string_equal(InstalledScalar(ctx, "data:variables.good_var"), "x"); ++ assert_true(ClassIsInstalled(ctx, "new_class")); ++ assert_false(ClassIsUntouched(ctx, "new_class")); ++ assert_true(ClassIsUntouched(ctx, "pre_class")); ++ assert_true(ClassIsUntouched(ctx, "pre_aliased")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_existing_variable_is_refused_and_left_alone(void) ++{ ++ /* Installing it would replace it, and a rollback could only remove it: ++ * the file is refused as a whole and the variable keeps its value. */ ++ WriteCMDBFile( ++ "{ \"vars\": { \"first\": \"new\", \"other\": \"2\" },\n" ++ " \"classes\": { \"new_class\": \"any::\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ VarRef *ref = VarRefParse("data:variables.first"); ++ assert_true(EvalContextVariablePut(ctx, ref, "old", CF_DATA_TYPE_STRING, ++ "pre_tag")); ++ VarRefDestroy(ref); ++ ++ assert_false(LoadCMDBData(ctx)); ++ ++ assert_string_equal(InstalledScalar(ctx, "data:variables.first"), "old"); ++ assert_false(VariableIsInstalled(ctx, "data:variables.other")); ++ assert_false(ClassIsInstalled(ctx, "new_class")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_rollback_removes_non_canonical_and_keeps_aliased(void) ++{ ++ /* 'dash-class' is filed as 'dash_class' and rolled back by that name; ++ * 'pre-class' is the existing 'pre_class' by another spelling, so neither ++ * installing nor rolling back touches it. */ ++ WriteCMDBFile( ++ "{ \"vars\": { \"first\": \"1\", \"fail_me\": \"2\" },\n" ++ " \"classes\": { \"dash-class\": \"any::\", \"pre-class\": \"any::\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(EvalContextClassPutSoftNS(ctx, "data", "pre_class", ++ CONTEXT_SCOPE_NAMESPACE, "pre_tag")); ++ ++ /* Without the failure the class is there under the canonical name */ ++ CMDBTestFailInsertion = NULL; ++ assert_true(LoadCMDBData(ctx)); ++ assert_true(ClassIsInstalled(ctx, "dash_class")); ++ assert_true(ClassIsUntouched(ctx, "pre_class")); ++ EvalContextClear(ctx); ++ assert_true(EvalContextClassPutSoftNS(ctx, "data", "pre_class", ++ CONTEXT_SCOPE_NAMESPACE, "pre_tag")); ++ ++ CMDBTestFailInsertion = FailInsertionOfFailMe; ++ const bool loaded = LoadCMDBData(ctx); ++ CMDBTestFailInsertion = NULL; ++ assert_false(loaded); ++ ++ assert_false(ClassIsInstalled(ctx, "dash_class")); ++ assert_false(VariableIsInstalled(ctx, "data:variables.first")); ++ assert_true(ClassIsUntouched(ctx, "pre_class")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static bool FailInsertionOfFailClass(const char *key) ++{ ++ return StringEqual(key, "fail_class"); ++} ++ ++static void test_failed_class_insertion_rolls_back_everything(void) ++{ ++ /* A class that cannot be inserted is not "one that exists": the classes ++ * before it are taken back, the ones after it and the variables are not ++ * installed, and a class that was there before is left alone. */ ++ WriteCMDBFile( ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"pre_class\": \"any::\", \"new-class\": \"any::\"," ++ " \"ns1:new_ns_class\": \"any::\"," ++ " \"fail_class\": \"any::\", \"late_class\": \"any::\" } }\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(EvalContextClassPutSoftNS(ctx, "data", "pre_class", ++ CONTEXT_SCOPE_NAMESPACE, "pre_tag")); ++ ++ CMDBTestFailInsertion = FailInsertionOfFailClass; ++ const bool loaded = LoadCMDBData(ctx); ++ CMDBTestFailInsertion = NULL; ++ assert_false(loaded); ++ ++ assert_false(VariableIsInstalled(ctx, "data:variables.good_var")); ++ assert_false(ClassIsInstalled(ctx, "new_class")); ++ assert_true(EvalContextClassGet(ctx, "ns1", "new_ns_class") == NULL); ++ assert_false(ClassIsInstalled(ctx, "fail_class")); ++ assert_false(ClassIsInstalled(ctx, "late_class")); ++ assert_true(ClassIsUntouched(ctx, "pre_class")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++static void test_repeated_member_or_trailing_input_installs_nothing(void) ++{ ++ /* What the lenient parser repaired or ignored before validation saw it: ++ * a repeated member replaced the earlier one (an invalid earlier value ++ * vanished, or a whole section did), and anything after the document was ++ * not looked at. Each file has a good variable and class next to it. */ ++ const char *const absent[] = { ++ "data:variables.good_var", "data:variables.v", "data:variables.good", ++ NULL ++ }; ++ const char *const absent_class[] = { "good_class", "c", NULL }; ++ const char *const files[] = { ++ /* repeated member at the top level (section replaced) */ ++ "{ \"vars\": { \"good_var\": \"x\" }, \"vars\": { \"good\": \"y\" },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ /* repeated member, the earlier one invalid */ ++ "{ \"vars\": { \"good_var\": \"x\", \"good\": null, \"good\": \"ok\" },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ /* repeated member in 'classes' */ ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"good_class\": \"any::\", \"c\": \"any::\"," ++ " \"c\": \"any::\" } }\n", ++ /* repeated member in 'variables', and in a nested object */ ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": { \"v\": { \"value\": \"1\" }, \"v\": { \"value\": \"2\" } },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": { \"v\": { \"value\": \"1\", \"value\": \"2\" } },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }\n", ++ /* trailing input */ ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"good_class\": \"any::\" } } {}\n", ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"good_class\": \"any::\" } }x\n", ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"variables\": {},\n" ++ " \"classes\": { \"good_class\": \"any::\" } } ]\n", ++ NULL ++ }; ++ ++ for (size_t i = 0; files[i] != NULL; i++) ++ { ++ assert_rejected_and_nothing_installed(files[i], absent, absent_class); ++ } ++} ++ ++static void test_whitespace_after_document_is_fine(void) ++{ ++ /* The control for the above: the same kind of file, nothing repeated and ++ * nothing after the document but whitespace, loads. */ ++ WriteCMDBFile( ++ "{ \"vars\": { \"good_var\": \"x\" },\n" ++ " \"classes\": { \"good_class\": \"any::\" } } \n\t\n\n"); ++ ++ EvalContext *ctx = EvalContextNew(); ++ assert_true(LoadCMDBData(ctx)); ++ assert_string_equal(InstalledScalar(ctx, "data:variables.good_var"), "x"); ++ assert_true(ClassIsInstalled(ctx, "good_class")); ++ ++ EvalContextDestroy(ctx); ++ RemoveCMDBFile(); ++} ++ ++int main() ++{ ++ PRINT_TEST_BANNER(); ++ tests_setup(); ++ ++ const UnitTest tests[] = ++ { ++ unit_test(test_missing_file_is_not_an_error), ++ unit_test(test_all_good_file_installs_everything), ++ unit_test(test_bad_var_entry_installs_nothing), ++ unit_test(test_bad_variables_entry_installs_nothing), ++ unit_test(test_bad_class_entry_installs_nothing), ++ unit_test(test_bad_metadata_installs_nothing), ++ unit_test(test_section_of_wrong_type_installs_nothing), ++ unit_test(test_reload_does_not_keep_earlier_data), ++ unit_test(test_unbalanced_index_key_installs_nothing), ++ unit_test(test_balanced_index_key_installs_nothing), ++ unit_test(test_frame_dependent_scopes_install_nothing), ++ unit_test(test_def_scope_is_installed), ++ unit_test(test_null_section_installs_nothing), ++ unit_test(test_rule_array_of_wrong_type_installs_nothing), ++ unit_test(test_unknown_top_level_key_installs_nothing), ++ unit_test(test_duplicate_variable_installs_nothing), ++ unit_test(test_failed_insertion_rolls_back_everything), ++ unit_test(test_whitespace_class_name_installs_nothing), ++ unit_test(test_existing_class_is_not_an_error), ++ unit_test(test_existing_variable_is_refused_and_left_alone), ++ unit_test(test_rollback_removes_non_canonical_and_keeps_aliased), ++ unit_test(test_failed_class_insertion_rolls_back_everything), ++ unit_test(test_repeated_member_or_trailing_input_installs_nothing), ++ unit_test(test_whitespace_after_document_is_fine), ++ }; ++ ++ int ret = run_tests(tests); ++ ++ tests_teardown(); ++ ++ return ret; ++} +-- +2.56.0 + diff --git a/overlay/libntech/0005-Added-JsonParseStrict-and-JsonParseFileStrict.patch b/overlay/libntech/0005-Added-JsonParseStrict-and-JsonParseFileStrict.patch new file mode 100644 index 0000000..ca22374 --- /dev/null +++ b/overlay/libntech/0005-Added-JsonParseStrict-and-JsonParseFileStrict.patch @@ -0,0 +1,665 @@ +From dc3fba212bd323358c4de951acdaae3b182a4ef8 Mon Sep 17 00:00:00 2001 +From: Daniel JB Clark +Date: Sat, 14 Nov 2026 15:44:21 -0500 +Subject: [PATCH 5/6] Added JsonParseStrict() and JsonParseFileStrict() + +JsonObjectAppendElement() removes an existing member of the same name +before appending, so a document with two members of one name loses the +earlier one while it is being parsed, before the caller can look at it. +JsonParse() and everything built on it (JsonParseFile(), JsonParseAnyFile(), +JsonReadDataFile()) also stop at the end of the first value and ignore +whatever follows. + +A caller that must not act on a document it only half-read, such as a +loader that validates every entry, needs both to be refused. The two new +functions do that: a name that an object already holds is refused with the +new JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY before the earlier member is +replaced, at any depth, and anything but whitespace after the value is +refused with JSON_PARSE_ERROR_INVALID_END. The file variant keeps the +size cap and also refuses a null byte, which would hide the rest of the +file from the parser. + +This is opt-in: the existing functions behave exactly as before, with +duplicates resolved last-wins and a valid prefix accepted. The same +choice is made by Jansson as JSON_REJECT_DUPLICATES, and RFC 8259 +section 4 leaves duplicate names to the implementation. + +Changelog: Title +Ticket: none + +Co-Authored-By: Claude Sonnet 5.5 +--- + libutils/json.c | 151 +++++++++++++++++++-- + libutils/json.h | 30 +++++ + tests/unit/json_test.c | 291 +++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 464 insertions(+), 8 deletions(-) + +diff --git a/libutils/json.c b/libutils/json.c +index e46e071..9a4b4f3 100644 +--- a/libutils/json.c ++++ b/libutils/json.c +@@ -2061,6 +2061,7 @@ void JsonWriteCompact(Writer *const w, const JsonElement *const element) + static JsonParseError JsonParseAsObject( + void *lookup_context, + JsonLookup *lookup_function, ++ bool reject_duplicates, + const char **data, + JsonElement **json_out); + +@@ -2176,7 +2177,9 @@ const char *JsonParseErrorToString(const JsonParseError error) + "CFEngine was not built with libyaml support", + [JSON_PARSE_ERROR_LIBYAML_FAILURE] = "libyaml internal failure", + [JSON_PARSE_ERROR_NO_SUCH_FILE] = "No such file or directory", +- [JSON_PARSE_ERROR_NO_DATA] = "No data"}; ++ [JSON_PARSE_ERROR_NO_DATA] = "No data", ++ [JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY] = ++ "Unwilling to parse json data with a duplicate object member"}; + + return parse_errors[error]; + } +@@ -2440,6 +2443,7 @@ static JsonParseError JsonParseAsPrimitive( + static JsonParseError JsonParseAsArray( + void *const lookup_context, + JsonLookup *const lookup_function, ++ const bool reject_duplicates, + const char **const data, + JsonElement **const json_out) + { +@@ -2490,7 +2494,11 @@ static JsonParseError JsonParseAsArray( + } + JsonElement *child_array = NULL; + JsonParseError err = JsonParseAsArray( +- lookup_context, lookup_function, data, &child_array); ++ lookup_context, ++ lookup_function, ++ reject_duplicates, ++ data, ++ &child_array); + if (err != JSON_PARSE_OK) + { + JsonDestroy(array); +@@ -2511,7 +2519,11 @@ static JsonParseError JsonParseAsArray( + } + JsonElement *child_object = NULL; + JsonParseError err = JsonParseAsObject( +- lookup_context, lookup_function, data, &child_object); ++ lookup_context, ++ lookup_function, ++ reject_duplicates, ++ data, ++ &child_object); + if (err != JSON_PARSE_OK) + { + JsonDestroy(array); +@@ -2618,9 +2630,22 @@ static const char *unquoted_key_with_colon(const char *data) + return data + i; + } + ++/** ++ * Does the object under construction already hold a member called key? ++ * JsonObjectAppendElement() would silently replace it. ++ */ ++static bool JsonParseIsDuplicateKey( ++ const bool reject_duplicates, ++ const JsonElement *const object, ++ const char *const key) ++{ ++ return reject_duplicates && JsonObjectGet(object, key) != NULL; ++} ++ + static JsonParseError JsonParseAsObject( + void *const lookup_context, + JsonLookup *const lookup_function, ++ const bool reject_duplicates, + const char **const data, + JsonElement **const json_out) + { +@@ -2685,6 +2710,13 @@ static JsonParseError JsonParseAsObject( + return err; + } + assert(property_name); ++ if (JsonParseIsDuplicateKey( ++ reject_duplicates, object, property_name)) ++ { ++ free(property_name); ++ JsonDestroy(object); ++ return JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY; ++ } + } + break; + +@@ -2712,7 +2744,11 @@ static JsonParseError JsonParseAsObject( + { + JsonElement *child_array = NULL; + JsonParseError err = JsonParseAsArray( +- lookup_context, lookup_function, data, &child_array); ++ lookup_context, ++ lookup_function, ++ reject_duplicates, ++ data, ++ &child_array); + if (err != JSON_PARSE_OK) + { + free(property_name); +@@ -2737,7 +2773,11 @@ static JsonParseError JsonParseAsObject( + { + JsonElement *child_object = NULL; + JsonParseError err = JsonParseAsObject( +- lookup_context, lookup_function, data, &child_object); ++ lookup_context, ++ lookup_function, ++ reject_duplicates, ++ data, ++ &child_object); + if (err != JSON_PARSE_OK) + { + free(property_name); +@@ -2792,6 +2832,13 @@ static JsonParseError JsonParseAsObject( + property_name = xstrndup(*data, ws - *data); + *data = colon; + ++ if (JsonParseIsDuplicateKey( ++ reject_duplicates, object, property_name)) ++ { ++ free(property_name); ++ JsonDestroy(object); ++ return JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY; ++ } + break; + } + else if (property_name != NULL) +@@ -2888,9 +2935,10 @@ JsonParseError JsonParseAll(const char **const data, JsonElement **const json_ou + return error; + } + +-JsonParseError JsonParseWithLookup( ++static JsonParseError JsonParseDocument( + void *const lookup_context, + JsonLookup *const lookup_function, ++ const bool reject_duplicates, + const char **const data, + JsonElement **const json_out) + { +@@ -2906,12 +2954,20 @@ JsonParseError JsonParseWithLookup( + if (**data == '{') + { + return JsonParseAsObject( +- lookup_context, lookup_function, data, json_out); ++ lookup_context, ++ lookup_function, ++ reject_duplicates, ++ data, ++ json_out); + } + else if (**data == '[') + { + return JsonParseAsArray( +- lookup_context, lookup_function, data, json_out); ++ lookup_context, ++ lookup_function, ++ reject_duplicates, ++ data, ++ json_out); + } + else if (IsWhitespace(**data)) + { +@@ -2926,6 +2982,48 @@ JsonParseError JsonParseWithLookup( + return JSON_PARSE_ERROR_NO_DATA; + } + ++JsonParseError JsonParseWithLookup( ++ void *const lookup_context, ++ JsonLookup *const lookup_function, ++ const char **const data, ++ JsonElement **const json_out) ++{ ++ return JsonParseDocument( ++ lookup_context, lookup_function, false, data, json_out); ++} ++ ++JsonParseError JsonParseStrict( ++ const char **const data, JsonElement **const json_out) ++{ ++ assert(json_out != NULL); ++ *json_out = NULL; ++ ++ const JsonParseError error = ++ JsonParseDocument(NULL, NULL, true, data, json_out); ++ if (error != JSON_PARSE_OK) ++ { ++ return error; ++ } ++ ++ /* Like JsonParseAll(), the parser has left data on the last byte it ++ * used, so only whitespace may follow until the terminating null. */ ++ const char *ch = *data; ++ if (*ch != '\0') ++ { ++ ch++; ++ } ++ for (; *ch != '\0'; ch++) ++ { ++ if (!IsWhitespace(*ch)) ++ { ++ JsonDestroy(*json_out); ++ *json_out = NULL; ++ return JSON_PARSE_ERROR_INVALID_END; ++ } ++ } ++ return JSON_PARSE_OK; ++} ++ + JsonParseError JsonParseAnyFile( + const char *const path, + const size_t size_max, +@@ -2970,6 +3068,43 @@ JsonParseError JsonParseFile( + return JsonParseAnyFile(path, size_max, json_out, false); + } + ++JsonParseError JsonParseFileStrict( ++ const char *const path, ++ const size_t size_max, ++ JsonElement **const json_out) ++{ ++ assert(json_out != NULL); ++ *json_out = NULL; ++ ++ bool truncated = false; ++ Writer *contents = FileRead(path, size_max, &truncated); ++ if (contents == NULL) ++ { ++ return JSON_PARSE_ERROR_NO_SUCH_FILE; ++ } ++ if (truncated) ++ { ++ WriterClose(contents); ++ return JSON_PARSE_ERROR_TRUNCATED; ++ } ++ ++ const char *data = StringWriterData(contents); ++ JsonParseError error; ++ if (strlen(data) != StringWriterLength(contents)) ++ { ++ /* A null byte would end the string early and hide the rest of ++ * the file from the parser. */ ++ error = JSON_PARSE_ERROR_INVALID_END; ++ } ++ else ++ { ++ error = JsonParseStrict(&data, json_out); ++ } ++ ++ WriterClose(contents); ++ return error; ++} ++ + bool JsonWalk(JsonElement *element, + JsonElementVisitor object_visitor, + JsonElementVisitor array_visitor, +diff --git a/libutils/json.h b/libutils/json.h +index fd0faa8..ad0c0fd 100644 +--- a/libutils/json.h ++++ b/libutils/json.h +@@ -118,6 +118,7 @@ typedef enum + JSON_PARSE_ERROR_NO_SUCH_FILE, + JSON_PARSE_ERROR_NO_DATA, + JSON_PARSE_ERROR_TRUNCATED, ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY, + + JSON_PARSE_ERROR_MAX + } JsonParseError; +@@ -604,6 +605,22 @@ JsonParseError JsonParse(const char **data, JsonElement **json_out); + */ + JsonParseError JsonParseAll(const char **data, JsonElement **json_out); + ++/** ++ @brief Parse the whole string, refusing input a lenient parse would repair ++ @param data [in] Pointer to the string to parse ++ @param json_out Resulting JSON object, NULL on error ++ @note In contrast to JsonParse() and JsonParseAll(), a document with two ++ members of the same name in one object, at any depth, is refused ++ with JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY before the earlier ++ member is replaced, rather than silently keeping the last one. ++ As in JsonParseAll(), trailing non-whitespace characters are refused ++ with JSON_PARSE_ERROR_INVALID_END. Names are compared as the parser ++ decodes them, which is the comparison JsonObjectGet() makes, so no ++ member is ever replaced. ++ @returns See JsonParseError and JsonParseErrorToString ++ */ ++JsonParseError JsonParseStrict(const char **data, JsonElement **json_out); ++ + /** + @brief Parse a string to create a JsonElement + @param lookup_data [in] Evaluation context for variable lookups +@@ -645,6 +662,19 @@ JsonParseError JsonParseAnyFile( + JsonParseError JsonParseFile( + const char *path, size_t size_max, JsonElement **json_out); + ++/** ++ * @brief Parse JSON from a file as JsonParseStrict() parses a string. ++ * @param path Path to the file ++ * @param size_max Maximum size to read in memory; a larger file is refused ++ * with JSON_PARSE_ERROR_TRUNCATED, as in JsonParseFile() ++ * @param json_out Resulting JSON object, NULL on error ++ * @note A null byte in the file is refused with JSON_PARSE_ERROR_INVALID_END, ++ * as it would otherwise end the text early and hide what follows. ++ * @return See JsonParseError and JsonParseErrorToString ++ */ ++JsonParseError JsonParseFileStrict( ++ const char *path, size_t size_max, JsonElement **json_out); ++ + const char *JsonParseErrorToString(JsonParseError error); + + +diff --git a/tests/unit/json_test.c b/tests/unit/json_test.c +index 200478a..762d601 100644 +--- a/tests/unit/json_test.c ++++ b/tests/unit/json_test.c +@@ -1662,6 +1662,288 @@ static void test_remove_key_from_object(void) + JsonDestroy(object); + } + ++/* Strict parsing: JsonParseStrict() and JsonParseFileStrict() refuse what the ++ * lenient functions repair, and the lenient functions stay as they were. */ ++ ++static void assert_strict_refused( ++ const char *const text, const JsonParseError expected) ++{ ++ const char *data = text; ++ JsonElement *json = NULL; ++ assert_int_equal(expected, JsonParseStrict(&data, &json)); ++ assert_true(json == NULL); ++} ++ ++static void test_parse_strict_valid(void) ++{ ++ const char *data = ++ "{\"vars\": {\"a\": \"1\", \"b\": null}," ++ " \"list\": [{\"k\": 1}, {\"k\": 2}, [{\"k\": 3}]]," ++ " \"other\": {\"a\": \"2\"}, plain: true}"; ++ JsonElement *json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); ++ assert_true(json != NULL); ++ assert_string_equal("1", JsonObjectGetAsString( ++ JsonObjectGetAsObject(json, "vars"), "a")); ++ assert_string_equal("2", JsonObjectGetAsString( ++ JsonObjectGetAsObject(json, "other"), "a")); ++ assert_int_equal(3, JsonLength(JsonObjectGet(json, "list"))); ++ JsonDestroy(json); ++ ++ /* Whitespace is the only thing allowed after the document. */ ++ data = "{} \n\t\r "; ++ json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); ++ assert_true(json != NULL); ++ JsonDestroy(json); ++ ++ data = "[]\n"; ++ json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); ++ JsonDestroy(json); ++ ++ data = " 12 "; ++ json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); ++ JsonDestroy(json); ++ ++ data = "\"s\""; ++ json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); ++ JsonDestroy(json); ++} ++ ++static void test_parse_strict_duplicate_member(void) ++{ ++ /* Top level, nested, in an object inside an array, unquoted names, and ++ * a quoted name against an unquoted one. */ ++ assert_strict_refused( ++ "{\"a\": \"1\", \"a\": \"2\"}", ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_strict_refused( ++ "{\"vars\": {\"x\": \"1\", \"y\": \"2\", \"x\": \"3\"}}", ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_strict_refused( ++ "{\"l\": [{\"k\": 1}, {\"k\": 1, \"k\": 2}]}", ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_strict_refused( ++ "[[{\"a\": {\"b\": [], \"b\": []}}]]", ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_strict_refused( ++ "{a: 1, a: 2}", JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_strict_refused( ++ "{a: 1, \"a\": 2}", JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ ++ /* Whatever the value types, however far apart the two are. */ ++ assert_strict_refused( ++ "{\"a\": {}, \"b\": 1, \"a\": []}", ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_strict_refused( ++ "{\"a\": true, \"a\": true}", JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++} ++ ++static void test_parse_strict_invalid_earlier_duplicate(void) ++{ ++ /* The lenient parser drops the null before anything can look at it. */ ++ const char *const text = "{\"vars\": {\"good\": null, \"good\": \"ok\"}}"; ++ ++ assert_strict_refused(text, JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ ++ const char *data = text; ++ JsonElement *json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); ++ JsonElement *vars = JsonObjectGetAsObject(json, "vars"); ++ assert_int_equal(1, JsonLength(vars)); ++ assert_string_equal("ok", JsonObjectGetAsString(vars, "good")); ++ JsonDestroy(json); ++} ++ ++static void test_parse_strict_repeated_section(void) ++{ ++ const char *const text = ++ "{\"vars\": {\"a\": \"1\"}, \"classes\": {}," ++ " \"vars\": {\"b\": \"2\"}}"; ++ ++ assert_strict_refused(text, JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ ++ /* The lenient parser keeps only the last section. */ ++ const char *data = text; ++ JsonElement *json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); ++ JsonElement *vars = JsonObjectGetAsObject(json, "vars"); ++ assert_true(JsonObjectGet(vars, "a") == NULL); ++ assert_string_equal("2", JsonObjectGetAsString(vars, "b")); ++ JsonDestroy(json); ++} ++ ++static void test_parse_strict_trailing_characters(void) ++{ ++ assert_strict_refused( ++ "{\"vars\": {}} not-json", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("{} {}", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("{}}", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("{}\n\n x", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("[] ,", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("12 x", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("true false", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("null x", JSON_PARSE_ERROR_INVALID_END); ++ assert_strict_refused("\"s\" \"t\"", JSON_PARSE_ERROR_INVALID_END); ++ ++ /* An ordinary syntax error is still its own error. */ ++ assert_strict_refused("{\"a\": ", JSON_PARSE_ERROR_OBJECT_END); ++ assert_strict_refused("", JSON_PARSE_ERROR_NO_DATA); ++ assert_strict_refused(" \n", JSON_PARSE_ERROR_NO_DATA); ++ ++ /* The lenient functions accept the valid prefix, as before. */ ++ const char *data = "{\"vars\": {}} not-json"; ++ JsonElement *json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); ++ assert_true(json != NULL); ++ JsonDestroy(json); ++ ++ data = "{\"vars\": {}} not-json"; ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_INVALID_END, JsonParseAll(&data, &json)); ++ assert_true(json == NULL); ++} ++ ++static void test_parse_lenient_duplicate_last_wins(void) ++{ ++ const char *const text = ++ "{\"a\": \"1\", \"b\": {\"x\": 1, \"x\": 2}, \"a\": \"3\"," ++ " c: 1, c: 4}"; ++ ++ const char *data = text; ++ JsonElement *json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); ++ assert_int_equal(3, JsonLength(json)); ++ assert_string_equal("3", JsonObjectGetAsString(json, "a")); ++ assert_int_equal(1, JsonLength(JsonObjectGetAsObject(json, "b"))); ++ JsonDestroy(json); ++ ++ data = text; ++ json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseAll(&data, &json)); ++ assert_string_equal("3", JsonObjectGetAsString(json, "a")); ++ JsonDestroy(json); ++ ++ data = text; ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_OK, JsonParseWithLookup(NULL, NULL, &data, &json)); ++ assert_string_equal("3", JsonObjectGetAsString(json, "a")); ++ JsonDestroy(json); ++} ++ ++static void test_parse_strict_error_string(void) ++{ ++ const char *const message = ++ JsonParseErrorToString(JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); ++ assert_true(message != NULL); ++ assert_true(strstr(message, "duplicate") != NULL); ++} ++ ++static void write_strict_test_file( ++ char *const filename, const char *const content, const size_t length) ++{ ++ const int fd = mkstemp(filename); ++ assert_true(fd >= 0); ++ assert_int_equal(length, FullWrite(fd, content, length)); ++ close(fd); ++} ++ ++static JsonParseError strict_file( ++ const char *const content, ++ const size_t length, ++ const size_t size_max, ++ JsonElement **const json_out) ++{ ++ char filename[] = "/tmp/json_test_strict_XXXXXX"; ++ write_strict_test_file(filename, content, length); ++ const JsonParseError error = ++ JsonParseFileStrict(filename, size_max, json_out); ++ unlink(filename); ++ return error; ++} ++ ++#define STRICT_FILE(text, size_max, json_out) \ ++ strict_file(text, sizeof(text) - 1, size_max, json_out) ++ ++static void test_parse_file_strict(void) ++{ ++ JsonElement *json = NULL; ++ assert_int_equal( ++ JSON_PARSE_OK, ++ STRICT_FILE("{\"vars\": {\"a\": \"1\"}}\n", 1024, &json)); ++ assert_string_equal( ++ "1", JsonObjectGetAsString(JsonObjectGetAsObject(json, "vars"), "a")); ++ JsonDestroy(json); ++ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY, ++ STRICT_FILE("{\"vars\": {\"a\": null, \"a\": \"1\"}}", 1024, &json)); ++ assert_true(json == NULL); ++ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY, ++ STRICT_FILE("{\"vars\": {}, \"vars\": {}}", 1024, &json)); ++ assert_true(json == NULL); ++ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_INVALID_END, ++ STRICT_FILE("{\"vars\": {}} not-json\n", 1024, &json)); ++ assert_true(json == NULL); ++ ++ /* A null byte must not hide what follows it from the parser. */ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_INVALID_END, ++ STRICT_FILE("{\"vars\": {}}\0not-json", 1024, &json)); ++ assert_true(json == NULL); ++ ++ /* The size cap is kept: a larger file is refused, not cut short. */ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_TRUNCATED, ++ STRICT_FILE("{\"vars\": {\"a\": \"1\"}}", 8, &json)); ++ assert_true(json == NULL); ++ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_ERROR_NO_SUCH_FILE, ++ JsonParseFileStrict("/nonexistent/json_test_strict", 1024, &json)); ++ assert_true(json == NULL); ++ ++ json = NULL; ++ assert_int_equal(JSON_PARSE_ERROR_NO_DATA, STRICT_FILE("", 1024, &json)); ++ assert_true(json == NULL); ++} ++ ++static void test_parse_file_lenient_unchanged(void) ++{ ++ /* JsonParseFile() still repairs both, as before. */ ++ char filename[] = "/tmp/json_test_lenient_XXXXXX"; ++ const char content[] = "{\"a\": \"1\", \"a\": \"2\"} not-json\n"; ++ write_strict_test_file(filename, content, sizeof(content) - 1); ++ ++ JsonElement *json = NULL; ++ assert_int_equal(JSON_PARSE_OK, JsonParseFile(filename, 1024, &json)); ++ assert_string_equal("2", JsonObjectGetAsString(json, "a")); ++ JsonDestroy(json); ++ ++ json = NULL; ++ assert_int_equal( ++ JSON_PARSE_OK, JsonParseAnyFile(filename, 1024, &json, false)); ++ assert_string_equal("2", JsonObjectGetAsString(json, "a")); ++ JsonDestroy(json); ++ ++ unlink(filename); ++} ++ + static void test_detach_key_from_object(void) + { + JsonElement *object = JsonObjectCreate(3); +@@ -2491,6 +2773,15 @@ int main() + { + PRINT_TEST_BANNER(); + const UnitTest tests[] = { ++ unit_test(test_parse_strict_valid), ++ unit_test(test_parse_strict_duplicate_member), ++ unit_test(test_parse_strict_invalid_earlier_duplicate), ++ unit_test(test_parse_strict_repeated_section), ++ unit_test(test_parse_strict_trailing_characters), ++ unit_test(test_parse_lenient_duplicate_last_wins), ++ unit_test(test_parse_strict_error_string), ++ unit_test(test_parse_file_strict), ++ unit_test(test_parse_file_lenient_unchanged), + unit_test(test_array_get_string), + unit_test(test_array_iterator), + unit_test(test_array_remove_range), +-- +2.56.0 + diff --git a/overlay/libntech/0006-Fixed-JsonParseFileStrict-missing-a-null-byte-in-the.patch b/overlay/libntech/0006-Fixed-JsonParseFileStrict-missing-a-null-byte-in-the.patch new file mode 100644 index 0000000..d4c500d --- /dev/null +++ b/overlay/libntech/0006-Fixed-JsonParseFileStrict-missing-a-null-byte-in-the.patch @@ -0,0 +1,67 @@ +From 8a57e5558ef126ccd2f2edf5dd650418fef38724 Mon Sep 17 00:00:00 2001 +From: Daniel JB Clark +Date: Sat, 14 Nov 2026 15:49:29 -0500 +Subject: [PATCH 6/6] Fixed JsonParseFileStrict() missing a null byte in the + file + +FileRead() builds its Writer with strnlen(), so a null byte cuts the +text off there and the Writer's length still equals the length of the +string that is left: the check that compared the two could never fire, +and a file such as {"vars": {}}not-json parsed as the valid +object. The function now reads through FileReadFromFd() and compares the +number of bytes it got with the size fstat() reports for the same +descriptor, so a null byte (or a file that changed while it was read) is +refused with JSON_PARSE_ERROR_INVALID_END. + +Found by the unit tests in fork CI (test_parse_file_strict). + +Changelog: None +Ticket: none + +Co-Authored-By: Claude Sonnet 5.5 +--- + libutils/json.c | 19 +++++++++++++++---- + 1 file changed, 15 insertions(+), 4 deletions(-) + +diff --git a/libutils/json.c b/libutils/json.c +index 9a4b4f3..513dbdd 100644 +--- a/libutils/json.c ++++ b/libutils/json.c +@@ -3076,8 +3076,17 @@ JsonParseError JsonParseFileStrict( + assert(json_out != NULL); + *json_out = NULL; + ++ const int fd = safe_open(path, O_RDONLY); ++ if (fd == -1) ++ { ++ return JSON_PARSE_ERROR_NO_SUCH_FILE; ++ } ++ + bool truncated = false; +- Writer *contents = FileRead(path, size_max, &truncated); ++ Writer *const contents = FileReadFromFd(fd, size_max, &truncated); ++ struct stat sb; ++ const bool have_size = (fstat(fd, &sb) == 0); ++ close(fd); + if (contents == NULL) + { + return JSON_PARSE_ERROR_NO_SUCH_FILE; +@@ -3090,10 +3099,12 @@ JsonParseError JsonParseFileStrict( + + const char *data = StringWriterData(contents); + JsonParseError error; +- if (strlen(data) != StringWriterLength(contents)) ++ if (!have_size || (size_t) sb.st_size != StringWriterLength(contents)) + { +- /* A null byte would end the string early and hide the rest of +- * the file from the parser. */ ++ /* The Writer is a C string: a null byte ends it early and hides ++ * the rest of the file from the parser, so fewer bytes arrive than ++ * the file holds. A file that changed while it was read is refused ++ * too. */ + error = JSON_PARSE_ERROR_INVALID_END; + } + else +-- +2.56.0 + diff --git a/overlay/overlay.json b/overlay/overlay.json index b82f690..3e79a09 100644 --- a/overlay/overlay.json +++ b/overlay/overlay.json @@ -1,6 +1,7 @@ { "components": { "core": { + "applied_tree": "5c9f3ce18a6b290903b6610b58cb535638453822", "base": "a0bca6aaf310515f69fef60e6c56b3e60bd337ac", "base_described": "upstream master as of 2026-08-15 (merge of #6276)", "fork": "https://github.com/djbclark/core", @@ -19,30 +20,42 @@ "why": "B-5b is stacked on it; it names the offending CMDB entry and file in the rejection message" }, { - "acceptance_test": "tests/acceptance/00_basics/06_host_specific_data/16-variable-references-good-entry-survives.cf", - "commit": "8f0076b814e8a0d69e48baeab8e637926e4d111a", - "file": "0002-Fixed-one-bad-CMDB-entry-discarding-every-entry-in-i.patch", - "fork_issue": "https://github.com/djbclark/core/issues/9", - "fork_ref": "fix/cmdb-one-bad-entry-skips-only-that-entry", - "jira": "CFE-4720", + "commit": "4158906f2a5b3f202af1615fb80357a9a85987cc", + "file": "0002-Made-CMDB-loading-all-or-nothing-one-bad-entry-insta.patch", + "fork_ref": "overlay/cmdb-all-or-nothing", "load_bearing_for": [ - "host_specific.json is loaded by core's CMDB loader; one malformed entry used to drop its whole section" + "host_specific.json is loaded by core's CMDB loader, which now validates every entry first and installs nothing if any one is invalid (operator ruling R13): a skipped entry would leave the interlock it names, and the services it guards, ungated", + "the loader's null validation relies on B-20's null guards (87490bb8f), which here refuse the whole file instead of skipping the entry" ], "register": "B-5b", + "registers": [ + "B-5b (replaced: this is the all-or-nothing loader, not 8f0076b81's per-entry skip)", + "B-7 (passenger: dotted-key warning, ee1f9d60c)", + "B-20 (load-bearing: its null guards, as a refusal of the file; 87490bb8f)" + ], + "requires": "libntech patches 0005 and 0006 (B-24): the loader parses host_specific.json with JsonParseFileStrict(). Known difference from the pinned fork (open with djbclark): libntech B-4/B-10 and core's rlist.c number fix are not in the overlay, so a top-level CMDB scalar or primitive array holding 1e5 or an integer past long makes cf-agent exit in the install step (StringToLongExitOnError), and a real installs as %.2f; the projector refuses floats and writes no exponents, so only a hand-edited file reaches it (upstream-overlay.md section 7)", "retire_when": "an upstream commit carries the same change (`bin/overlay.py retired` says RETIRE), or the code it protects is no longer shipped", "role": "load-bearing", - "subject": "Fixed one bad CMDB entry discarding every entry in its section", - "superseded_on_fork": "2026-11-02, operator ruling R13 (PR #41): the fork's tendcf-integration branch, which overlay/fork.json pins, replaces this patch's per-entry skip with an all-or-nothing loader (branch cmdb-fail-closed: a750b1dfe, dbf9dede6, 61065e551, 736becc07; merged d32cade13, hardened by djbclark/core PR #98 and a strict libntech parse; test seam under TEST_UNIT_TEST by fork PR 99, pin 4443504). A skipped interlock entry leaves the services it guards unguarded. This entry still describes, and apply reproduces, commit 8f0076b81 exactly, so upstream plus the overlay skips a bad entry and does NOT behave like the pinned fork. Operator ruling 2026-11-14: Replace. A later, separate PR makes the overlay carry the all-or-nothing loader instead of this per-entry skip, because that changes what the overlay reproduces (upstream-overlay.md section 7); this PR leaves the patch and its manifest entry as they are.", - "upstream_pr": "https://github.com/cfengine/core/pull/6320", - "upstream_pr_state": "closed unmerged 2026-08-19" + "subject": "Made CMDB loading all-or-nothing: one bad entry installs nothing", + "tests": [ + "tests/unit/cmdb_test.c", + "tests/acceptance/00_basics/06_host_specific_data/16-variable-references-bad-entry-installs-nothing.cf", + "tests/acceptance/00_basics/06_host_specific_data/17-duplicate-member-installs-nothing.cf", + "tests/acceptance/00_basics/06_host_specific_data/18-trailing-input-installs-nothing.cf", + "tests/acceptance/00_basics/06_host_specific_data/13-null-values.cf", + "tests/acceptance/00_basics/06_host_specific_data/14-dotted-keys.cf" + ], + "upstream_pr": "none: B-5b's per-entry skip was cfengine/core#6320 (closed unmerged 2026-08-19); the all-or-nothing loader has no upstream PR yet", + "why": "replaces 8f0076b81 (B-5b's per-entry skip) per the 2026-11-14 operator ruling Replace; a squash of the fork's tendcf-integration loader (djbclark/core 4443504) onto the overlay base plus B-5a, with EvalContextVariableGetPlaintext() spelled EvalContextVariableGet() because the base has no secrets tracking (upstream-overlay.md section 7, risk R1)" } ], "upstream": "https://github.com/cfengine/core" }, "libntech": { + "applied_tree": "8d5fda1f9938c6b69bb9bb4d8a6e3be6c174f9e0", "base": "0c0620d6c5f8f8d1cae212f084b021cff0b86ce6", "base_described": "upstream master as of 2026-08-15 (merge of #289); exactly the libntech submodule pin of core's base a0bca6aaf, so the two components are one coherent tree", - "build_note": "after `apply`, point the core checkout's libntech submodule at the applied libntech HEAD (tree 5cbd6f7425fa, identical to the local merge 45c816c)", + "build_note": "after `apply`, point the core checkout's libntech submodule at the applied libntech HEAD (tree 8d5fda1f9938 with all six patches; 0001-0004 alone give 5cbd6f7425fa, identical to the local merge 45c816c)", "fork": "https://github.com/djbclark/libntech", "patches": [ { @@ -101,6 +114,32 @@ "subject": "Fixed a log-truncation format bug and an OOB pointer construction", "upstream_pr": "https://github.com/NorthernTechHQ/libntech/pull/298", "upstream_pr_state": "closed unmerged 2026-08-19" + }, + { + "commit": "dc3fba212bd323358c4de951acdaae3b182a4ef8", + "file": "0005-Added-JsonParseStrict-and-JsonParseFileStrict.patch", + "fork_ref": "fix/json-strict-parse", + "load_bearing_for": [ + "core's CMDB loader (overlay/core 0002, B-5b replaced) reads host_specific.json through JsonParseFileStrict(): a repeated object member or non-whitespace after the document refuses the whole file" + ], + "register": "B-24", + "retire_when": "an upstream commit carries the same change (`bin/overlay.py retired` says RETIRE), or the code it protects is no longer shipped", + "role": "load-bearing", + "subject": "Added JsonParseStrict() and JsonParseFileStrict()", + "upstream_pr": "none: fork PR https://github.com/djbclark/libntech/pull/11 only" + }, + { + "commit": "8a57e5558ef126ccd2f2edf5dd650418fef38724", + "file": "0006-Fixed-JsonParseFileStrict-missing-a-null-byte-in-the.patch", + "fork_ref": "fix/json-strict-parse", + "load_bearing_for": [ + "fixes JsonParseFileStrict() (0005), which the CMDB loader calls: a file with an embedded null byte, such as {\"vars\": {}}not-json, parsed as the valid object; it is refused now" + ], + "register": "B-24", + "retire_when": "an upstream commit carries the same change (`bin/overlay.py retired` says RETIRE), or the code it protects is no longer shipped", + "role": "load-bearing", + "subject": "Fixed JsonParseFileStrict() missing a null byte in the file", + "upstream_pr": "none: fork PR https://github.com/djbclark/libntech/pull/11 only" } ], "upstream": "https://github.com/NorthernTechHQ/libntech" diff --git a/tests/policy/run.sh b/tests/policy/run.sh index d19822a..5c17077 100755 --- a/tests/policy/run.sh +++ b/tests/policy/run.sh @@ -26,7 +26,9 @@ # 2026-11-14) each fork_only assertion prints `SKIP (loader not all-or-nothing): # ` and the run is expected to pass with those SKIP lines and a final # count, never a silent pass. The Linux CI legs set TENDCF_REQUIRE_FORK_LOADER=1, -# which turns a non-fork loader (so, any skip) into a failure. The non-fork +# which turns a non-fork loader (so, any skip) into a failure; so does the +# overlay-build leg, which runs this script on upstream plus overlay/ (its +# loader, overlay/core/0002, is the fork's, so check 4b passes unchanged). The non-fork # outcome has been derived from the parser behaviour, not yet run on the Mac. # Checks: # @@ -287,7 +289,7 @@ no_render_start() { ! grep -Eq "Created file|launchctl|systemctl" "$1"; } # evaluation, and CI shows the policy's own refusal after the strict loader's # on the fork (Linux legs, run 43649349555). The separate probe policy # (probe()) runs a second process and proves only what the data loaded as. -policy_completed() { grep -q "tendcf-test-done: the entry policy evaluated tendcf_interlocks and tendcf_services" "$1"; } +policy_completed() { grep -q "^R: tendcf-test-done: the entry policy evaluated tendcf_interlocks and tendcf_services" "$1"; } # Lenient-path assertions apply only where the loader is NOT the fork's (a # repeated member, a repeated `vars` or trailing input is read leniently and # the data it keeps is acted on). Not a skip: on the fork the same file is @@ -955,8 +957,8 @@ strict_case() { # strict_case NAME FIXTURE LOADER-ERROR-PATTERN INSTALL-NONE PO cp "$repo/tests/policy/fixtures/$fixture" "$work/data/host_specific.json" "$cf_agent" -K -I -f "$work/inputs/entry.cf" >"$out" 2>&1 || true # Same output file, any loader: this run evaluated both tendcf bundles (the - # test-only marker bundle runs after them). The negative greps below, which - # are the only assertions left on a non-fork run, mean nothing without it. + # test-only marker bundle runs after them). The marker ensures the negative + # greps below cannot pass on an invocation that never reached the policy. check "$name: this run evaluated the policy to the end (test-only marker)" policy_completed "$out" "$pcheck" "$name: the policy refuses and reports it" refusal_reported "$out" "$pcheck" "$name: the policy reports it once" refusal_once "$out" diff --git a/tests/policy/sim-loader.cf b/tests/policy/sim-loader.cf index 273dd08..cd5fe3e 100644 --- a/tests/policy/sim-loader.cf +++ b/tests/policy/sim-loader.cf @@ -2,8 +2,9 @@ # skipped, or half installed, a container of host_specific.json. No loader the # tests run does that: the fork at the pin (overlay/fork.json) is all-or-nothing # (a file it finds invalid, by the reasons listed in policy/tendcf_services.cf -# tendcf_cmdb, installs nothing, since PR #41, operator ruling R13) and the Mac -# dev build rejects the whole `vars`. The fork before R13 +# tendcf_cmdb, installs nothing, since PR #41, operator ruling R13), so is the +# upstream-plus-overlay build of the overlay-build job (overlay/core/0002 replaced +# the per-entry skip on 2026-11-14), and the Mac dev build rejects the whole `vars`. The fork before R13 # (`8f0076b81`, B-5b) did skip one bad entry, which was the 2026-11-02 # fail-open; this stand-in keeps that state testable, and the cross-check # must refuse it however a loader came to leave it. Run