From 2315b0027943959eb7db7e23b97084bd804ef147 Mon Sep 17 00:00:00 2001 From: Marco Ciavarella Date: Thu, 10 Sep 2026 20:06:57 +0200 Subject: [PATCH 1/5] Introduce support for mutations: planted defects for a model's tests to catch --- packages/dex-core/src/exmergo_dex_core/cli.py | 17 +- .../dex-core/src/exmergo_dex_core/engine.py | 25 + .../src/exmergo_dex_core/sql_shape.py | 35 +- .../src/exmergo_dex_core/transform/build.py | 264 +++++- .../exmergo_dex_core/transform/commands.py | 838 ++++++++++++++++-- .../exmergo_dex_core/transform/mutation.py | 812 +++++++++++++++++ .../src/exmergo_dex_core/transform/results.py | 40 + .../transform/row_attribution.py | 13 +- 8 files changed, 1920 insertions(+), 124 deletions(-) create mode 100644 packages/dex-core/src/exmergo_dex_core/transform/mutation.py diff --git a/packages/dex-core/src/exmergo_dex_core/cli.py b/packages/dex-core/src/exmergo_dex_core/cli.py index 63c88382..76032465 100644 --- a/packages/dex-core/src/exmergo_dex_core/cli.py +++ b/packages/dex-core/src/exmergo_dex_core/cli.py @@ -573,10 +573,19 @@ def _build_parser() -> argparse.ArgumentParser: # calling one if a caller can ask for it cheaply and disagree. sp.add_argument("--explain", action="store_true", default=False) if group == "transform" and name == "test": - # `test` is scaffold-only for now: the model to derive a - # unit_tests: skeleton from. No bare `transform test` - # mode exists yet, unlike `macro`'s list-when-bare shape. - sp.add_argument("--scaffold", default=None) + # Two modes, and they are opposites: `--scaffold` writes a + # unit test, `--mutate` measures the tests that already + # exist. Mutually exclusive rather than ordered, because + # asking for both in one call names no coherent outcome. No + # bare `transform test` mode exists, unlike `macro`'s + # list-when-bare shape. + mode = sp.add_mutually_exclusive_group() + mode.add_argument("--scaffold", default=None) + mode.add_argument("--mutate", default=None) + # Only ever narrows: the engine ceiling is what keeps a run + # that invokes dbt once per mutant predictable. + sp.add_argument("--max-mutants", type=int, default=None) + sp.add_argument("--target", default=None) if group == "semantic" and name in {"define", "update", "plan"}: sp.add_argument("argument", nargs="?", default=None) sp.add_argument("--edits-file", default=None) diff --git a/packages/dex-core/src/exmergo_dex_core/engine.py b/packages/dex-core/src/exmergo_dex_core/engine.py index 45af71e0..0fd4b279 100644 --- a/packages/dex-core/src/exmergo_dex_core/engine.py +++ b/packages/dex-core/src/exmergo_dex_core/engine.py @@ -93,6 +93,7 @@ InitResult, MacroListResult, MacroResult, + MutationCoverageResult, PlacementResult, PlanExportResult, PlanListResult, @@ -1333,6 +1334,30 @@ def build( dependencies=dependencies or DependencyPolicy.INSTALL, ) + def test_mutations( + self, + model: str, + *, + max_mutants: int | None = None, + target: str | None = None, + ) -> MutationCoverageResult: + """Measure a model's tests by planting defects and seeing what they catch. + + Answers what a green suite cannot: whether the tests would notice if the + model were wrong. Each mutant is one standard analytics defect, built in + a throwaway copy as an ephemeral model, so nothing is written to the + project or materialized in the warehouse. Dev target only, and on a + billed connector the whole batch is priced and confirmed as one number. + + ``max_mutants`` may only narrow the engine's ceiling, never raise it. + """ + + from .transform import commands as transform + + return transform.test_mutations( + self, model, max_mutants=max_mutants, target=target + ) + def deps(self) -> DepsResult: from .transform import commands as transform diff --git a/packages/dex-core/src/exmergo_dex_core/sql_shape.py b/packages/dex-core/src/exmergo_dex_core/sql_shape.py index a3ba03ce..cc28e817 100644 --- a/packages/dex-core/src/exmergo_dex_core/sql_shape.py +++ b/packages/dex-core/src/exmergo_dex_core/sql_shape.py @@ -14,11 +14,18 @@ a WITH has to accept either. Spread across call sites that is a rename waiting to be half-applied; here it is absorbed once. -Everything is a pure read of a parsed tree: nothing mutates, nothing executes, -nothing opens a connection. The module imports sqlglot at the top, so a caller -that must survive its absence (the base install carries no dialect engine) -should reach it behind :func:`~.guards.dialect.ensure_available` and degrade on -the refusal rather than importing unconditionally. +Almost everything here is a pure read of a parsed tree. The one exception is +:func:`set_predicates`, which is the inverse of :func:`predicates`: the reader +flattens a WHERE across its top-level ANDs, and rebuilding the clause from a +flattened list is the operation that undoes it. The two belong together, because +a caller that splits a clause one way and reassembles it another produces a +statement neither function describes. + +Nothing here executes anything or opens a connection. The module imports sqlglot +at the top, so a caller that must survive its absence (the base install carries +no dialect engine) should reach it behind +:func:`~.guards.dialect.ensure_available` and degrade on the refusal rather than +importing unconditionally. """ from __future__ import annotations @@ -77,6 +84,24 @@ def predicates(select: exp.Select, key: str) -> list[exp.Expression]: return flat +def set_predicates(select: exp.Select, key: str, preds: list[exp.Expression]) -> None: + """Rebuild a WHERE/HAVING/QUALIFY clause from flattened predicates, in place. + + The inverse of :func:`predicates`. An empty list removes the clause outright + rather than leaving an empty wrapper, because a ``WHERE`` with nothing under + it is not something a generator can print. + """ + + if not preds: + select.set(key, None) + return + condition = preds[0] + for extra in preds[1:]: + condition = exp.And(this=condition, expression=extra) + wrapper = {"where": exp.Where, "having": exp.Having, "qualify": exp.Qualify}[key] + select.set(key, wrapper(this=condition)) + + def text(node: exp.Expression | None) -> str: """What a fragment says, normalized by sqlglot's own generator and no further. diff --git a/packages/dex-core/src/exmergo_dex_core/transform/build.py b/packages/dex-core/src/exmergo_dex_core/transform/build.py index 9cc7b3ab..f48d74a5 100644 --- a/packages/dex-core/src/exmergo_dex_core/transform/build.py +++ b/packages/dex-core/src/exmergo_dex_core/transform/build.py @@ -14,6 +14,7 @@ from __future__ import annotations +import contextlib import json import os import re @@ -21,7 +22,7 @@ import subprocess import sys import tempfile -from collections.abc import Callable +from collections.abc import Callable, Iterator, Sequence from enum import Enum from pathlib import Path from typing import Any @@ -245,6 +246,47 @@ def build( return summary, cost +@contextlib.contextmanager +def shadow_project( + project_dir: Path | str, edits: Sequence[Edit] = () +) -> Iterator[Path]: + """A throwaway copy of the project with ``edits`` overlaid, yielded by path. + + The copy is what lets dex run dbt against a hypothetical version of a project + without touching the real one: everything dbt writes into it (``target/``, + ``logs/``, any stray database a relative profile path would create) lives and + dies with the copy. + + ``dbt_packages/`` is deliberately copied, because parsing needs the installed + macros. Warehouse files deliberately are not: they can be huge, and a copy of + a database is not the database, so a caller that needs the real warehouse must + reach it through the profile rather than through the tree. ``.dex`` matters + when the project is the repo root. + """ + + project = Path(project_dir).resolve() + view = load_project(project) + with tempfile.TemporaryDirectory(prefix="dex-shadow-") as tmp: + shadow = Path(tmp) / (project.name or "project") + shutil.copytree( + project, + shadow, + ignore=shutil.ignore_patterns( + "target", "logs", ".git", ".venv", ".dex", "*.duckdb", "*.db" + ), + ) + for edit in edits: + edit_path = contained_path(shadow, edit.path, view) + if edit.op is EditOp.DELETE: + # Remove it from the copy so the parse runs against the true + # post-deletion tree: a surviving ref() to it fails dbt's parse. + edit_path.unlink(missing_ok=True) + else: + edit_path.parent.mkdir(parents=True, exist_ok=True) + edit_path.write_text(edit.new_content, encoding="utf-8") + yield shadow + + def shadow_parse( project_dir: Path | str, edits: list[Edit], @@ -293,29 +335,7 @@ def shadow_parse( "messages": [], } - view = load_project(project) - with tempfile.TemporaryDirectory(prefix="dex-shadow-") as tmp: - shadow = Path(tmp) / (project.resolve().name or "project") - # dbt_packages/ is deliberately copied (parse needs installed macros); - # warehouse files are deliberately not (parse never reads them, and - # they can be huge). `.dex` matters when the project is the repo root. - shutil.copytree( - project, - shadow, - ignore=shutil.ignore_patterns( - "target", "logs", ".git", ".venv", ".dex", "*.duckdb", "*.db" - ), - ) - for edit in edits: - edit_path = contained_path(shadow, edit.path, view) - if edit.op is EditOp.DELETE: - # Remove it from the copy so the parse runs against the true - # post-deletion tree: a surviving ref() to it fails dbt's parse. - edit_path.unlink(missing_ok=True) - else: - edit_path.parent.mkdir(parents=True, exist_ok=True) - edit_path.write_text(edit.new_content, encoding="utf-8") - + with shadow_project(project, edits) as shadow: # A profiles.yml edit only takes effect if dbt reads the shadowed copy; # pointing --profiles-dir at the real project would parse the edit # against the unedited profile. Redirect to the shadow when the @@ -352,6 +372,202 @@ def shadow_parse( return {"available": True, "reason": None, "success": success, "messages": messages} +# dbt reads these from the environment, and each one can silently change which +# tests run, what a status means, or where dbt writes. A mutation run has to mean +# the same thing on every machine, and the one that matters most is +# `DBT_INDIRECT_SELECTION`: set to `cautious` it drops the tests from the +# selection, every mutant then survives, and the report says the suite is weak +# when in fact it was never asked. +_ISOLATED_ENV_SCRUBBED = ( + "DBT_TARGET_PATH", + "DBT_LOG_PATH", + "DBT_STATE", + "DBT_DEFER_STATE", + "DBT_STORE_FAILURES", + "DBT_WARN_ERROR", + "DBT_WARN_ERROR_OPTIONS", + "DBT_RESOURCE_TYPES", + "DBT_EXCLUDE_RESOURCE_TYPES", + "DBT_SELECTOR", + "DBT_EMPTY", + "DBT_FULL_REFRESH", + "DBT_INDIRECT_SELECTION", + "DBT_DEFER", + "DBT_FAVOR_STATE", + "DBT_FAIL_FAST", +) + + +class ShadowRun: + """A copied project that dbt can be run against many times, never the real one. + + :func:`shadow_parse` copies a project to parse it once. This is the same + isolation held open across a sequence of invocations, which is what mutation + coverage needs: one copy, then a compile and N test runs against it, each + with a different version of one model's file. + + Stateful by nature, hence a class rather than a function taking the same six + arguments each call: it owns the copy's lifetime, and dbt's partial-parse + cache inside the copy is what keeps run N+1 from re-parsing the whole project. + + **cwd stays at the real project, and every artifact path is passed as a flag.** + That split is deliberate and it is the only arrangement that is safe on every + connector. dbt resolves ``target/`` and ``logs/`` against ``--project-dir``, + so the flags keep its writes inside the copy; dbt-duckdb resolves a relative + ``path:`` in the profile against the *process* cwd, so leaving cwd at the real + project is what lets a copied project still reach the real dev database. + Copying or linking the database instead would split it from its + write-ahead log, which risks the user's data rather than merely confusing dbt. + """ + + def __init__( + self, + project_dir: Path | str, + *, + target: str, + connector: str | None = None, + paradigm: Paradigm = Paradigm.FREE_LOCAL, + ceiling: float | None = None, + runner: Runner | None = None, + timeout: float = _DBT_TIMEOUT_SECONDS, + ): + self.project = Path(project_dir).resolve() + self.target = target + self._connector = connector + self._paradigm = paradigm + self._ceiling = ceiling + self._runner = runner + self._timeout = timeout + self._stack = contextlib.ExitStack() + self.shadow: Path | None = None + self.view = None + + def __enter__(self) -> ShadowRun: + self.view = load_project(self.project) + self.shadow = self._stack.enter_context(shadow_project(self.project)) + return self + + def __exit__(self, *exc: object) -> None: + self._stack.close() + self.shadow = None + + def write(self, rel_path: str, text: str) -> None: + """Put a file into the copy, confined to the project's editing surface.""" + + path = contained_path(self._require_shadow(), rel_path, self.view) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def strip_run_hooks(self) -> bool: + """Drop ``on-run-start`` and ``on-run-end`` from the copy. True if any went. + + A hook fires once per invocation, and this runs dbt once per mutant, so a + project whose hooks grant permissions or write an audit row would do that + N+1 times for a command the user thinks of as read-only. + """ + + manifest = self._require_shadow() / "dbt_project.yml" + if not manifest.is_file(): + return False + parsed = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {} + present = [key for key in ("on-run-start", "on-run-end") if key in parsed] + if not present: + return False + for key in present: + parsed.pop(key) + manifest.write_text(yaml.safe_dump(parsed, sort_keys=False), encoding="utf-8") + return True + + def manifest(self) -> dict[str, Any]: + path = self._require_shadow() / "target" / "manifest.json" + if not path.is_file(): + raise DbtRunError("dbt wrote no manifest for the copied project") + return json.loads(path.read_text(encoding="utf-8")) + + def compile(self, select: str) -> dict[str, Any]: + """Compile one selection in the copy and return the manifest dbt wrote.""" + + completed = self._invoke("compile", "--select", select) + if completed.returncode != 0: + messages = _collect_messages(completed) + raise DbtRunError(messages[0] if messages else "dbt compile failed") + return self.manifest() + + def test( + self, select: str, *, exclude: Sequence[str] = () + ) -> dict[str, Any] | None: + """Run one selection's tests. ``None`` when dbt wrote no results at all. + + ``dbt test``, never ``dbt build``. Two reasons, and both are load-bearing. + A build runs a model's unit tests before the model, so one failing unit + test marks the model skipped and the skip cascades onto every data test + attached to it: the run would then report four tests as skipped and the + caller could not tell which of them would have caught the defect. And + ``dbt test`` executes no model materialization at all, so even a config + override that failed to apply could not write a relation. + """ + + args = ["--select", select] + for name in exclude: + args += ["--exclude", name] + completed = self._invoke("test", *args) + results = self._require_shadow() / "target" / "run_results.json" + if not results.is_file(): + return None + return _summarize(self._require_shadow(), self.target, completed) + + def _invoke(self, verb: str, *args: str) -> subprocess.CompletedProcess: + shadow = self._require_shadow() + # Cleared first: dbt writes this as part of running, so a leftover from + # the previous mutant would otherwise be read as this one's answer. + (shadow / "target" / "run_results.json").unlink(missing_ok=True) + argv = [ + _dbt_executable(), + verb, + "--target", + self.target, + "--project-dir", + str(shadow), + "--profiles-dir", + str(profiles_dir(self.project).resolve()), + "--target-path", + str(shadow / "target"), + "--log-path", + str(shadow / "logs"), + "--log-format", + "json", + # Pinned rather than inherited, for the reason the environment is + # scrubbed: each of these changes what a run means. + "--indirect-selection", + "eager", + "--no-defer", + "--no-favor-state", + "--no-fail-fast", + *args, + ] + run = self._runner or _default_runner( + self._timeout, self.project, env=self._env() + ) + return run(argv) + + def _env(self) -> dict[str, str]: + env = { + key: value + for key, value in os.environ.items() + if key not in _ISOLATED_ENV_SCRUBBED + } + # dbt writes a `.user.yml` into the profiles directory when usage + # tracking is on, and the profiles directory here is the real project. + env["DO_NOT_TRACK"] = "1" + env.update(_build_env(self._connector, self._paradigm, self._ceiling) or {}) + return env + + def _require_shadow(self) -> Path: + if self.shadow is None: + raise DbtRunError("the shadow project is only open inside a `with` block") + return self.shadow + + def has_package_spec(project_dir: Path | str) -> bool: """True when the project declares dbt packages (packages.yml, or a dependencies.yml with a ``packages:`` key).""" diff --git a/packages/dex-core/src/exmergo_dex_core/transform/commands.py b/packages/dex-core/src/exmergo_dex_core/transform/commands.py index 13b4cce2..7429f5fd 100644 --- a/packages/dex-core/src/exmergo_dex_core/transform/commands.py +++ b/packages/dex-core/src/exmergo_dex_core/transform/commands.py @@ -53,6 +53,7 @@ InitResult, MacroListResult, MacroResult, + MutationCoverageResult, PlacementResult, PlanExportResult, PlanListResult, @@ -720,15 +721,633 @@ def test_scaffold(engine: DexEngine, model_name: str | None) -> TestScaffoldResu return TestScaffoldResult(**planned.model_dump(), model=model_name, inputs=inputs) +def test_mutations( + engine: DexEngine, + model: str | None, + *, + max_mutants: int | None = None, + target: str | None = None, +) -> MutationCoverageResult: + """Plant standard analytics defects in a model and report which tests miss them. + + The question this answers is the one a passing suite cannot: whether the + tests would notice if the model were wrong. Each mutant is one defect, built + in a throwaway copy as an ephemeral model so nothing is materialized, and run + through the model's own tests. + + The order mirrors `transform build`, and for the same reasons: the free + refusals come first, so a model dex cannot mutate costs nothing to find out + about; the dev-target check runs next, because a broken target makes every + run impossible and the caller should learn that before weighing a budget; + then the whole batch is priced and confirmed once. N runs behind one + handshake is the only shape that works here, since a per-mutant ask would + make the caller answer twenty times for one question. + """ + + from ..adapters import get_dialect + from ..connect import paradigm_for + from ..envelope import Paradigm + from ..guards.cost_guard import skipped_handshake_warning + from . import dev_target + from . import mutation as mutation_mod + from .build import ShadowRun, assert_dev_target + + if not model: + raise ValueError( + "transform test needs a model: `transform test --mutate `" + ) + if max_mutants is not None and max_mutants > mutation_mod.MAX_MUTANTS: + raise ValueError( + f"--max-mutants {max_mutants} is above the engine ceiling of " + f"{mutation_mod.MAX_MUTANTS}; every mutant is a dbt run, so the " + "ceiling is what keeps this command's cost predictable. Lower it " + "to narrow the run" + ) + cap = mutation_mod.MAX_MUTANTS if max_mutants is None else max(0, max_mutants) + + config = engine.config + store = engine.store + repo_root = engine.require_repo_root("running mutation coverage") + target = target or config.dbt_target or "dev" + assert_dev_target(target, config.dbt_target) + connector = engine.connector or config.connector + effective = config.model_copy(update={"connector": connector}) + paradigm = paradigm_for(connector, effective) + ceiling = engine.budget if engine.budget is not None else config.budget.ceiling + project = engine.project_dir() + + dev_warnings = dev_target.check( + project, + target, + effective, + repo_root, + store=store, + connection=engine.connection, + ) + + warnings: list[str] = list(dev_warnings) + adapter = None + gate = None + runs = 0 + spend: dict[str, float | None] | None = None + try: + with ShadowRun( + project, + target=target, + connector=connector, + paradigm=paradigm, + ceiling=ceiling, + ) as shadow: + if shadow.strip_run_hooks(): + warnings.append( + "this project's on-run-start/on-run-end hooks are not run: " + "mutation coverage invokes dbt once per mutant, and a hook " + "that grants or audits would otherwise fire once per run" + ) + node, model_path = _mutable_model(engine, shadow, model, mutation_mod) + prepared, batch = _plan_mutants( + shadow, node, model_path, cap, get_dialect(connector), mutation_mod + ) + warnings.extend(prepared.notes) + if batch.unparsed: + warnings.append( + f"{batch.unparsed} mutant(s) did not survive a re-read and " + "were dropped rather than run" + ) + if not batch.mutants: + raise mutation_mod.MutationError( + f"'{model}' has no SQL dex knows how to plant a defect in " + "(no comparison, filter, join, CASE, division, window frame " + "or aggregate), so there is nothing to measure its tests against" + ) + + estimate = None + if paradigm is not Paradigm.FREE_LOCAL: + adapter = engine._adapter("transform test") + estimate, per_mutant, price_notes = _price_mutations( + adapter, shadow, model_path, batch, node, mutation_mod + ) + warnings.extend(price_notes) + if estimate is not None: + command_args.billed_handshake( + "transform test", + adapter, + estimate, + per_table=per_mutant or None, + notes=price_notes or None, + ) + gate = command_args.cost_gate(adapter) + + result = _run_mutants( + shadow, + model_path, + batch, + model=node["name"], + paradigm=paradigm, + connector=connector, + store=store, + ceiling=ceiling, + estimate=estimate, + mutation_mod=mutation_mod, + ) + runs, spend, run_warnings = result.pop("_meta") + warnings.extend(run_warnings) + # Released before the day's total is read back, exactly as + # `_shape_build_result` releases before reading. Every run settles + # while this command still holds its whole-batch reservation, so a + # total read during the loop counts that reservation on top of what + # the runs actually billed and overstates the day by the estimate. + spend = _refresh_session_total(spend, gate, store, paradigm, connector) + finally: + if gate is not None: + gate.settle() + + if paradigm is Paradigm.FREE_LOCAL: + warnings.extend(skipped_handshake_warning(paradigm, engine.confirmed)) + + survivors = result["counts"].get("survived", 0) + if survivors: + warnings.append( + f"{survivors} planted defect(s) survived this model's tests: no test " + "told the mutant apart, on the dev data or on the unit test fixtures. " + "They are in data.mutants, each with the test that would catch it" + ) + if batch.elided_total: + warnings.append( + f"{batch.elided_total} further mutant(s) were not run: the cap is " + f"{batch.cap}, and what it cut is in data.cap.elided, per defect class" + ) + + return MutationCoverageResult( + model=node["name"], + target=target, + baseline=result["baseline"], + mutants=result["mutants"], + counts=result["counts"], + score=result["score"], + cap={ + "limit": batch.cap, + "generated": len(batch.mutants), + "considered": batch.considered, + "elided": batch.elided, + }, + runs=runs, + spend=spend, + cost=_mutation_cost(paradigm, estimate, ceiling, adapter), + warnings=warnings, + ) + + +def _mutation_cost(paradigm, estimate, ceiling, adapter): + from ..envelope import Cost + from ..guards.cost_guard import estimate_quality_of + + return Cost( + paradigm=paradigm, + estimate=estimate, + ceiling=ceiling, + estimate_quality=estimate_quality_of(adapter, estimate, paradigm=paradigm), + ) + + +def _mutable_model(engine, shadow, model: str, mutation_mod): + """The node to mutate, or the free refusal saying why this model is not one. + + Every check here is answered from the parse alone, before a connection is + opened or a mutant is priced, because a model dex cannot mutate should cost + nothing to ask about. + """ + + from ..dbt_project import load as load_project + + view = load_project(shadow.project) + original = _model_file(view, model) + if original is None: + raise mutation_mod.MutationError( + f"no model named '{model}' in this project's model paths" + ) + path, content = original + # The header is appended rather than prepended: dbt gives a scalar config key + # to the last `config()` call in a file, so a model declaring its own + # `materialized` would otherwise win and the mutant would build a relation. + shadow.write(path, content + "\n" + mutation_mod.EPHEMERAL_HEADER + "\n") + manifest = shadow.compile(model) + + matches = [ + node + for uid, node in manifest.get("nodes", {}).items() + if uid.startswith("model.") and node.get("name") == model + ] + if not matches: + raise mutation_mod.MutationError(f"dbt compiled no model named '{model}'") + node = matches[0] + if node.get("language") == "python": + raise mutation_mod.MutationError( + f"'{model}' is a Python model; mutation coverage plants defects in SQL" + ) + if node.get("package_name") != manifest.get("metadata", {}).get("project_name"): + raise mutation_mod.MutationError( + f"'{model}' belongs to an installed package rather than this project, " + "and dex does not mutate a package's own models" + ) + if (node.get("config") or {}).get("sql_header"): + raise mutation_mod.MutationError( + f"'{model}' declares a sql_header, which only a materialization emits; " + "an ephemeral mutant would run without it and the tests would fail for " + "that reason rather than for the defect" + ) + if (node.get("config") or {}).get("materialized") != "ephemeral": + # The override is what keeps every mutant from writing a relation, so a + # run must not proceed on a project where it silently did not apply. + raise mutation_mod.MutationError( + f"dex could not make '{model}' ephemeral for the run (it compiled as " + f"{(node.get('config') or {}).get('materialized')}), and it will not " + "run mutants that would materialize into your dev target" + ) + if not _attached_tests(manifest, node): + # Free, and the more useful answer than "nothing to mutate": with no test + # to catch anything, every mutant survives by construction and the run + # would spend N dbt invocations to say so. + raise mutation_mod.MutationError( + f"'{model}' has no tests, so there is nothing to measure: every " + "planted defect would survive by construction. Write one first " + f"(`transform test --scaffold {model}` scaffolds a unit test), " + "then measure it" + ) + return node, path + + +def _attached_tests(manifest: dict, node: dict) -> list[str]: + """Every test dbt would run for this model, generic, singular and unit alike. + + Read from the manifest rather than from a run, because it is the free answer + and it is what makes "this model has no tests" a refusal instead of N dbt + invocations that all report a survivor. + """ + + unique_id = node.get("unique_id") + attached = [ + uid + for uid, other in manifest.get("nodes", {}).items() + if other.get("resource_type") == "test" + and ( + other.get("attached_node") == unique_id + or unique_id in (other.get("depends_on") or {}).get("nodes", []) + ) + ] + attached += [ + uid + for uid, unit in (manifest.get("unit_tests") or {}).items() + if unique_id in (unit.get("depends_on") or {}).get("nodes", []) + or unit.get("model") == node.get("name") + ] + return attached + + +def _model_file(view, model: str): + """The model's path and content, read off the project view.""" + + for path, file in view.files.items(): + if not path.endswith(".sql"): + continue + if Path(path).stem == model and any( + path.startswith(str(Path(root))) for root in view.model_paths + ): + return path, file.content + return None + + +def _plan_mutants(shadow, node, model_path, cap, dialect, mutation_mod): + """The compiled model, references restored, and the defects to plant in it.""" + + manifest = shadow.manifest() + parents = [] + for uid in node.get("depends_on", {}).get("nodes", []): + parent = manifest.get("nodes", {}).get(uid) or manifest.get("sources", {}).get( + uid + ) + if parent is None: + continue + ephemeral = (parent.get("config") or {}).get("materialized") == "ephemeral" + rendered = ( + f"{mutation_mod.DBT_CTE_PREFIX}{parent['name']}" + if ephemeral + else parent.get("relation_name") + ) + if not rendered: + continue + parents.append( + mutation_mod.ParentRelation( + rendered=rendered, + jinja=_reference_jinja(parent, uid), + ephemeral=ephemeral, + ) + ) + prepared = mutation_mod.prepare( + node.get("compiled_code") or "", dialect=dialect, parents=parents + ) + return prepared, mutation_mod.enumerate_mutants(prepared, cap=cap) + + +def _reference_jinja(parent: dict, unique_id: str) -> str: + """How a dbt file has to name this input for a unit test fixture to bind.""" + + if unique_id.startswith("source."): + return f"{{{{ source('{parent['source_name']}', '{parent['name']}') }}}}" + if parent.get("version") is not None: + return f"{{{{ ref('{parent['name']}', v={parent['version']}) }}}}" + return f"{{{{ ref('{parent['name']}') }}}}" + + +def _price_mutations(adapter, shadow, model_path, batch, node, mutation_mod): + """Price the whole batch upfront, per mutant, as one number to confirm. + + Each mutant is priced as the statements the warehouse will actually run, + which means splicing it into each test's compiled SQL rather than pricing the + model alone: a mutant that drops a partition predicate scans more than the + model it came from, and pricing the batch at the baseline's cost would + under-report it. Under-reporting is the one direction a cost guard must never + round. + """ + + estimator = getattr(adapter, "query_estimate", None) + if estimator is None: + return None, {}, ["connector exposes no estimator; the batch is not priced"] + + manifest = shadow.manifest() + tests = [ + node.get("compiled_code") + for node in manifest.get("nodes", {}).values() + if node.get("resource_type") == "test" and node.get("compiled_code") + ] + notes: list[str] = [] + if not tests: + notes.append( + "only unit tests read this model, and a unit test runs against " + "fixtures rather than the warehouse, so the batch prices at zero" + ) + + def price(body: str) -> float: + total = 0.0 + for test_sql in tests: + spliced = mutation_mod.inline_into_test( + test_sql, model_name=node["name"], body=body, dialect=adapter.dialect + ) + with contextlib.suppress(Exception): + total += estimator(spliced if spliced is not None else test_sql) + return total + + per_mutant: dict[str, float] = {} + with contextlib.suppress(Exception): + per_mutant["(baseline)"] = price(batch.identity) + for mutant in batch.mutants: + with contextlib.suppress(Exception): + per_mutant[f"{mutant.id} {mutant.operator}"] = price(mutant.body) + if not per_mutant: + return None, {}, [*notes, "the batch could not be priced upfront"] + return sum(per_mutant.values()), per_mutant, notes + + +def _run_mutants( + shadow, + model_path, + batch, + *, + model: str, + paradigm, + connector: str, + store, + ceiling: float | None, + estimate: float | None, + mutation_mod, +): + """The baseline, then one run per mutant, stopping if the budget runs out.""" + + from ..envelope import Paradigm + + warnings: list[str] = [] + runs = 0 + spent = 0.0 + spend: dict[str, float | None] | None = None + + shadow.write(model_path, batch.identity) + baseline_summary = shadow.test(model) + runs += 1 + spend, _ = _settle_dbt_spend( + baseline_summary or {}, + [], + paradigm=paradigm, + connector=connector, + store=store, + estimate=None, + command="transform test", + ) + baseline = _test_statuses(baseline_summary) + excluded = [ + {"name": name, "status": status, "reason": _exclusion_reason(status)} + for name, status in sorted(baseline.items()) + if status != "pass" + ] + if not any(status == "pass" for status in baseline.values()): + raise mutation_mod.MutationError( + f"no test of '{model}' passes against the unmutated model, so there is " + "nothing that could catch a defect. Build its parents first " + f"(`transform build --select +{model}`) and fix the failing tests, " + "then measure them" + ) + + mutants: list[dict] = [] + counts = {"killed": 0, "survived": 0, "rejected": 0, "not_run": 0} + per_run = (estimate or 0.0) / max(len(batch.mutants) + 1, 1) + for mutant in batch.mutants: + if ( + paradigm is not Paradigm.FREE_LOCAL + and ceiling is not None + and spent + per_run > ceiling + ): + # An unknown settlement counts at its estimate, so the guard never + # rounds spend down on the way to deciding it can afford another run. + for remaining in batch.mutants[len(mutants) :]: + mutants.append({**remaining.payload(), "status": "not_run"}) + counts["not_run"] += 1 + warnings.append( + f"the confirmed budget covered {runs - 1} of {len(batch.mutants)} " + "mutants; the rest are reported as not_run. Re-run with a larger " + "--budget, or narrow the run with --max-mutants" + ) + break + shadow.write(model_path, mutant.body) + summary = shadow.test(model) + runs += 1 + run_spend, _ = _settle_dbt_spend( + summary or {}, + [], + paradigm=paradigm, + connector=connector, + store=store, + estimate=None, + command="transform test", + ) + spent += _spent_in_run(run_spend, paradigm, fallback=per_run) + spend = _merge_spend(spend, run_spend) + verdict = mutation_mod.classify(baseline, _test_statuses(summary)) + counts[verdict.outcome] = counts.get(verdict.outcome, 0) + 1 + mutants.append( + { + **mutant.payload(), + "status": verdict.outcome, + "caught_by": verdict.caught_by, + "warn_only": verdict.warn_only, + } + ) + + # Survivors first: the reader is deciding which test to write next, and the + # mutants that were caught are the ones they need to read least. + order = {"survived": 0, "not_run": 1, "rejected": 2, "killed": 3} + mutants.sort(key=lambda m: (order.get(m["status"], 9), m["id"])) + decided = counts["killed"] + counts["survived"] + return { + "baseline": { + "tests": [ + {"name": name, "status": status} + for name, status in sorted(baseline.items()) + ], + "excluded": excluded, + }, + "mutants": mutants, + "counts": {"generated": len(batch.mutants), **counts}, + "score": (counts["killed"] / decided) if decided else None, + "_meta": (runs, spend, warnings), + } + + +def _refresh_session_total(spend, gate, store, paradigm, connector): + """Settle the gate, then re-read the day's total the envelope will report. + + A build settles its gate before reading the ledger back for the same reason: + the reservation is headroom the command is holding, not spend it has + incurred, so a total read while it stands reports work nobody did. This + command holds one reservation across N runs, which makes the overstatement + exactly the whole batch estimate rather than a rounding difference. + """ + + from ..envelope import Paradigm + from ..guards.cost_guard import ledger_field, utc_day_start + + if gate is None or spend is None or paradigm is Paradigm.FREE_LOCAL: + return spend + gate.settle() + with contextlib.suppress(Exception): + spend = { + **spend, + "session_spent_today": store.spend_since( + utc_day_start(), field=ledger_field(paradigm), connector=connector + ), + } + return spend + + +def _test_statuses(summary: dict | None) -> dict[str, str]: + """Each test node's status, keyed by the name a caller would recognize.""" + + if not summary: + return {} + return { + node["name"]: node["status"] + for node in summary.get("nodes", []) + if str(node.get("unique_id", "")).startswith(("test.", "unit_test.")) + } + + +def _exclusion_reason(status: str) -> str: + if status == "error": + return ( + "this test could not run against the unmutated model, so it cannot " + "testify about a mutant either" + ) + return ( + "this test was already failing before anything was mutated; counting it " + "as a catch would report the suite as strong because it is broken" + ) + + +def _spent_in_run(spend: dict | None, paradigm, *, fallback: float) -> float: + from ..guards.cost_guard import spend_field + + if not spend: + return fallback + value = spend.get(spend_field(paradigm)) + # Unknown settlement counts at the estimate rather than at zero. + return fallback if value is None else float(value) + + +#: Spend keys that are a reading of the world rather than this command's own +#: contribution to it, so the newest answer replaces the previous one instead of +#: being added to it. Summing `session_spent_today` across nine runs reported a +#: day's total nine times over, which is the one direction a spend report must +#: not err in. +_SPEND_LATEST_WINS = {"session_spent_today", "reserved"} + + +def _merge_spend(running: dict | None, latest: dict | None) -> dict | None: + """Accumulate what a sequence of dbt runs billed into one command's spend. + + Three kinds of key and three rules, because a spend payload is not uniformly + additive. What each run billed sums. A reading of the day's cumulative total + is already cumulative, so the latest one wins. The two flags are claims about + the whole command: it settled only if every run did, and its settlement is + unknown if any run's was, which is why they are combined rather than added. + Booleans are integers in Python, so an additive rule silently turned + ``settled: true`` into ``settled: 9``. + """ + + if latest is None: + return running + if running is None: + return dict(latest) + merged = dict(running) + for key, value in latest.items(): + previous = merged.get(key) + if isinstance(value, bool) or isinstance(previous, bool): + if key == "unknown_settlement": + merged[key] = bool(previous) or bool(value) + else: + merged[key] = bool(previous) and bool(value) + elif key in _SPEND_LATEST_WINS: + merged[key] = value if value is not None else previous + elif isinstance(value, (int, float)) and isinstance(previous, (int, float)): + merged[key] = previous + value + elif previous is None: + merged[key] = value + return merged + + def cmd_test(args: argparse.Namespace, engine: DexEngine) -> env.Envelope: + from .mutation import MutationError from .test_scaffold import TestScaffoldError + scaffold = getattr(args, "scaffold", None) + mutate = getattr(args, "mutate", None) try: - result = test_scaffold(engine, getattr(args, "scaffold", None)) + if scaffold and mutate: + raise ValueError( + "--scaffold writes a unit test and --mutate measures the tests " + "that exist; run one, then the other" + ) + if mutate: + return to_envelope( + test_mutations( + engine, + mutate, + max_mutants=getattr(args, "max_mutants", None), + target=getattr(args, "target", None), + ) + ) + result = test_scaffold(engine, scaffold) return to_envelope(result, hints=plan_hint(result)) except DbtParseError as exc: return env.error_for(exc, warnings=exc.warnings) - except (ValueError, TestScaffoldError) as exc: + except (ValueError, TestScaffoldError, MutationError) as exc: return env.error_for(exc) @@ -1612,12 +2231,15 @@ def _record_build_spend( estimate: float | None = None, *, gate_billed: float = 0.0, + command: str = "transform build", ) -> dict[str, float | None]: - """Account a billed dbt build in the spend ledger and report what it cost. + """Account a billed dbt run in the spend ledger and report what it cost. - The paradigm names both units, so a build draws against the same session + The paradigm names both units, so a run draws against the same session budget as an explore scan and reports spend under the key every other - command reports it under. + command reports it under. ``command`` is what the ledger row is filed under, + which matters because more than one command now runs dbt: a reader summing + what `transform build` spent must not be handed another command's rows. ``billed`` is ``None`` only where dbt executed statements and reported no billing figure for any of them, which is unknown spend rather than none. @@ -1660,7 +2282,7 @@ def _record_build_spend( # settled outside any reservation". ledger_row( connector=connector, - command="transform build", + command=command, entry="settlement", field=field, amount=billed, @@ -1699,6 +2321,127 @@ def _record_build_spend( } +def _settle_dbt_spend( + summary: dict, + notes: list[str], + *, + paradigm, + connector: str, + store: Store, + estimate: float | None, + adapter=None, + gate_billed: float = 0.0, + command: str = "transform build", +) -> tuple[dict[str, float | None] | None, list[str]]: + """What one finished dbt run cost, ledgered, with the note that explains it. + + One function per paradigm's accounting rather than three scattered branches, + because every caller that runs dbt owes the same three things: read the + actual out of the artifact in that paradigm's unit, append a ledger row, and + say which server-side cap bound the run. Returns the spend payload and the + notes with the cap note in front, since the cap is context for the number + rather than an afterthought to it. + + ``command`` names the ledger row's command, so a caller that is not a build + settles under its own name and a reader summing one command's spend is not + handed another's. + """ + + from ..envelope import Paradigm + + spend: dict[str, float | None] | None = None + if paradigm is Paradigm.BYTES_SCANNED: + notes = [ + "each statement was capped server-side by the profile's " + "maximum_bytes_billed (a per-statement cap, not per run)", + *notes, + ] + # Popped, not read: `data.spend` is the one place any command reports + # what it billed, and a second copy at the top of `data` that only this + # command carried was worse than no copy at all (issue #276). A caller + # reading `data.bytes_billed` saw a build's spend and nothing for a + # `maintain check` that had just scanned 0.89 GB, and an absent key + # defaulted to zero reads as free. + billed = summary.pop("bytes_billed", None) + if billed is None and summary.get("nodes"): + # Statements ran and not one of them reported a billing figure, so + # what this run cost is unknown rather than nothing. Reporting the + # key as null says exactly that; zero would under-report spend, which + # is the one direction a cost guard must never round. A run that + # died before executing anything has an empty `nodes` and really did + # bill nothing, so it settles at zero like any other free run. + notes = [ + *notes, + "dbt reported no billing figure for any statement in this run, " + "so spend is unknown rather than zero and nothing was appended " + "to the spend ledger", + ] + if gate_billed: + notes = [ + *notes, + "the verification row counts did bill, and are in the " + "ledger and in session_spent_today; an unknown build " + "figure plus a known one is still unknown, so they are " + "not added into the reported spend", + ] + else: + billed = float(billed or 0.0) + spend = _record_build_spend( + store, + connector, + billed, + paradigm, + estimate, + gate_billed=gate_billed, + command=command, + ) + elif paradigm is Paradigm.COMPUTE_TIME: + cap_note = _COMPUTE_TIME_CAP_NOTES.get( + connector, _DEFAULT_COMPUTE_TIME_CAP_NOTE + ) + notes = [cap_note, *notes] + # dbt-snowflake, dbt-databricks, and dbt-redshift report no billing figure; + # per-node execution time is the honest compute-seconds actual. It is + # always known (a run with no nodes burned no seconds), so unlike the + # bytes path there is no unknown case to distinguish from zero. + seconds = sum( + float(node.get("execution_time") or 0) for node in summary.get("nodes", []) + ) + spend = _record_build_spend( + store, + connector, + seconds, + paradigm, + estimate, + gate_billed=gate_billed, + command=command, + ) + translate = getattr(adapter, "compute_spend_translation", None) + if translate is not None: + # Unconditional, including at zero seconds: the translated keys + # (compute-unit-hours, USD) are part of this connector's spend shape, + # and a run that omitted them would be the same present-sometimes key + # this issue is about, one level down. + spend.update(translate(seconds)) + elif paradigm is Paradigm.DB_LOAD: + notes = [_DB_LOAD_CAP_NOTES.get(connector, _UNCAPPED_BUILD_NOTE), *notes] + # The db-load dbt adapters report no billing figure; per-node execution + # time is the honest database-seconds actual. + seconds = sum( + float(node.get("execution_time") or 0) for node in summary.get("nodes", []) + ) + spend = _record_build_spend( + store, + connector, + seconds, + paradigm, + estimate, + gate_billed=gate_billed, + command=command, + ) + return spend, notes + + def _price_build( engine: DexEngine, project, @@ -1904,7 +2647,6 @@ def _shape_build_result( releasing first would overstate it by the whole estimate. """ - from ..envelope import Paradigm from ..guards.cost_guard import ( no_session_ceiling_warning, unserialized_ledger_warning, @@ -1927,83 +2669,21 @@ def _shape_build_result( "nodes this build touched; they are reported in " "data.verification.findings and do not change the build's status", ] - spend: dict[str, float | None] | None = None # What the handshake priced this build at, ledgered beside what it billed so # a later over-ceiling refusal on this connector can say how far the two # have run apart. `None` on the degraded-pricing path, where there was no # estimate to compare against and inventing one would be worse than none. estimate = getattr(cost, "estimate", None) - if paradigm is Paradigm.BYTES_SCANNED: - notes = [ - "each statement was capped server-side by the profile's " - "maximum_bytes_billed (a per-statement cap, not per run)", - *notes, - ] - # Popped, not read: `data.spend` is the one place any command reports - # what it billed, and a second copy at the top of `data` that only this - # command carried was worse than no copy at all (issue #276). A caller - # reading `data.bytes_billed` saw a build's spend and nothing for a - # `maintain check` that had just scanned 0.89 GB, and an absent key - # defaulted to zero reads as free. - billed = summary.pop("bytes_billed", None) - if billed is None and summary.get("nodes"): - # Statements ran and not one of them reported a billing figure, so - # what this build cost is unknown rather than nothing. Reporting the - # key as null says exactly that; zero would under-report spend, which - # is the one direction a cost guard must never round. A build that - # died before executing anything has an empty `nodes` and really did - # bill nothing, so it settles at zero like any other free run. - notes = [ - *notes, - "dbt reported no billing figure for any statement in this run, " - "so spend is unknown rather than zero and nothing was appended " - "to the spend ledger", - ] - if gate_billed: - notes = [ - *notes, - "the verification row counts did bill, and are in the " - "ledger and in session_spent_today; an unknown build " - "figure plus a known one is still unknown, so they are " - "not added into the reported spend", - ] - else: - billed = float(billed or 0.0) - spend = _record_build_spend( - store, connector, billed, paradigm, estimate, gate_billed=gate_billed - ) - elif paradigm is Paradigm.COMPUTE_TIME: - cap_note = _COMPUTE_TIME_CAP_NOTES.get( - connector, _DEFAULT_COMPUTE_TIME_CAP_NOTE - ) - notes = [cap_note, *notes] - # dbt-snowflake, dbt-databricks, and dbt-redshift report no billing figure; - # per-node execution time is the honest compute-seconds actual. It is - # always known (a run with no nodes burned no seconds), so unlike the - # bytes path there is no unknown case to distinguish from zero. - seconds = sum( - float(node.get("execution_time") or 0) for node in summary.get("nodes", []) - ) - spend = _record_build_spend( - store, connector, seconds, paradigm, estimate, gate_billed=gate_billed - ) - translate = getattr(adapter, "compute_spend_translation", None) - if translate is not None: - # Unconditional, including at zero seconds: the translated keys - # (compute-unit-hours, USD) are part of this connector's spend shape, - # and a run that omitted them would be the same present-sometimes key - # this issue is about, one level down. - spend.update(translate(seconds)) - elif paradigm is Paradigm.DB_LOAD: - notes = [_DB_LOAD_CAP_NOTES.get(connector, _UNCAPPED_BUILD_NOTE), *notes] - # The db-load dbt adapters report no billing figure; per-node execution - # time is the honest database-seconds actual. - seconds = sum( - float(node.get("execution_time") or 0) for node in summary.get("nodes", []) - ) - spend = _record_build_spend( - store, connector, seconds, paradigm, estimate, gate_billed=gate_billed - ) + spend, notes = _settle_dbt_spend( + summary, + notes, + paradigm=paradigm, + connector=connector, + store=store, + estimate=estimate, + adapter=adapter, + gate_billed=gate_billed, + ) notes = [ *notes, *no_session_ceiling_warning( diff --git a/packages/dex-core/src/exmergo_dex_core/transform/mutation.py b/packages/dex-core/src/exmergo_dex_core/transform/mutation.py new file mode 100644 index 00000000..d353ad3d --- /dev/null +++ b/packages/dex-core/src/exmergo_dex_core/transform/mutation.py @@ -0,0 +1,812 @@ +"""Planted defects for a model's tests to catch, and the reading of what they caught. + +A test suite that passes proves the tests ran, not that they would notice if the +model were wrong. Counting tests does not distinguish the two, and neither does +coverage in any sense dbt can report: a model with a `not_null` on its key and a +model with a unit test pinning its arithmetic both read as "tested". + +The way to tell them apart is to break the model on purpose and see whether +anything complains. Each mutation here stands for a defect class that recurs in +real analytics code, so a mutant that survives is not a curiosity about SQL: it +is a sentence about the suite, that this defect could ship through it. The +report is written that way, in the defect's terms rather than as a diff, because +the reader's next action is to write a test, not to read SQL they already know. + +Three properties this module keeps, all of them load-bearing: + +- **Pure.** Nothing here runs dbt, opens a connection, or touches the filesystem. + It turns SQL into other SQL and reads statuses. That is what makes the defect + library testable without a warehouse, and it is why the caller owns every + decision about cost. +- **Mutants never compound.** Each one is applied to a fresh copy of the parsed + tree, so a survivor means *this* defect survived, not this defect on top of the + previous one. +- **The identity is a mutant too.** The unmutated tree goes through the same + render, so a round trip that changes what the tests see shows up once, in the + baseline, instead of being blamed on every mutant separately. + +Mutation runs against the model's **compiled** SQL, which is what makes it work +on real projects: dbt has already expanded the macros, the ``var()`` calls and +the ``{% if %}`` branches, so a model that plan-time attribution has to refuse +for its jinja is mutated here without complaint. The cost is that the compiled +form names physical relations, and a test fixture addresses its inputs by +``ref()``. :func:`prepare` restores those references so both halves work. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field + +import sqlglot +from sqlglot import expressions as exp + +from .. import sql_shape +from ..errors import DexError + +#: How many mutants one run may execute. Every mutant is a dbt invocation, so +#: this is a bound on wall time and on spend, not a matter of taste. A caller may +#: ask for fewer; nothing may ask for more, because the ceiling is what makes the +#: command's cost predictable before it is priced. +MAX_MUTANTS = 20 + +#: What every mutant is written into the copied project with. Appended rather +#: than prepended: dbt gives scalar config keys to the *last* ``config()`` call +#: in a file, so a model carrying its own ``materialized='incremental'`` would +#: otherwise win and the mutant would build a relation. +#: +#: Each key earns its place. ``ephemeral`` means dbt inlines the mutant into each +#: test as a CTE and materializes nothing, so no relation is created, replaced or +#: dropped, and there is nothing to clean up. ``contract`` off because a mutant +#: legitimately changes the shape a contract pins, and the contract failing would +#: mask whether the tests noticed. ``access='protected'`` because dbt refuses to +#: parse a `public` model that is ephemeral, which would otherwise make this +#: command unusable on exactly the models most worth testing. +EPHEMERAL_HEADER = ( + "{{ config(materialized='ephemeral', contract={'enforced': false}, " + "access='protected') }}" +) + +#: dbt's own prefix for an inlined ephemeral parent. +DBT_CTE_PREFIX = "__dbt__cte__" + +#: What an operator hands back: the fragment before, the fragment after, and +#: the sentence describing the defect. ``None`` when the site turned out not to +#: be mutable after all. +Applied = tuple[str, str, str] + +_PLACEHOLDER = "__dex_ref_{}__" +_PLACEHOLDER_RE = re.compile(r"__dex_ref_(\d+)__") + +# A defect is only worth reporting if the reader can act on it, and the action is +# always the same shape: write the test that would have caught this. Naming that +# test per operator is what turns a finding into a next step. +_SUGGESTED_TEST = { + "comparison": "a unit test with a row exactly on the boundary", + "predicate_drop": "an `expression_is_true` test asserting the filter holds", + "predicate_negate": "an `expression_is_true` test asserting the filter holds", + "join_type": "a `relationships` test, or a row-count assertion against the parent", + # Deliberately not `accepted_values`: a dropped branch makes its rows fall + # through to a category that is still in the allowed list, so that test + # cannot see it. Only an assertion that a known input lands in a known + # category can. + "case_branch": "a unit test with a row in each branch's category", + "division": "a unit test pinning a known ratio", + "window_frame": "a unit test over several rows in one partition", + "aggregate": "a unit test with more than one row per group", +} + + +class MutationError(DexError): + """A model dex will not mutate, always saying which property stopped it.""" + + +@dataclass(frozen=True) +class ParentRelation: + """One input of the model, in both the spellings that matter. + + ``rendered`` is how the compiled SQL names it: a physical relation for an + ordinary parent, or dbt's ``__dbt__cte__`` alias for one that is ephemeral. + ``jinja`` is how a dbt file has to name it for a unit test fixture to bind. + """ + + rendered: str + jinja: str + ephemeral: bool = False + + +@dataclass +class PreparedModel: + """A model's compiled SQL, parsed, with its inputs turned back into refs.""" + + tree: exp.Expression + dialect: str + refs: dict[str, str] = field(default_factory=dict) + notes: list[str] = field(default_factory=list) + + def readable(self, text: str) -> str: + """The same text with dex's internal ref placeholders spelled as names. + + Placeholders exist so a reference survives SQL generation as an + identifier. They must never reach a person: a finding that says the join + to ``__dex_ref_1__`` changed names nothing the reader can act on. + """ + + def name(match: re.Match[str]) -> str: + jinja = self.refs.get(match.group(0), match.group(0)) + quoted = re.search(r"'([^']+)'\s*\)\s*\}\}", jinja) + return quoted.group(1) if quoted else jinja + + return _PLACEHOLDER_RE.sub(name, text) + + +@dataclass(frozen=True) +class Mutant: + """One planted defect: what it is, where, and what would have caught it.""" + + id: str + operator: str + scope: str + defect: str + before: str + after: str + suggested_test: str + body: str + + def payload(self) -> dict[str, object]: + return { + "id": self.id, + "operator": self.operator, + "scope": self.scope, + "defect": self.defect, + "before": self.before, + "after": self.after, + "suggested_test": self.suggested_test, + } + + +@dataclass +class MutantBatch: + """The mutants a run will execute, and an honest account of the rest. + + ``elided`` is per operator rather than a single number because the cap is + spread across defect classes: a caller who sees five comparison mutants and + no join mutant has to be able to tell "this model has no joins" from "the cap + cut the joins off". + """ + + identity: str + mutants: list[Mutant] = field(default_factory=list) + elided: dict[str, int] = field(default_factory=dict) + considered: int = 0 + unparsed: int = 0 + cap: int = MAX_MUTANTS + + @property + def elided_total(self) -> int: + return sum(self.elided.values()) + + +@dataclass(frozen=True) +class Verdict: + """What one mutant's run says about the suite.""" + + outcome: str + caught_by: list[str] = field(default_factory=list) + warn_only: bool = False + + +# --- preparing the model ------------------------------------------------------- + + +def prepare( + compiled_code: str, + *, + dialect: str, + parents: Sequence[ParentRelation] = (), +) -> PreparedModel: + """Parse a model's compiled SQL and put its ``ref()`` and ``source()`` back. + + dbt compiles a reference into a physical relation name, and inlines an + ephemeral parent as a ``__dbt__cte__`` CTE. Both have to be undone: a unit + test binds its fixtures to the *reference*, so a mutant that still named the + warehouse relation would quietly read real data in a test that was supposed + to be reading fixtures, and would pass for the wrong reason. + + Matching is exact on the parsed relation, never on a suffix. dbt renders a + ``relation_name`` through the same code that renders a ``ref()``, so the two + texts agree by construction, and an exact match is therefore available; a + tolerant one would risk binding two same-named relations in different + databases to a single reference, which silently mutates the wrong model. + A relation matching no parent is left exactly as written and noted, since + dex would rather read a hardcoded table honestly than guess at a reference + the project never declared. + """ + + try: + tree = sqlglot.parse_one(compiled_code, read=dialect) + except Exception as exc: + raise MutationError( + f"dex could not parse this model's compiled SQL ({_clip(str(exc))}), " + "so it cannot plant a defect in it without guessing" + ) from exc + if tree is None: + raise MutationError("the model's compiled SQL is empty") + if not isinstance(tree, (exp.Select, exp.SetOperation, exp.With, exp.Subquery)): + raise MutationError( + f"the model's compiled SQL is a {type(tree).__name__.lower()} rather " + "than a single query, and dex will not mutate a statement it cannot " + "reason about as one SELECT" + ) + + prepared = PreparedModel(tree=tree, dialect=dialect) + by_rendered = {p.rendered: p for p in parents} + + # The ephemeral parents first: they are CTEs in this tree, and removing one + # has to happen before its references are rewritten, or the CTE body itself + # gets a placeholder pointing at the parent it *is*. + cte_jinja: dict[str, str] = {} + with_clause = tree.args.get("with_") or tree.args.get("with") + if with_clause is not None: + keep = [] + for cte in list(with_clause.expressions): + alias = cte.alias_or_name + if not alias.startswith(DBT_CTE_PREFIX): + keep.append(cte) + continue + parent = by_rendered.get(alias) + if parent is None: + raise MutationError( + f"the compiled SQL inlines '{alias}', which dex cannot match to " + "any of this model's declared parents, so it cannot restore the " + "reference it stands for" + ) + cte_jinja[alias.lower()] = parent.jinja + if keep: + with_clause.set("expressions", keep) + else: + tree.set("with_", None) + tree.set("with", None) + + for table in tree.find_all(exp.Table): + bare = _relation_text(table, dialect) + parent = by_rendered.get(bare) + jinja = parent.jinja if parent is not None else cte_jinja.get(bare.lower()) + if jinja is None: + if not _names_a_scope(table, tree): + prepared.notes.append( + f"{bare} matches none of this model's declared parents and is " + "left as written; a unit test fixture cannot stand in for it" + ) + continue + token = _PLACEHOLDER.format(len(prepared.refs)) + prepared.refs[token] = jinja + table.set("this", exp.to_identifier(token, quoted=False)) + table.set("db", None) + table.set("catalog", None) + + return prepared + + +def render(prepared: PreparedModel, tree: exp.Expression | None = None) -> str: + """A parsed tree back to a dbt model file: refs restored, the rest inert. + + Everything that is not a reference is wrapped in ``{% raw %}``. Compiled SQL + is data as far as this command is concerned, and dbt would otherwise render a + ``{{`` inside a string literal or a comment a second time, which is both a + wrong mutant and a way to make dbt evaluate text that came out of a + warehouse. + """ + + sql = (tree if tree is not None else prepared.tree).sql( + dialect=prepared.dialect, pretty=True + ) + if "endraw" in sql: + raise MutationError( + "the model's compiled SQL contains the text 'endraw', which dex " + "cannot safely quote for dbt; mutation coverage is unavailable here" + ) + out: list[str] = [] + cursor = 0 + for match in _PLACEHOLDER_RE.finditer(sql): + literal = sql[cursor : match.start()] + if literal: + out.append("{% raw %}" + literal + "{% endraw %}") + out.append(prepared.refs.get(match.group(0), match.group(0))) + cursor = match.end() + tail = sql[cursor:] + if tail: + out.append("{% raw %}" + tail + "{% endraw %}") + return "".join(out) + "\n" + EPHEMERAL_HEADER + "\n" + + +# --- the defect library -------------------------------------------------------- + + +def _comparison_sites(tree: exp.Expression) -> list[exp.Expression]: + # Boundary comparisons only. Flipping `=` to `<>` is a different defect and, + # in a join condition, one the warehouse or the tests reject immediately, + # which would spend a run to learn nothing. + return list(tree.find_all(exp.GT, exp.GTE, exp.LT, exp.LTE)) + + +def _flip_comparison(node: exp.Expression, dialect: str) -> Applied | None: + pairs = {exp.GT: exp.GTE, exp.GTE: exp.GT, exp.LT: exp.LTE, exp.LTE: exp.LT} + replacement = pairs[type(node)](this=node.this, expression=node.expression) + before, after = _text(dialect, node), _text(dialect, replacement) + node.replace(replacement) + inclusive = isinstance(replacement, (exp.GTE, exp.LTE)) + return ( + before, + after, + f"the boundary '{before}' " + f"{'now includes' if inclusive else 'now excludes'} the edge value " + f"('{after}'), so a row sitting exactly on it " + f"{'enters' if inclusive else 'leaves'} the model", + ) + + +def _predicate_sites(tree: exp.Expression) -> list[exp.Expression]: + sites: list[exp.Expression] = [] + for select in tree.find_all(exp.Select): + for clause in ("where", "having", "qualify"): + sites.extend(sql_shape.predicates(select, clause)) + return sites + + +def _drop_predicate(node: exp.Expression, dialect: str) -> Applied | None: + select, clause = _owning_clause(node) + if select is None: + return None + before = _text(dialect, node) + kept = [ + p + for p in sql_shape.predicates(select, clause) + if sql_shape.match_key(p) != sql_shape.match_key(node) + ] + sql_shape.set_predicates(select, clause, kept) + return ( + before, + "", + f"the filter '{before}' stops filtering, so every row it was excluding " + "now flows into the model", + ) + + +def _negate_predicate(node: exp.Expression, dialect: str) -> Applied | None: + select, clause = _owning_clause(node) + if select is None: + return None + before = _text(dialect, node) + negated = exp.Not(this=exp.Paren(this=node.copy())) + swapped = [ + negated if sql_shape.match_key(p) == sql_shape.match_key(node) else p + for p in sql_shape.predicates(select, clause) + ] + sql_shape.set_predicates(select, clause, swapped) + return ( + before, + _text(dialect, negated), + f"the filter '{before}' is inverted, so the model keeps exactly the rows " + "it was written to exclude", + ) + + +def _join_sites(tree: exp.Expression) -> list[exp.Expression]: + sites: list[exp.Expression] = [] + for select in tree.find_all(exp.Select): + sites.extend( + join.node + for join in sql_shape.joins(select) + if join.side in {"inner", "left"} + ) + return sites + + +def _retype_join(node: exp.Expression, dialect: str) -> Applied | None: + side = " ".join(p for p in (node.side or "", node.kind or "") if p).lower() + relation = _text(dialect, node.this) + if side == "left": + node.set("side", None) + node.set("kind", "INNER") + return ( + f"left join {relation}", + f"inner join {relation}", + f"the left join to {relation} became an inner join, so every row " + "whose match is missing is silently dropped from the model", + ) + node.set("side", "LEFT") + node.set("kind", None) + return ( + f"inner join {relation}", + f"left join {relation}", + f"the inner join to {relation} became a left join, so rows that should " + "have been filtered out survive, carrying nulls", + ) + + +def _case_sites(tree: exp.Expression) -> list[exp.Expression]: + return [c for c in tree.find_all(exp.Case) if (c.args.get("ifs") or [])] + + +def _drop_case_branch(node: exp.Expression, dialect: str) -> Applied | None: + branches = list(node.args.get("ifs") or []) + dropped = branches[0] + # A lone branch prints as a whole `CASE WHEN ... END`, which reads as though + # the entire expression were the thing being dropped. + before = ( + f"when {_text(dialect, dropped.this)} " + f"then {_text(dialect, dropped.args.get('true'))}" + ) + remaining = branches[1:] + if remaining: + node.set("ifs", remaining) + after = _text(dialect, node) + else: + # A CASE with no branches is not printable, so the whole expression + # collapses to whatever it would have fallen through to. + fallback = node.args.get("default") or exp.null() + after = _text(dialect, fallback) + node.replace(fallback) + return ( + before, + after, + f"the branch '{before}' is gone, so the category it handled falls " + "through to the default and is reported as something it is not", + ) + + +def _division_sites(tree: exp.Expression) -> list[exp.Expression]: + return list(tree.find_all(exp.Div, exp.SafeDivide)) + + +def _swap_division(node: exp.Expression, dialect: str) -> Applied | None: + before = _text(dialect, node) + numerator, denominator = node.this, node.expression + node.set("this", denominator) + node.set("expression", numerator) + after = _text(dialect, node) + # A guarded division (Snowflake's DIV0, a NULLIF wrapper) tests the + # denominator by name, and swapping the operands without the guard leaves it + # checking the operand that is no longer the divisor. + guard = _enclosing_zero_guard(node, denominator) + if guard is not None: + guard.set("this", numerator.copy()) + return ( + before, + after, + f"the ratio '{before}' is inverted, so it reports the reciprocal of the " + "rate it is named for", + ) + + +def _window_sites(tree: exp.Expression) -> list[exp.Expression]: + return [ + spec + for spec in tree.find_all(exp.WindowSpec) + if any( + isinstance(spec.args.get(bound), exp.Literal) + and not spec.args[bound].args.get("is_string") + for bound in ("start", "end") + ) + ] + + +def _shift_window_frame(node: exp.Expression, dialect: str) -> Applied | None: + for bound in ("start", "end"): + literal = node.args.get(bound) + if not isinstance(literal, exp.Literal) or literal.args.get("is_string"): + continue + try: + value = int(literal.this) + except (TypeError, ValueError): + continue + before = _text(dialect, node) + node.set(bound, exp.Literal.number(value + 1)) + return ( + before, + _text(dialect, node), + f"the window frame reaches one row further than it should " + f"({value} became {value + 1}), so every rolling value is computed " + "over the wrong span", + ) + return None + + +def _aggregate_sites(tree: exp.Expression) -> list[exp.Expression]: + return list(tree.find_all(exp.Sum, exp.Max)) + + +def _swap_aggregate(node: exp.Expression, dialect: str) -> Applied | None: + before = _text(dialect, node) + replacement = (exp.Max if isinstance(node, exp.Sum) else exp.Sum)(this=node.this) + node.replace(replacement) + after = _text(dialect, replacement) + original, swapped = ( + ("total", "largest single value") + if isinstance(node, exp.Sum) + else ("largest single value", "total") + ) + return ( + before, + after, + f"'{before}' now reports the {swapped} rather than the {original}, which " + "agrees with it whenever a group holds exactly one row", + ) + + +@dataclass(frozen=True) +class _Operator: + name: str + sites: Callable[[exp.Expression], list[exp.Expression]] + apply: Callable[[exp.Expression, str], Applied | None] + + +# Ordered as the issue's defect table is, and iterated round robin, so a cap cuts +# the tail of each class rather than everything after the first. +_OPERATORS: tuple[_Operator, ...] = ( + _Operator("comparison", _comparison_sites, _flip_comparison), + _Operator("predicate_drop", _predicate_sites, _drop_predicate), + _Operator("join_type", _join_sites, _retype_join), + _Operator("case_branch", _case_sites, _drop_case_branch), + _Operator("division", _division_sites, _swap_division), + _Operator("window_frame", _window_sites, _shift_window_frame), + _Operator("aggregate", _aggregate_sites, _swap_aggregate), + _Operator("predicate_negate", _predicate_sites, _negate_predicate), +) + + +def enumerate_mutants( + prepared: PreparedModel, *, cap: int = MAX_MUTANTS +) -> MutantBatch: + """Every defect this model can carry, ordered so a cap stays representative. + + Sites are counted on the original tree and each mutant is then applied to a + fresh copy, which is what keeps them independent: mutant seven is this model + with one defect, not with seven. + + The round robin is the whole reason the order is not simply "all comparisons, + then all joins". A model with forty comparisons and one join would otherwise + spend an entire capped run proving things about comparisons and never test + whether anything catches a join defect, which is the more expensive bug. + """ + + cap = max(0, min(cap, MAX_MUTANTS)) + identity = render(prepared) + + planned: list[tuple[str, int]] = [] + counts: dict[str, int] = {} + for operator in _OPERATORS: + found = len(operator.sites(prepared.tree)) + counts[operator.name] = found + planned.extend((operator.name, index) for index in range(found)) + + ordered = _round_robin(planned) + batch = MutantBatch(identity=identity, considered=len(ordered), cap=cap) + by_name = {operator.name: operator for operator in _OPERATORS} + + for operator_name, index in ordered: + if len(batch.mutants) >= cap: + batch.elided[operator_name] = batch.elided.get(operator_name, 0) + 1 + continue + operator = by_name[operator_name] + tree = prepared.tree.copy() + sites = operator.sites(tree) + if index >= len(sites): + continue + node = sites[index] + scope = _scope_label(node, tree) + applied = operator.apply(node, prepared.dialect) + if applied is None: + continue + before, after, defect = (prepared.readable(part) for part in applied) + try: + body = render(prepared, tree) + # A mutant that cannot be re-read is a mutant dex cannot stand + # behind, so it is dropped here rather than sent to the warehouse to + # fail there and be reported as a defect the tests "rejected". + sqlglot.parse_one( + _PLACEHOLDER_RE.sub("x", tree.sql(dialect=prepared.dialect)), + read=prepared.dialect, + ) + except Exception: + batch.unparsed += 1 + continue + batch.mutants.append( + Mutant( + id=f"m{len(batch.mutants) + 1:02d}", + operator=operator_name, + scope=scope, + defect=defect, + before=before, + after=after, + suggested_test=_SUGGESTED_TEST[operator_name], + body=body, + ) + ) + return batch + + +# --- reading the runs ---------------------------------------------------------- + + +def classify( + baseline: dict[str, str], + run: dict[str, str] | None, + *, + warn_severity: Iterable[str] = (), +) -> Verdict: + """What one mutant's test statuses say, judged only against what passed before. + + Only the tests that passed at baseline can testify. A test that was already + failing says nothing about this defect, and counting it as a catch would + report a suite as strong precisely because it was broken. + + ``rejected`` is kept apart from ``killed`` deliberately. A mutant every test + errors on is one the warehouse refused to run, so a real change of that shape + would fail the build outright rather than ship: that is not evidence the + tests would have caught it, and folding it into ``killed`` would flatter the + suite. + """ + + if not run: + return Verdict(outcome="not_run") + passing = [name for name, status in baseline.items() if status == "pass"] + missing = [name for name in passing if name not in run] + if missing: + return Verdict(outcome="not_run") + + caught = [n for n in passing if run[n] in {"fail", "warn"}] + errored = [n for n in passing if run[n] == "error"] + if caught: + warned = set(warn_severity) + return Verdict( + outcome="killed", + caught_by=sorted(caught), + warn_only=all(run[n] == "warn" or n in warned for n in caught), + ) + if errored: + return Verdict(outcome="rejected", caught_by=sorted(errored)) + return Verdict(outcome="survived") + + +def inline_into_test( + test_sql: str, *, model_name: str, body: str, dialect: str +) -> str | None: + """A test's compiled SQL with the mutant standing in for the model. + + Only used for pricing, and only on connectors that charge by what a statement + scans: a mutant that drops a partition predicate scans more than the model it + came from, so pricing every mutant at the baseline's cost would under-report + the batch, which is the one direction a cost guard must never round. + + Substring replacement is not available here. dbt inlines a model's ephemeral + parents into the model's own compiled SQL, so the text in the manifest and + the text inside the test's CTE are different strings whenever the model has + an ephemeral parent. Returns ``None`` when the CTE cannot be found or the + test will not parse, and the caller prices that test at its baseline. + """ + + try: + tree = sqlglot.parse_one(test_sql, read=dialect) + replacement = sqlglot.parse_one(body, read=dialect) + except Exception: + return None + if tree is None or replacement is None: + return None + alias = f"{DBT_CTE_PREFIX}{model_name}".lower() + with_clause = tree.args.get("with_") or tree.args.get("with") + if with_clause is None: + return None + for cte in with_clause.expressions: + if cte.alias_or_name.lower() == alias: + cte.set("this", replacement) + return tree.sql(dialect=dialect) + return None + + +# --- helpers ------------------------------------------------------------------- + + +def _round_robin(planned: list[tuple[str, int]]) -> list[tuple[str, int]]: + by_operator: dict[str, list[tuple[str, int]]] = {} + for entry in planned: + by_operator.setdefault(entry[0], []).append(entry) + ordered: list[tuple[str, int]] = [] + while any(by_operator.values()): + for operator in _OPERATORS: + queue = by_operator.get(operator.name) + if queue: + ordered.append(queue.pop(0)) + return ordered + + +def _text(dialect: str, node: exp.Expression | None) -> str: + """A fragment as the model's own dialect spells it. + + :func:`sql_shape.text` is dialect free, which is right for matching and + wrong for a report: read as duckdb, ``a / b`` carries a safe-division flag + that the default dialect prints as ``a / NULLIF(b, 0)``. Showing a reader SQL + that is not what will run undermines the one thing a finding has to be. + """ + + return "" if node is None else node.sql(dialect=dialect, comments=False).strip() + + +def _relation_text(table: exp.Table, dialect: str) -> str: + """A relation's identity with any alias removed, in the dialect's spelling.""" + + bare = table.copy() + bare.set("alias", None) + return bare.sql(dialect=dialect) + + +def _names_a_scope(table: exp.Table, tree: exp.Expression) -> bool: + """True when this table reference is really a CTE name defined in the tree.""" + + return table.name.lower() in {name.lower() for name in sql_shape.scopes(tree)} + + +def _owning_clause(node: exp.Expression) -> tuple[exp.Select | None, str]: + """The SELECT whose WHERE/HAVING/QUALIFY this predicate belongs to.""" + + clauses = ( + (exp.Where, "where"), + (exp.Having, "having"), + (exp.Qualify, "qualify"), + ) + for clause, key in clauses: + owner = node.find_ancestor(clause) + if owner is not None: + select = owner.find_ancestor(exp.Select) + if select is not None: + return select, key + return None, "" + + +def _enclosing_zero_guard( + node: exp.Expression, denominator: exp.Expression +) -> exp.EQ | None: + """The ``denominator = 0`` test guarding this division, if there is one. + + Snowflake's ``DIV0`` does not survive a parse and a print: it comes back as a + conditional testing the divisor by name. Swapping the operands underneath + that guard would leave it checking a column that is no longer the divisor. + """ + + conditional = node.find_ancestor(exp.If, exp.Case) + if conditional is None: + return None + wanted = sql_shape.match_key(denominator) + for candidate in conditional.find_all(exp.EQ): + other = candidate.expression + if ( + isinstance(other, exp.Literal) + and str(other.this) == "0" + and sql_shape.match_key(candidate.this) == wanted + ): + return candidate + return None + + +def _scope_label(node: exp.Expression, tree: exp.Expression) -> str: + """Where in the model a site sits, named the way its author would name it.""" + + select = node if isinstance(node, exp.Select) else node.find_ancestor(exp.Select) + if select is None: + return sql_shape.MAIN_SCOPE + cte = select.find_ancestor(exp.CTE) + if cte is not None: + return cte.alias_or_name + union = select.find_ancestor(exp.SetOperation) + if union is not None: + branches = list(union.find_all(exp.Select)) + if select in branches: + return f"branch {branches.index(select) + 1} of the final union" + return sql_shape.MAIN_SCOPE + + +def _clip(text: str, limit: int = 160) -> str: + flat = " ".join(text.split()) + return flat if len(flat) <= limit else f"{flat[: limit - 3]}..." diff --git a/packages/dex-core/src/exmergo_dex_core/transform/results.py b/packages/dex-core/src/exmergo_dex_core/transform/results.py index 408b1d91..62e10f4e 100644 --- a/packages/dex-core/src/exmergo_dex_core/transform/results.py +++ b/packages/dex-core/src/exmergo_dex_core/transform/results.py @@ -311,6 +311,46 @@ def data(self) -> dict[str, Any]: return {"ran": True, **self.summary} +class MutationCoverageResult(Result): + """Which planted defects a model's tests caught, and which they missed. + + Survivors lead the list. A caller reading this is deciding what test to write + next, and the mutants that were caught are the ones they need to read least, + so ordering by outcome is the difference between an answer and a table. + + ``score`` is deliberately not the headline and is null rather than zero when + nothing ran. It is a ratio of two small integers over one model, and a number + that invites comparison between models would be read as a quality metric it + cannot support; the survivors themselves are the finding. + + ``baseline`` is the honesty field. Every verdict here is relative to the + tests that passed before anything was mutated, so the excluded ones have to + be visible: a suite whose only real test was already failing would otherwise + report a clean sweep of survivors and read as though it had been measured. + """ + + model: str = "" + target: str = "" + baseline: dict[str, Any] = Field(default_factory=dict) + mutants: list[dict[str, Any]] = Field(default_factory=list) + counts: dict[str, int] = Field(default_factory=dict) + score: float | None = None + cap: dict[str, Any] = Field(default_factory=dict) + runs: int = 0 + + def data(self) -> dict[str, Any]: + return { + "model": self.model, + "target": self.target, + "baseline": self.baseline, + "mutants": self.mutants, + "counts": self.counts, + "score": self.score, + "cap": self.cap, + "runs": self.runs, + } + + class BuildResult(Result): """A finished dbt run, dev-target only and cost-surfaced beforehand. diff --git a/packages/dex-core/src/exmergo_dex_core/transform/row_attribution.py b/packages/dex-core/src/exmergo_dex_core/transform/row_attribution.py index 41163015..1d7c698a 100644 --- a/packages/dex-core/src/exmergo_dex_core/transform/row_attribution.py +++ b/packages/dex-core/src/exmergo_dex_core/transform/row_attribution.py @@ -287,17 +287,6 @@ def finding(self) -> RowPopulationChange: ) -def _set_predicates(select: exp.Select, key: str, preds: list[exp.Expression]) -> None: - if not preds: - select.set(key, None) - return - condition = preds[0] - for extra in preds[1:]: - condition = exp.And(this=condition, expression=extra) - wrapper = {"where": exp.Where, "having": exp.Having, "qualify": exp.Qualify}[key] - select.set(key, wrapper(this=condition)) - - def _predicate_mutator( clause: str, dialect: str, *, drop: str | None = None, add: str | None = None ) -> Callable[[exp.Select], None]: @@ -307,7 +296,7 @@ def mutate(select: exp.Select) -> None: preds = [p for p in preds if sql_shape.match_key(p) != drop] if add is not None: preds.append(sqlglot.parse_one(add, dialect=dialect)) - _set_predicates(select, clause, preds) + sql_shape.set_predicates(select, clause, preds) return mutate From b4ca768d808ec2450f75eb5b8ea793e6901002e3 Mon Sep 17 00:00:00 2001 From: Marco Ciavarella Date: Thu, 10 Sep 2026 20:07:16 +0200 Subject: [PATCH 2/5] Update unit tests with mutations support --- .../integration/test_bigquery_transform.py | 124 ++++ .../integration/test_snowflake_transform.py | 124 ++++ packages/dex-core/tests/test_cli_contract.py | 7 +- packages/dex-core/tests/test_engine.py | 3 +- packages/dex-core/tests/test_errors.py | 1 + packages/dex-core/tests/test_spend_parity.py | 123 ++++ packages/dex-core/tests/test_sql_shape.py | 31 + .../dex-core/tests/transform/test_mutation.py | 495 ++++++++++++++ .../tests/transform/test_mutation_command.py | 603 ++++++++++++++++++ 9 files changed, 1507 insertions(+), 4 deletions(-) create mode 100644 packages/dex-core/tests/transform/test_mutation.py create mode 100644 packages/dex-core/tests/transform/test_mutation_command.py diff --git a/packages/dex-core/tests/integration/test_bigquery_transform.py b/packages/dex-core/tests/integration/test_bigquery_transform.py index 4605d8d4..6386f836 100644 --- a/packages/dex-core/tests/integration/test_bigquery_transform.py +++ b/packages/dex-core/tests/integration/test_bigquery_transform.py @@ -241,3 +241,127 @@ def test_a_missing_dev_dataset_warns_rather_than_refusing( assert "dex_absent_dev_dataset does not exist" in warnings[0] assert "bigquery.datasets.create" in warnings[0] assert "bq mk --dataset --location=US" in warnings[0] + + +def test_mutation_coverage_prices_the_batch_then_measures_the_tests( + tmp_path: Path, capsys, bq_project: str, bq_scratch_dataset: str +): + """Mutation coverage against a metered connector, end to end. + + Two things only a live run can establish. The batch is priced and confirmed + as one number before any mutant executes, which is the cost contract for the + most expensive command dex has. And nothing the run does materializes: + mutants build as ephemeral models, so the dev dataset holds exactly the + relations it held before, which a fake dbt cannot demonstrate. + """ + + pytest.importorskip("dbt.adapters.bigquery") + root = str(tmp_path) + _seed_transform_repo(tmp_path, bq_project, bq_scratch_dataset) + + rc, envelope = run_cli( + ["--repo-root", root, "transform", "init", "analytics"], capsys + ) + assert_ok(rc, envelope) + + model = "dex_mutation_probe" + edits_file = tmp_path / "edits.json" + edits_file.write_text( + json.dumps( + { + "edits": [ + { + "path": f"models/staging/{model}.sql", + "kind": "model_sql", + "content": ( + "select id, amount from unnest([\n" + " struct(1 as id, 10 as amount),\n" + " struct(2 as id, 200 as amount)\n" + "]) where amount > 5\n" + ), + }, + { + "path": "models/staging/schema.yml", + "kind": "schema_yml", + "content": ( + "version: 2\n" + "models:\n" + f" - name: {model}\n" + " columns:\n" + " - name: id\n" + " data_tests: [not_null, unique]\n" + ), + }, + ] + } + ), + encoding="utf-8", + ) + rc, planned = run_cli( + [ + "--repo-root", + root, + "transform", + "plan", + "mutation probe", + "--edits-file", + str(edits_file), + ], + capsys, + ) + assert rc == 0, planned + rc, applied = run_cli(["--repo-root", root, "transform", "apply"], capsys) + assert rc == 0, applied + + from google.cloud import bigquery + + client = bigquery.Client(project=bq_project) + try: + before = { + t.table_id for t in client.list_tables(f"{bq_project}.{bq_scratch_dataset}") + } + + # Unconfirmed: one estimate for the whole batch, and nothing has run. + rc, unconfirmed = run_cli( + ["--repo-root", root, "transform", "test", "--mutate", model], capsys + ) + assert unconfirmed["status"] == "needs_confirmation", unconfirmed + assert unconfirmed["cost"]["estimate"] is not None + per_mutant = unconfirmed["data"].get("per_table_bytes") or {} + assert "(baseline)" in per_mutant, per_mutant + + rc, measured = run_cli( + [ + "--repo-root", + root, + "transform", + "test", + "--mutate", + model, + "--max-mutants", + "2", + "--confirm", + "--budget", + str(MAX_BYTES), + ], + capsys, + ) + assert rc == 0, measured + assert measured["status"] == "ok" + data = measured["data"] + assert data["counts"]["generated"] == 2 + assert data["runs"] == 3, data["runs"] + assert data["baseline"]["tests"], data["baseline"] + assert data["spend"]["bytes_billed"] is not None + + # Ephemeral mutants materialize nothing, so the dataset is unchanged. + after = { + t.table_id for t in client.list_tables(f"{bq_project}.{bq_scratch_dataset}") + } + assert after == before + + client.delete_table( + f"{bq_project}.{bq_scratch_dataset}.{model}", not_found_ok=True + ) + finally: + client.close() diff --git a/packages/dex-core/tests/integration/test_snowflake_transform.py b/packages/dex-core/tests/integration/test_snowflake_transform.py index 05c19f0c..e389ea2e 100644 --- a/packages/dex-core/tests/integration/test_snowflake_transform.py +++ b/packages/dex-core/tests/integration/test_snowflake_transform.py @@ -280,3 +280,127 @@ def test_config_drift_from_the_rendered_profile_is_refused( assert "DEX_RETARGETED_ELSEWHERE" in error assert sf_scratch_database in error assert "disagree about the dev target" in error + + +def test_mutation_coverage_prices_in_seconds_and_materializes_nothing( + tmp_path: Path, capsys, sf_scratch_database, sf_warehouse, sf_connection_name +): + """The compute-time half of the cost contract, and the isolation guarantee. + + Snowflake prices the batch in warehouse-seconds from a heuristic rather than + a dry run, so what matters here is that one estimate covers every mutant and + that the confirmed run settles seconds against the same command. The dev + schema is checked before and after: an ephemeral mutant creates no object, + so a run that left one behind would be writing where dex promised not to. + """ + + root = str(tmp_path) + seed_repo(tmp_path, sf_scratch_database, sf_warehouse, sf_connection_name) + + rc, envelope = run_cli( + ["--repo-root", root, "transform", "init", "analytics"], capsys + ) + assert_ok(rc, envelope) + + model = "dex_mutation_probe" + edits_file = tmp_path / "edits.json" + edits_file.write_text( + json.dumps( + { + "edits": [ + { + "path": f"models/staging/{model}.sql", + "kind": "model_sql", + "content": ( + "select column1 as id, column2 as amount\n" + "from values (1, 10), (2, 200)\n" + "where column2 > 5\n" + ), + }, + { + "path": "models/staging/schema.yml", + "kind": "schema_yml", + "content": ( + "version: 2\n" + "models:\n" + f" - name: {model}\n" + " columns:\n" + " - name: id\n" + " data_tests: [not_null, unique]\n" + ), + }, + ] + } + ), + encoding="utf-8", + ) + rc, planned = run_cli( + [ + "--repo-root", + root, + "transform", + "plan", + "mutation probe", + "--edits-file", + str(edits_file), + ], + capsys, + ) + assert rc == 0, planned + rc, applied = run_cli(["--repo-root", root, "transform", "apply"], capsys) + assert rc == 0, applied + + import snowflake.connector + + from exmergo_dex_core.config import SnowflakeTarget + from exmergo_dex_core.connect import resolve_snowflake_connection + + params, _method = resolve_snowflake_connection( + SnowflakeTarget(connection_name=sf_connection_name), os.environ, tmp_path + ) + conn = snowflake.connector.connect(**params) + + def objects() -> set: + cursor = conn.cursor() + cursor.execute(f'SHOW OBJECTS IN SCHEMA "{sf_scratch_database}"."DBT_DEV"') + return {row[1] for row in cursor.fetchall()} + + try: + before = objects() + + rc, unconfirmed = run_cli( + ["--repo-root", root, "transform", "test", "--mutate", model], capsys + ) + assert unconfirmed["status"] == "needs_confirmation", unconfirmed + assert unconfirmed["cost"]["paradigm"] == "compute_time" + assert unconfirmed["cost"]["estimate"] is not None + + rc, measured = run_cli( + [ + "--repo-root", + root, + "transform", + "test", + "--mutate", + model, + "--max-mutants", + "2", + "--confirm", + "--budget", + str(SF_MAX_SECONDS * 10), + ], + capsys, + ) + assert rc == 0, measured + assert measured["status"] == "ok" + data = measured["data"] + assert data["counts"]["generated"] == 2 + assert data["spend"]["seconds_billed"] >= 0 + + assert objects() == before + finally: + cursor = conn.cursor() + cursor.execute( + f'DROP VIEW IF EXISTS "{sf_scratch_database}"."DBT_DEV"."{model.upper()}"' + ) + conn.close() diff --git a/packages/dex-core/tests/test_cli_contract.py b/packages/dex-core/tests/test_cli_contract.py index dc51de8b..46c4330d 100644 --- a/packages/dex-core/tests/test_cli_contract.py +++ b/packages/dex-core/tests/test_cli_contract.py @@ -655,9 +655,10 @@ def test_a_refusal_before_the_engine_exists_reports_the_flagged_connector( }, ("transform", "test"): { "reason": ( - "scaffold-only; reachable as " - "exmergo_dex_core.transform.test_scaffold.test_scaffold(engine, " - "scaffold), not a DexEngine method" + "two modes on one verb: --mutate is DexEngine.test_mutations, while " + "--scaffold is reachable as " + "exmergo_dex_core.transform.commands.test_scaffold(engine, scaffold) " + "and is not a DexEngine method" ), }, ("semantic", "define"): { diff --git a/packages/dex-core/tests/test_engine.py b/packages/dex-core/tests/test_engine.py index cb077771..b2fb0971 100644 --- a/packages/dex-core/tests/test_engine.py +++ b/packages/dex-core/tests/test_engine.py @@ -102,7 +102,8 @@ def test_methods_return_domain_objects_never_envelopes(duckdb_file: Path): #: Subcommands that are not engine methods, each for a reason the reader can -#: check. `test` is `transform test --scaffold`, reached as `test_scaffold`; +#: check. `test` carries two modes and only one of them is a method: `--mutate` +#: is `test_mutations`, while `--scaffold` is reached as `test_scaffold`. #: `semantic *` is spelled `semantic_*`; `demo` writes a warehouse and is not a #: command a library caller drives. Everything else must have a method. _NOT_ENGINE_METHODS = { diff --git a/packages/dex-core/tests/test_errors.py b/packages/dex-core/tests/test_errors.py index 7ff9fdca..128f6118 100644 --- a/packages/dex-core/tests/test_errors.py +++ b/packages/dex-core/tests/test_errors.py @@ -104,6 +104,7 @@ def test_every_refusal_the_engine_defines_roots_on_dex_error(): "DevTargetError", "EditValidationError", "InitError", + "MutationError", "PlacementRefusedError", "ProdTargetRefusedError", "PropagationRefusedError", diff --git a/packages/dex-core/tests/test_spend_parity.py b/packages/dex-core/tests/test_spend_parity.py index 78ac3873..f5cf5f4a 100644 --- a/packages/dex-core/tests/test_spend_parity.py +++ b/packages/dex-core/tests/test_spend_parity.py @@ -27,6 +27,7 @@ import importlib import json +import shutil import subprocess from dataclasses import dataclass from datetime import UTC, datetime @@ -252,6 +253,99 @@ def run(argv: list[str]): monkeypatch.setattr(build_module, "_default_runner", fake) +def _route_mutation(monkeypatch, root: Path, project: Path) -> None: + """Make a billed `transform test --mutate` run without a warehouse or dbt. + + Same shape as :func:`_route_build` and for the same reason, with one + addition: this command prices its batch through the adapter's own + ``query_estimate``, so the stub has to answer that, and it settles once per + dbt invocation rather than once per command, which is the property the + ledger assertions below are here to pin. + """ + + from exmergo_dex_core.transform import dev_target + + build_module = importlib.import_module("exmergo_dex_core.transform.build") + store = FilesystemStore(root) + + class StubAdapter: + paradigm = Paradigm.BYTES_SCANNED + name = "bigquery" + dialect = "bigquery" + + def __init__(self): + self.cost_gate = _billed_gate(store, "transform test") + + def query_estimate(self, sql: str) -> float: + return float(MB) + + def close(self): + pass + + monkeypatch.setattr( + DexEngine, "_adapter", lambda self, cmd=None, **kw: StubAdapter() + ) + monkeypatch.setattr(dev_target, "check", lambda *a, **k: []) + + manifest = json.dumps( + { + "metadata": {"project_name": "dex_test"}, + "nodes": { + "model.dex_test.stg_customers": { + "name": "stg_customers", + "unique_id": "model.dex_test.stg_customers", + "package_name": "dex_test", + "language": "sql", + "config": {"materialized": "ephemeral"}, + "depends_on": {"nodes": []}, + "compiled_code": "select id from raw where id > 1", + }, + "test.dex_test.not_null_stg_customers_id.abc": { + "name": "not_null_stg_customers_id", + "unique_id": "test.dex_test.not_null_stg_customers_id.abc", + "resource_type": "test", + "attached_node": "model.dex_test.stg_customers", + "depends_on": {"nodes": ["model.dex_test.stg_customers"]}, + "compiled_code": ( + "with __dbt__cte__stg_customers as (select id from raw) " + "select id from __dbt__cte__stg_customers" + ), + }, + }, + "unit_tests": {}, + } + ) + run_results = json.dumps( + { + "results": [ + { + "unique_id": "test.dex_test.not_null_stg_customers_id.abc", + "status": "pass", + "execution_time": 1.0, + "adapter_response": {"bytes_billed": 3000}, + } + ] + } + ) + + def fake(timeout: float, cwd, env=None): + def run(argv: list[str]): + target_path = Path(argv[argv.index("--target-path") + 1]) + target_path.mkdir(parents=True, exist_ok=True) + (target_path / "manifest.json").write_text(manifest, encoding="utf-8") + if argv[1] == "test": + (target_path / "run_results.json").write_text( + run_results, encoding="utf-8" + ) + return subprocess.CompletedProcess( + args=argv, returncode=0, stdout="", stderr="" + ) + + return run + + monkeypatch.setattr(build_module, "_default_runner", fake) + + def _seed_query_cache(root: Path) -> None: """A cache that already adjudicates `shop.customers`, so `explore query` prices the query itself instead of auto-profiling first. The column signature @@ -380,6 +474,35 @@ def root(name: str) -> Path: capsys, )["data"] + # `transform test --mutate` needs a dbt project under its root too, so it + # gets a copy rather than sharing the build's: one ledger per command is what + # lets a row be attributed to the command that wrote it, and this is the one + # billed command that invokes dbt more than once and so settles more than one + # row per run. + mutation_root = root("transform test") + shutil.copytree(bigquery_project, mutation_root / bigquery_project.name) + roots["transform test"] = mutation_root + with monkeypatch.context() as patch: + _route_mutation(patch, mutation_root, mutation_root / bigquery_project.name) + payloads["transform test"] = _run( + [ + "--repo-root", + str(mutation_root), + "--connector", + "bigquery", + "transform", + "test", + "--mutate", + "stg_customers", + "--max-mutants", + "1", + "--confirm", + "--budget", + BUDGET, + ], + capsys, + )["data"] + fake_bq_client.row_resolver = _scan_resolver check_root = root("maintain-check") diff --git a/packages/dex-core/tests/test_sql_shape.py b/packages/dex-core/tests/test_sql_shape.py index ae51d65a..757a61a7 100644 --- a/packages/dex-core/tests/test_sql_shape.py +++ b/packages/dex-core/tests/test_sql_shape.py @@ -152,6 +152,37 @@ def test_predicates_flatten_across_top_level_ands(): assert len(sql_shape.predicates(tree, "where")) == 3 +def test_set_predicates_round_trips_what_predicates_flattened(): + """The pair has to compose: whatever the reader splits, the writer must put + back unchanged, or a caller that drops one predicate silently rewrites the + rest of the clause too.""" + + tree = _parse("select * from t where a = 1 and b = 2 and c = 3") + sql_shape.set_predicates(tree, "where", sql_shape.predicates(tree, "where")) + assert [sql_shape.text(p) for p in sql_shape.predicates(tree, "where")] == [ + "a = 1", + "b = 2", + "c = 3", + ] + + +def test_set_predicates_removes_the_clause_when_nothing_is_left(): + """An empty WHERE wrapper is not printable, so dropping the last predicate + has to drop the clause itself.""" + + tree = _parse("select * from t where a = 1") + sql_shape.set_predicates(tree, "where", []) + assert "where" not in tree.sql().lower() + + +@pytest.mark.parametrize("clause", ["where", "having", "qualify"]) +def test_set_predicates_rebuilds_each_clause_it_can_read(clause): + tree = _parse(f"select a from t {clause} a = 1 and b = 2") # noqa: S608 + preds = sql_shape.predicates(tree, clause) + sql_shape.set_predicates(tree, clause, [preds[0]]) + assert [sql_shape.text(p) for p in sql_shape.predicates(tree, clause)] == ["a = 1"] + + def test_group_by_reports_its_expressions_as_written(): tree = _parse("select a, count(*) from t group by a") assert sql_shape.group_by(tree) == ["a"] diff --git a/packages/dex-core/tests/transform/test_mutation.py b/packages/dex-core/tests/transform/test_mutation.py new file mode 100644 index 00000000..68b3a1a1 --- /dev/null +++ b/packages/dex-core/tests/transform/test_mutation.py @@ -0,0 +1,495 @@ +"""The defect library: what it plants, what it refuses, and how it reads. + +Everything here is pure. No dbt, no warehouse, no filesystem: a mutation is SQL +in and SQL out, which is what lets the defect taxonomy be checked exhaustively +and in three dialects without spending anything. + +The assertions are deliberately about the *report* as much as the SQL. A mutant +whose defect sentence a reader cannot act on has failed at its job even if the +SQL it generated is perfect, so the wording is tested like any other output. +""" + +from __future__ import annotations + +import pytest + +sqlglot = pytest.importorskip("sqlglot") + +from exmergo_dex_core.transform import mutation # noqa: E402 + +ORDERS = "{{ ref('stg_orders') }}" +CUSTOMERS = "{{ ref('stg_customers') }}" + + +def _parents(dialect: str = "duckdb"): + rendered = { + "duckdb": ('"dev"."main"."stg_orders"', '"dev"."main"."stg_customers"'), + "bigquery": ("`proj`.`dev`.`stg_orders`", "`proj`.`dev`.`stg_customers`"), + "snowflake": ("DEV.MAIN.STG_ORDERS", "DEV.MAIN.STG_CUSTOMERS"), + }[dialect] + return [ + mutation.ParentRelation(rendered=rendered[0], jinja=ORDERS), + mutation.ParentRelation(rendered=rendered[1], jinja=CUSTOMERS), + ] + + +def _prepare(sql: str, dialect: str = "duckdb", parents=None): + return mutation.prepare( + sql, + dialect=dialect, + parents=_parents(dialect) if parents is None else parents, + ) + + +def _operators(batch): + return [m.operator for m in batch.mutants] + + +def _of(batch, operator: str): + return [m for m in batch.mutants if m.operator == operator] + + +# --- restoring references ------------------------------------------------------ + + +@pytest.mark.parametrize( + "dialect,sql", + [ + ("duckdb", 'select a from "dev"."main"."stg_orders"'), + ("bigquery", "select a from `proj`.`dev`.`stg_orders`"), + ("snowflake", "select a from DEV.MAIN.STG_ORDERS"), + ], +) +def test_a_compiled_relation_becomes_the_ref_it_came_from(dialect, sql): + """dbt compiles a ref() into a physical name, and a unit test fixture binds + to the ref. Without this the mutant would read the warehouse in a test that + was meant to be reading fixtures, and would pass for the wrong reason.""" + + prepared = _prepare(sql, dialect) + assert ORDERS in mutation.render(prepared) + assert "stg_orders" not in mutation.render(prepared).replace(ORDERS, "") + + +def test_an_unmatched_relation_is_left_alone_and_said_so(): + """dex would rather read a hardcoded table honestly than invent a reference + the project never declared.""" + + prepared = _prepare('select a from "dev"."main"."not_a_parent"') + assert prepared.refs == {} + assert any("not_a_parent" in note for note in prepared.notes) + + +def test_a_cte_name_is_not_mistaken_for_an_unmatched_relation(): + prepared = _prepare( + 'with base as (select a from "dev"."main"."stg_orders") select * from base' + ) + assert prepared.notes == [] + + +def test_an_inlined_ephemeral_parent_is_stripped_and_restored(): + """dbt hoists an ephemeral parent into the model's own SQL as a CTE. Left + there, the mutant would carry a copy of the parent that no fixture can + replace.""" + + prepared = _prepare( + "with __dbt__cte__stg_orders as (select 1 as a) " + "select a from __dbt__cte__stg_orders", + parents=[ + mutation.ParentRelation( + rendered="__dbt__cte__stg_orders", jinja=ORDERS, ephemeral=True + ) + ], + ) + rendered = mutation.render(prepared) + assert "__dbt__cte__" not in rendered + assert ORDERS in rendered + + +def test_an_inlined_cte_dex_cannot_place_is_refused(): + with pytest.raises(mutation.MutationError, match="cannot match"): + _prepare( + "with __dbt__cte__mystery as (select 1 as a) " + "select a from __dbt__cte__mystery", + parents=[], + ) + + +def test_sql_that_is_not_one_query_is_refused(): + with pytest.raises(mutation.MutationError): + _prepare("create table t as select 1") + + +def test_sql_that_will_not_parse_is_refused(): + with pytest.raises(mutation.MutationError, match="could not parse"): + _prepare("select from where )(") + + +def test_every_literal_segment_is_wrapped_so_dbt_renders_none_of_it(): + """Compiled SQL is data. A `{{` inside a string literal would otherwise be + rendered a second time by dbt, which is both a wrong mutant and a way to make + dbt evaluate text that came out of a warehouse.""" + + prepared = _prepare( + 'select \'{{ this_is_data }}\' as tag from "dev"."main"."stg_orders"' + ) + rendered = mutation.render(prepared) + assert "{% raw %}" in rendered and "{% endraw %}" in rendered + body = rendered.split(ORDERS)[0] + assert body.index("{% raw %}") < body.index("{{ this_is_data }}") + + +def test_sql_dex_cannot_quote_is_refused(): + prepared = _prepare("select 'endraw' as x") + with pytest.raises(mutation.MutationError, match="endraw"): + mutation.render(prepared) + + +def test_the_rendered_model_carries_the_ephemeral_header_last(): + """Appended, not prepended: dbt gives a scalar config key to the last + config() call, so a model with its own materialized= would otherwise win and + the mutant would build a relation.""" + + rendered = mutation.render(_prepare("select 1 as a")) + assert rendered.rstrip().endswith(mutation.EPHEMERAL_HEADER) + assert "ephemeral" in mutation.EPHEMERAL_HEADER + assert "'enforced': false" in mutation.EPHEMERAL_HEADER + assert "access='protected'" in mutation.EPHEMERAL_HEADER + + +# --- the operators ------------------------------------------------------------- + + +def test_a_boundary_comparison_flips_to_its_neighbour(): + batch = mutation.enumerate_mutants(_prepare("select a from t where a > 10")) + mutant = _of(batch, "comparison")[0] + assert mutant.before == "a > 10" and mutant.after == "a >= 10" + assert "includes the edge value" in mutant.defect + + +def test_an_equality_is_not_treated_as_a_boundary(): + """Flipping = to <> is a different defect, and in a join condition it is one + the warehouse rejects at once, so a run would be spent learning nothing.""" + + batch = mutation.enumerate_mutants(_prepare("select a from t where a = 10")) + assert "comparison" not in _operators(batch) + + +@pytest.mark.parametrize("clause", ["where", "having", "qualify"]) +def test_a_predicate_is_dropped_and_negated_in_every_clause_that_filters(clause): + batch = mutation.enumerate_mutants( + _prepare(f"select a from t {clause} a <> 'x' and b > 1") # noqa: S608 + ) + dropped = _of(batch, "predicate_drop") + assert any(m.before == "a <> 'x'" and m.after == "" for m in dropped) + negated = _of(batch, "predicate_negate") + assert any(m.after.startswith("NOT (") for m in negated) + + +def test_dropping_the_only_predicate_removes_the_clause(): + batch = mutation.enumerate_mutants(_prepare("select a from t where a > 1")) + body = _of(batch, "predicate_drop")[0].body + assert "WHERE" not in body.upper() + + +@pytest.mark.parametrize( + "written,expected_before,expected_after", + [ + ("inner join", "inner join", "left join"), + ("left join", "left join", "inner join"), + ], +) +def test_a_join_swaps_type_in_both_directions(written, expected_before, expected_after): + batch = mutation.enumerate_mutants( + _prepare(f"select a from x {written} y on x.id = y.id") # noqa: S608 + ) + mutant = _of(batch, "join_type")[0] + assert mutant.before.startswith(expected_before) + assert mutant.after.startswith(expected_after) + + +def test_a_join_defect_names_the_relation_not_an_internal_placeholder(): + """The placeholder is how a reference survives SQL generation. A finding + that named it would tell the reader nothing they can act on.""" + + batch = mutation.enumerate_mutants( + _prepare( + 'select a from "dev"."main"."stg_orders" o ' + 'inner join "dev"."main"."stg_customers" c on o.cid = c.id' + ) + ) + mutant = _of(batch, "join_type")[0] + assert "__dex_ref" not in mutant.defect + assert "__dex_ref" not in mutant.before + assert "stg_customers" in mutant.defect + + +def test_a_dropped_case_branch_does_not_suggest_accepted_values(): + """The dogfood case: a dropped branch sends its rows to a category that is + still in the allowed list, so `accepted_values` passes and suggesting it + would point the reader at a test they may already have.""" + + batch = mutation.enumerate_mutants( + _prepare("select case when a > 1 then 'x' else 'y' end as b from t") + ) + suggestion = _of(batch, "case_branch")[0].suggested_test + assert "accepted_values" not in suggestion + assert "unit test" in suggestion + + +def test_a_case_branch_is_dropped_and_named_as_a_branch(): + batch = mutation.enumerate_mutants( + _prepare( + "select case when a > 1 then 'x' when a > 2 then 'y' else 'z' end from t" + ) + ) + mutant = _of(batch, "case_branch")[0] + assert mutant.before == "when a > 1 then 'x'" + assert not mutant.before.upper().startswith("CASE") + + +def test_the_only_case_branch_collapses_to_the_default(): + """An empty CASE is not printable, so the expression has to become whatever + it would have fallen through to.""" + + batch = mutation.enumerate_mutants( + _prepare("select case when a > 1 then 'x' else 'z' end as b from t") + ) + mutant = _of(batch, "case_branch")[0] + assert mutant.after == "'z'" + assert "CASE" not in mutant.body.upper() + + +def test_a_case_with_no_default_collapses_to_null(): + batch = mutation.enumerate_mutants( + _prepare("select case when a > 1 then 'x' end as b from t") + ) + assert _of(batch, "case_branch")[0].after.upper() == "NULL" + + +def test_a_division_is_inverted(): + batch = mutation.enumerate_mutants(_prepare("select num / den as rate from t")) + mutant = _of(batch, "division")[0] + assert mutant.after.replace(" ", "") == "den/num" + assert "reciprocal" in mutant.defect + + +def test_a_guarded_division_moves_its_zero_check_with_the_denominator(): + """Snowflake's DIV0 does not survive a parse and a print: it comes back as a + conditional testing the divisor by name. Swapping the operands underneath + that guard would leave it checking a column that is no longer the divisor.""" + + batch = mutation.enumerate_mutants( + _prepare("select DIV0(num, den) as rate from t", "snowflake"), + cap=20, + ) + mutant = _of(batch, "division")[0] + body = mutant.body + assert "den = 0" not in body, body + assert "num = 0" in body, body + + +def test_a_window_frame_bound_shifts_by_one(): + batch = mutation.enumerate_mutants( + _prepare( + "select sum(a) over " + "(order by d rows between 2 preceding and current row) from t" + ) + ) + mutant = _of(batch, "window_frame")[0] + assert "2 became 3" in mutant.defect + + +def test_a_frame_with_no_numeric_bound_is_not_a_site(): + batch = mutation.enumerate_mutants( + _prepare( + "select sum(a) over (order by d rows between unbounded preceding " + "and current row) from t" + ) + ) + assert "window_frame" not in _operators(batch) + + +@pytest.mark.parametrize( + "written,expected", [("sum(a)", "MAX(a)"), ("max(a)", "SUM(a)")] +) +def test_an_aggregate_swaps_with_its_counterpart(written, expected): + batch = mutation.enumerate_mutants( + _prepare(f"select {written} from t group by b") # noqa: S608 + ) + mutant = _of(batch, "aggregate")[0] + assert mutant.after == expected + assert "exactly one row" in mutant.defect + + +# --- ordering, the cap, and scope ---------------------------------------------- + + +def test_the_cap_spreads_across_defect_classes_rather_than_truncating_one(): + """A model with many comparisons and one join must not spend a whole + capped run on comparisons: the join defect is the more expensive bug.""" + + sql = ( + "select a from x inner join y on x.id = y.id " + "where a > 1 and b > 2 and c > 3 and d > 4 and e > 5" + ) + batch = mutation.enumerate_mutants(_prepare(sql), cap=3) + assert len(batch.mutants) == 3 + assert "join_type" in _operators(batch) + assert len(set(_operators(batch))) == 3 + + +def test_a_cap_that_binds_reports_what_it_cut_per_class(): + """A capped run that said nothing about the cap would read as 'everything + was covered', and a total alone cannot tell 'no joins here' from 'the joins + were cut off'.""" + + sql = "select a from t where a > 1 and b > 2 and c > 3" + full = mutation.enumerate_mutants(_prepare(sql)) + capped = mutation.enumerate_mutants(_prepare(sql), cap=2) + assert capped.elided_total == len(full.mutants) - 2 + assert set(capped.elided) <= {"comparison", "predicate_drop", "predicate_negate"} + + +def test_the_cap_can_never_be_raised_above_the_engine_ceiling(): + """Every mutant is a dbt invocation, so the ceiling bounds wall time and + spend; a caller may narrow it and nothing may widen it.""" + + sql = "select a from t where " + " and ".join( # noqa: S608 + f"c{i} > {i}" for i in range(40) + ) + batch = mutation.enumerate_mutants(_prepare(sql), cap=999) + assert len(batch.mutants) == mutation.MAX_MUTANTS + + +def test_mutants_never_compound(): + """Mutant seven has to be the model with one defect, not with seven, or a + survivor says nothing about the defect it is named for.""" + + batch = mutation.enumerate_mutants( + _prepare("select a from t where a > 1 and b > 2") + ) + for mutant in _of(batch, "comparison"): + assert mutant.body.count(">=") == 1 + + +def test_mutant_ids_are_stable_and_ordered(): + batch = mutation.enumerate_mutants( + _prepare("select a from t where a > 1 and b > 2") + ) + assert [m.id for m in batch.mutants] == [ + f"m{i:02d}" for i in range(1, len(batch.mutants) + 1) + ] + + +def test_a_site_is_labelled_with_the_scope_its_author_would_name(): + batch = mutation.enumerate_mutants( + _prepare( + "with base as (select a from t where a > 1) select a from base where a > 2" + ) + ) + scopes = {m.scope for m in _of(batch, "comparison")} + assert scopes == {"base", "(final select)"} + + +def test_a_union_branch_is_named_by_its_position(): + batch = mutation.enumerate_mutants( + _prepare("select a from t where a > 1 union all select a from u where a > 2") + ) + assert any("branch" in m.scope for m in batch.mutants) + + +def test_every_mutant_suggests_the_test_that_would_catch_it(): + """The reader's next action is to write a test, so a finding that stops at + 'this survived' has stopped one step short of useful.""" + + sql = ( + "select case when a > 1 then 'x' else 'y' end as b, sum(a) as t, n / d as r " + "from x inner join y on x.id = y.id where a > 1" + ) + batch = mutation.enumerate_mutants(_prepare(sql)) + assert batch.mutants + for mutant in batch.mutants: + assert mutant.suggested_test + assert mutant.defect.strip() + + +def test_a_model_with_no_mutable_site_generates_nothing(): + batch = mutation.enumerate_mutants(_prepare("select a, b from t")) + assert batch.mutants == [] and batch.considered == 0 + + +# --- verdicts ------------------------------------------------------------------ + + +def test_only_tests_that_passed_at_baseline_can_testify(): + """Counting an already-failing test as a catch would report a suite as + strong precisely because it was broken.""" + + baseline = {"a": "pass", "b": "fail"} + verdict = mutation.classify(baseline, {"a": "pass", "b": "fail"}) + assert verdict.outcome == "survived" + + +def test_a_failing_test_kills_and_is_named(): + verdict = mutation.classify({"a": "pass", "b": "pass"}, {"a": "fail", "b": "pass"}) + assert verdict.outcome == "killed" and verdict.caught_by == ["a"] + + +def test_a_warning_test_kills_but_says_it_only_warned(): + verdict = mutation.classify({"a": "pass"}, {"a": "warn"}) + assert verdict.outcome == "killed" and verdict.warn_only + + +def test_a_mutant_the_warehouse_refused_is_rejected_rather_than_killed(): + """A mutant every test errors on is one a build would have failed on + outright, so it is not evidence the tests would have caught it.""" + + verdict = mutation.classify({"a": "pass"}, {"a": "error"}) + assert verdict.outcome == "rejected" + + +def test_a_run_that_produced_nothing_is_not_run(): + assert mutation.classify({"a": "pass"}, {}).outcome == "not_run" + assert mutation.classify({"a": "pass"}, None).outcome == "not_run" + + +def test_a_run_missing_a_test_that_should_have_reported_is_not_run(): + """A test that vanished from the results was not answered, and reading its + absence as silence would count it as failing to catch the defect.""" + + verdict = mutation.classify({"a": "pass", "b": "pass"}, {"a": "pass"}) + assert verdict.outcome == "not_run" + + +# --- pricing support ----------------------------------------------------------- + + +def test_a_mutant_is_spliced_into_the_test_that_reads_it(): + """Pricing needs the statement the warehouse will actually run: a mutant + that drops a partition predicate scans more than the model it came from.""" + + test_sql = ( + "with __dbt__cte__fct_orders as (select a from t where a > 1) " + "select count(*) from __dbt__cte__fct_orders" + ) + out = mutation.inline_into_test( + test_sql, model_name="fct_orders", body="select a from t", dialect="duckdb" + ) + assert out is not None and "a > 1" not in out + + +def test_splicing_a_test_that_does_not_read_the_model_returns_nothing(): + out = mutation.inline_into_test( + "select 1", model_name="fct_orders", body="select a from t", dialect="duckdb" + ) + assert out is None + + +def test_splicing_unparseable_test_sql_returns_nothing(): + """The caller prices that test at its baseline instead, which is the + partial-floor convention the build estimate already follows.""" + + out = mutation.inline_into_test( + "not sql )(", model_name="fct_orders", body="select 1", dialect="duckdb" + ) + assert out is None diff --git a/packages/dex-core/tests/transform/test_mutation_command.py b/packages/dex-core/tests/transform/test_mutation_command.py new file mode 100644 index 00000000..33ac9c73 --- /dev/null +++ b/packages/dex-core/tests/transform/test_mutation_command.py @@ -0,0 +1,603 @@ +"""`transform test --mutate`: what it refuses for free, what it runs, and where. + +Two halves. The first fakes the dbt subprocess, so the refusals, the argv, the +environment and the budget loop are asserted without a warehouse. The second +runs real dbt against DuckDB, because the properties that matter most here are +properties of dbt's own behaviour: that an ephemeral mutant still gets its tests +run, and that nothing is written to the project or the warehouse while it +happens. Faking dbt cannot establish either. +""" + +from __future__ import annotations + +import hashlib +import importlib +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +pytest.importorskip("sqlglot") + +from exmergo_dex_core.config import DexConfig +from exmergo_dex_core.engine import DexEngine +from exmergo_dex_core.storage import FilesystemStore +from exmergo_dex_core.transform import commands as transform_commands +from exmergo_dex_core.transform import mutation + +MODEL_SQL = """select + o.id, + o.amount, + c.region, + case when o.amount > 100 then 'large' else 'small' end as band +from {{ ref('stg_orders') }} o +inner join {{ ref('stg_customers') }} c on o.cid = c.id +where o.status <> 'cancelled' +""" + + +def _engine(project: Path, connector: str = "duckdb", **kwargs) -> DexEngine: + return DexEngine( + connector=connector, + repo_root=str(project.parent), + store=FilesystemStore(project.parent), + config=DexConfig( + connector=connector, dbt_target="dev", dbt_project_dir=project.name + ), + **kwargs, + ) + + +def _skip_dev_target_check(monkeypatch) -> None: + """The dev-target preflight opens a connection; these cases are about what + dbt was asked to do, not about whether the target is deployable.""" + + monkeypatch.setattr( + importlib.import_module("exmergo_dex_core.transform.dev_target"), + "check", + lambda *a, **k: [], + ) + + +def _tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if not path.is_file() or any( + part in {"target", "logs", ".git"} for part in path.parts + ): + continue + digest.update(str(path.relative_to(root)).encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +# --- the faked-dbt half -------------------------------------------------------- + + +@pytest.fixture +def recorded_dbt(monkeypatch): + """Record every dbt argv and answer with artifacts the caller asked for.""" + + build_module = importlib.import_module("exmergo_dex_core.transform.build") + calls: list[dict] = [] + plan: dict[str, object] = {"manifest": {}, "results": [], "returncode": 0} + + def fake(timeout, cwd, env=None): + def run(argv: list[str]): + calls.append({"argv": argv, "cwd": Path(cwd), "env": env or {}}) + target_path = Path(argv[argv.index("--target-path") + 1]) + target_path.mkdir(parents=True, exist_ok=True) + (target_path / "manifest.json").write_text(json.dumps(plan["manifest"])) + if argv[1] == "test": + (target_path / "run_results.json").write_text( + json.dumps({"results": plan["results"]}) + ) + return subprocess.CompletedProcess( + args=argv, returncode=plan["returncode"], stdout="", stderr="" + ) + + return run + + monkeypatch.setattr(build_module, "_default_runner", fake) + return calls, plan + + +def _manifest(materialized: str = "ephemeral", *, tests=True, **node_overrides): + nodes = { + "model.p.fct": { + "name": "fct", + "unique_id": "model.p.fct", + "package_name": "p", + "language": "sql", + "config": {"materialized": materialized}, + "depends_on": {"nodes": ["model.p.stg_orders"]}, + "compiled_code": 'select a from "d"."m"."stg_orders" where a > 1', + **node_overrides, + }, + "model.p.stg_orders": { + "name": "stg_orders", + "unique_id": "model.p.stg_orders", + "package_name": "p", + "config": {"materialized": "view"}, + "relation_name": '"d"."m"."stg_orders"', + }, + } + if tests: + nodes["test.p.not_null_fct_a.abc123"] = { + "name": "not_null_fct_a", + "unique_id": "test.p.not_null_fct_a.abc123", + "resource_type": "test", + "attached_node": "model.p.fct", + "depends_on": {"nodes": ["model.p.fct"]}, + "compiled_code": "select 1", + } + return {"metadata": {"project_name": "p"}, "nodes": nodes, "unit_tests": {}} + + +def _results(status: str = "pass"): + return [ + { + "unique_id": "test.p.not_null_fct_a.abc123", + "status": status, + "execution_time": 0.1, + } + ] + + +@pytest.fixture +def project(dbt_project_dir: Path) -> Path: + (dbt_project_dir / "models" / "staging" / "fct.sql").write_text( + MODEL_SQL, encoding="utf-8" + ) + return dbt_project_dir + + +def test_it_runs_dbt_test_and_never_build_or_run(project, recorded_dbt, monkeypatch): + """A build runs a model's unit tests before the model, so one failing unit + test skips the model and the skip cascades onto every data test attached to + it. The run would then report tests as skipped with no way to tell which + would have caught the defect. `dbt test` also executes no materialization, + so nothing can write a relation.""" + + calls, plan = recorded_dbt + plan["manifest"] = _manifest() + plan["results"] = _results() + _skip_dev_target_check(monkeypatch) + transform_commands.test_mutations(_engine(project), "fct") + verbs = {call["argv"][1] for call in calls} + assert verbs <= {"compile", "test", "parse", "deps"} + assert "build" not in verbs and "run" not in verbs + + +def test_dbt_writes_into_the_copy_while_cwd_stays_at_the_real_project( + project, recorded_dbt, monkeypatch +): + """dbt resolves target/ and logs/ against --project-dir, and dbt-duckdb + resolves a relative profile path against the process cwd. Only this split + keeps the artifacts in the copy and the warehouse reachable.""" + + calls, plan = recorded_dbt + plan["manifest"] = _manifest() + plan["results"] = _results() + _skip_dev_target_check(monkeypatch) + transform_commands.test_mutations(_engine(project), "fct") + for call in calls: + argv = call["argv"] + assert call["cwd"] == project.resolve() + assert Path(argv[argv.index("--project-dir") + 1]) != project.resolve() + for flag in ("--target-path", "--log-path"): + assert project.resolve() not in Path(argv[argv.index(flag) + 1]).parents + + +def test_the_selection_flags_are_pinned_rather_than_inherited( + project, recorded_dbt, monkeypatch +): + """`DBT_INDIRECT_SELECTION=cautious` in a caller's environment would drop the + tests from the selection. Every mutant would then survive and the report + would call the suite weak when it was never asked.""" + + calls, plan = recorded_dbt + plan["manifest"] = _manifest() + plan["results"] = _results() + monkeypatch.setenv("DBT_INDIRECT_SELECTION", "cautious") + monkeypatch.setenv("DBT_TARGET_PATH", "/elsewhere/target") + _skip_dev_target_check(monkeypatch) + transform_commands.test_mutations(_engine(project), "fct") + for call in (c for c in calls if c["argv"][1] == "test"): + argv = call["argv"] + assert argv[argv.index("--indirect-selection") + 1] == "eager" + assert "--no-defer" in argv and "--no-fail-fast" in argv + assert "DBT_INDIRECT_SELECTION" not in call["env"] + assert "DBT_TARGET_PATH" not in call["env"] + assert call["env"]["DO_NOT_TRACK"] == "1" + + +@pytest.mark.parametrize( + "manifest,message", + [ + (_manifest(tests=False), "has no tests"), + (_manifest(language="python"), "Python model"), + (_manifest("table"), "could not make"), + ( + _manifest(config={"materialized": "ephemeral", "sql_header": "set x=1"}), + "sql_header", + ), + ], +) +def test_a_model_dex_will_not_mutate_is_refused_before_any_test_runs( + project, recorded_dbt, monkeypatch, manifest, message +): + """Every one of these is answered from the parse, so a model dex cannot + measure costs nothing to ask about.""" + + calls, plan = recorded_dbt + plan["manifest"] = manifest + plan["results"] = _results() + _skip_dev_target_check(monkeypatch) + with pytest.raises(mutation.MutationError, match=message): + transform_commands.test_mutations(_engine(project), "fct") + assert not [c for c in calls if c["argv"][1] == "test"] + + +def test_a_cap_above_the_ceiling_is_refused_without_touching_dbt(project, recorded_dbt): + calls, _ = recorded_dbt + with pytest.raises(ValueError, match="above the engine ceiling"): + transform_commands.test_mutations(_engine(project), "fct", max_mutants=999) + assert calls == [] + + +def test_a_suite_with_nothing_passing_stops_rather_than_measuring_it( + project, recorded_dbt, monkeypatch +): + """Every verdict is relative to what passed before anything was mutated, so + a suite with nothing passing has nothing that could catch a defect.""" + + _calls, plan = recorded_dbt + plan["manifest"] = _manifest() + plan["results"] = _results("fail") + _skip_dev_target_check(monkeypatch) + with pytest.raises(mutation.MutationError, match="passes against the unmutated"): + transform_commands.test_mutations(_engine(project), "fct") + + +def test_run_hooks_are_stripped_from_the_copy_and_said_so( + project, recorded_dbt, monkeypatch +): + """A hook fires once per invocation and this invokes dbt once per mutant, so + a hook that grants or audits would fire N+1 times for a command the caller + thinks of as read only.""" + + _calls, plan = recorded_dbt + plan["manifest"] = _manifest() + plan["results"] = _results() + manifest_path = project / "dbt_project.yml" + manifest_path.write_text( + manifest_path.read_text() + 'on-run-start:\n - "select 1"\n', encoding="utf-8" + ) + _skip_dev_target_check(monkeypatch) + result = transform_commands.test_mutations(_engine(project), "fct") + assert any("hooks are not run" in w for w in result.warnings) + assert "on-run-start" in manifest_path.read_text() + + +# --- the real-dbt half --------------------------------------------------------- + + +@pytest.fixture +def measurable_project(tmp_path: Path, duckdb_file: Path) -> Path: + """A project whose one mart has a suite worth measuring, on real DuckDB.""" + + duckdb = pytest.importorskip("duckdb") + warehouse = tmp_path / "dev.duckdb" + shutil.copy(duckdb_file, warehouse) + con = duckdb.connect(str(warehouse)) + con.execute( + "create or replace table main.raw_orders as select * from (values " + "(1, 7, 100.0::double, 'placed'), (2, 7, 10.0::double, 'placed'), " + "(3, 8, 20.0::double, 'cancelled')) t(id, cid, amount, status)" + ) + con.execute( + "create or replace table main.raw_customers as select * from (values " + "(7, 'eu'), (8, 'us')) t(id, region)" + ) + con.close() + + project = tmp_path / "analytics" + (project / "models").mkdir(parents=True) + (project / "dbt_project.yml").write_text( + 'name: p\nversion: "1.0.0"\nprofile: p\nmodel-paths: ["models"]\n', + encoding="utf-8", + ) + (project / "profiles.yml").write_text( + f"p:\n target: dev\n outputs:\n dev:\n type: duckdb\n" + f" path: {warehouse}\n", + encoding="utf-8", + ) + (project / "models" / "stg_orders.sql").write_text( + "select id, cid, amount, status from main.raw_orders", encoding="utf-8" + ) + (project / "models" / "stg_customers.sql").write_text( + "select id, region from main.raw_customers", encoding="utf-8" + ) + (project / "models" / "fct.sql").write_text(MODEL_SQL, encoding="utf-8") + return project + + +def _schema_yml(unit_test: bool) -> str: + base = """version: 2 +models: + - name: fct + columns: + - name: id + data_tests: [not_null] +""" + if not unit_test: + return base + return ( + base + + """ +unit_tests: + - name: fct_pins_its_rules + model: fct + given: + - input: ref('stg_orders') + rows: + - {id: 1, cid: 7, amount: 100.0, status: 'placed'} + - {id: 2, cid: 7, amount: 10.0, status: 'placed'} + - {id: 3, cid: 8, amount: 20.0, status: 'cancelled'} + - {id: 4, cid: 99, amount: 30.0, status: 'placed'} + - input: ref('stg_customers') + rows: + - {id: 7, region: 'eu'} + expect: + rows: + - {id: 1, region: 'eu', band: 'small'} + - {id: 2, region: 'eu', band: 'small'} +""" + ) + + +@pytest.mark.parametrize("with_unit_test", [False, True]) +def test_a_stronger_suite_catches_more_planted_defects( + measurable_project: Path, with_unit_test: bool +): + """The acceptance the issue asks for, both directions in one test: a model + carrying only a `not_null` lets the defects through, and adding one unit test + that pins the model's actual rules catches most of them.""" + + pytest.importorskip("dbt.adapters.duckdb") + (measurable_project / "models" / "schema.yml").write_text( + _schema_yml(with_unit_test), encoding="utf-8" + ) + engine = _engine(measurable_project) + engine.build(target="dev") + + result = engine.test_mutations("fct") + counts = result.counts + assert counts["generated"] >= 4 + if with_unit_test: + assert counts["killed"] > counts["survived"], result.mutants + assert result.score > 0.5 + else: + assert counts["killed"] == 0 + assert counts["survived"] == counts["generated"] + assert result.score == 0.0 + + +def test_every_kind_of_test_gets_to_answer_for_the_mutant(measurable_project: Path): + """Generic, singular and unit tests all run against an ephemeral mutant. If + any kind were silently dropped, its defects would all read as survivors.""" + + pytest.importorskip("dbt.adapters.duckdb") + (measurable_project / "models" / "schema.yml").write_text( + _schema_yml(True), encoding="utf-8" + ) + (measurable_project / "tests").mkdir() + (measurable_project / "tests" / "no_null_region.sql").write_text( + "select id from {{ ref('fct') }} where region is null", encoding="utf-8" + ) + engine = _engine(measurable_project) + engine.build(target="dev") + + result = engine.test_mutations("fct") + names = {test["name"] for test in result.baseline["tests"]} + assert "not_null_fct_id" in names + assert "no_null_region" in names + assert "fct_pins_its_rules" in names + + +def test_a_mutation_run_writes_nothing_and_materializes_nothing( + measurable_project: Path, +): + """The two guarantees the safety spine turns on: the project is a byte for + byte match afterwards, and the dev warehouse gained no relation.""" + + duckdb = pytest.importorskip("duckdb") + pytest.importorskip("dbt.adapters.duckdb") + (measurable_project / "models" / "schema.yml").write_text( + _schema_yml(True), encoding="utf-8" + ) + engine = _engine(measurable_project) + engine.build(target="dev") + + warehouse = measurable_project.parent / "dev.duckdb" + + def relations(): + con = duckdb.connect(str(warehouse), read_only=True) + try: + return set( + con.execute( + "select table_schema, table_name from information_schema.tables" + ).fetchall() + ) + finally: + con.close() + + before_tree, before_relations = _tree_digest(measurable_project), relations() + engine.test_mutations("fct") + + assert _tree_digest(measurable_project) == before_tree + assert relations() == before_relations + + +def test_the_cap_narrows_the_run_and_reports_what_it_cut(measurable_project: Path): + pytest.importorskip("dbt.adapters.duckdb") + (measurable_project / "models" / "schema.yml").write_text( + _schema_yml(False), encoding="utf-8" + ) + engine = _engine(measurable_project) + engine.build(target="dev") + + result = engine.test_mutations("fct", max_mutants=2) + assert result.counts["generated"] == 2 + assert result.cap["limit"] == 2 + assert sum(result.cap["elided"].values()) > 0 + assert any("not run" in w and "cap" in w for w in result.warnings) + + +def test_the_batch_reports_one_spend_rather_than_a_sum_of_flags(): + """A spend payload is not uniformly additive, and treating it that way is + how nine runs reported `settled: 9`. + + What each run billed sums. The day's cumulative total is already cumulative, + so summing it reports the day nine times over. The two flags are claims about + the command as a whole: it settled only if every run did, and its settlement + is unknown if any run's was. + """ + + from exmergo_dex_core.transform.commands import _merge_spend + + first = { + "bytes_billed": 100.0, + "session_spent_today": 1_000.0, + "settled": True, + "unknown_settlement": False, + "reserved": None, + } + second = { + "bytes_billed": 50.0, + "session_spent_today": 1_050.0, + "settled": True, + "unknown_settlement": False, + "reserved": None, + } + merged = _merge_spend(_merge_spend(None, first), second) + assert merged["bytes_billed"] == 150.0 + assert merged["session_spent_today"] == 1_050.0 + assert merged["settled"] is True + assert merged["unknown_settlement"] is False + + +def test_one_run_with_unknown_spend_makes_the_batch_unknown(): + """Known plus unknown is unknown, so the flags cannot be averaged away by a + majority of well-behaved runs.""" + + from exmergo_dex_core.transform.commands import _merge_spend + + known = {"bytes_billed": 100.0, "settled": True, "unknown_settlement": False} + unknown = {"bytes_billed": None, "settled": False, "unknown_settlement": True} + merged = _merge_spend(_merge_spend(None, known), unknown) + assert merged["settled"] is False + assert merged["unknown_settlement"] is True + + +def test_the_run_stops_when_the_confirmed_budget_runs_out( + project, recorded_dbt, monkeypatch +): + """The batch is priced upfront, but a run can still outrun its estimate, and + the guard has to bind on what was actually spent rather than on what was + predicted. + + Stopping and reporting the remainder as `not_run` is the honest outcome: a + shorter list of survivors read as a clean bill would be the one way this + command could mislead about cost and coverage at once. + """ + + from exmergo_dex_core.envelope import Paradigm + from exmergo_dex_core.transform import mutation + from exmergo_dex_core.transform.commands import _run_mutants + + class _Shadow: + """Answers every run with spend that overshoots the per-run estimate.""" + + def __init__(self): + self.runs = 0 + + def write(self, *args, **kwargs): + pass + + def test(self, model, **kwargs): + self.runs += 1 + return { + "success": True, + "nodes": [ + { + "unique_id": "test.p.not_null.abc", + "name": "not_null", + "status": "pass", + "execution_time": 40.0, + } + ], + } + + prepared = mutation.prepare( + "select a from t where a > 1 and b > 2 and c > 3", dialect="duckdb" + ) + batch = mutation.enumerate_mutants(prepared) + shadow = _Shadow() + result = _run_mutants( + shadow, + "models/staging/fct.sql", + batch, + model="fct", + paradigm=Paradigm.COMPUTE_TIME, + connector="snowflake", + store=FilesystemStore(project.parent), + ceiling=100.0, + estimate=90.0, + mutation_mod=mutation, + ) + _runs, _spend, warnings = result.pop("_meta") + + assert result["counts"]["not_run"] > 0 + assert shadow.runs < len(batch.mutants) + 1, "the run kept going past the budget" + assert any("budget covered" in w for w in warnings) + statuses = {m["id"]: m["status"] for m in result["mutants"]} + assert "not_run" in statuses.values() + + +def test_the_days_total_excludes_the_reservation_the_command_is_still_holding(): + """A reservation is headroom, not spend, and this command holds one across + every run in the batch. + + Each run settles its own row while that reservation still stands, so a day's + total read during the loop counts the whole batch estimate on top of what the + runs actually billed. `transform build` releases before it reads for the same + reason; the difference here is that the overstatement is the entire estimate + rather than a rounding error. + """ + + from exmergo_dex_core.envelope import Paradigm + from exmergo_dex_core.transform.commands import _refresh_session_total + + class _Gate: + def __init__(self): + self.settled = False + + def settle(self): + self.settled = True + + class _Store: + def spend_since(self, cutoff, *, field, connector): + return 6_000.0 + + gate = _Gate() + spend = {"bytes_billed": 6_000.0, "session_spent_today": 2_103_152.0} + refreshed = _refresh_session_total( + spend, gate, _Store(), Paradigm.BYTES_SCANNED, "bigquery" + ) + assert gate.settled, "the reservation has to be released before the read" + assert refreshed["session_spent_today"] == 6_000.0 + assert refreshed["bytes_billed"] == 6_000.0, "what the runs billed is unchanged" From 03c240d60c2ae8f8ac866206ee3cae5596a8e76e Mon Sep 17 00:00:00 2001 From: Marco Ciavarella Date: Thu, 10 Sep 2026 20:07:43 +0200 Subject: [PATCH 3/5] Add safety_spine test for mutations --- packages/dex-core/tests/test_safety_spine.py | 263 +++++++++++++++++++ 1 file changed, 263 insertions(+) diff --git a/packages/dex-core/tests/test_safety_spine.py b/packages/dex-core/tests/test_safety_spine.py index 3d20ffe2..9bc5d801 100644 --- a/packages/dex-core/tests/test_safety_spine.py +++ b/packages/dex-core/tests/test_safety_spine.py @@ -9,8 +9,10 @@ from __future__ import annotations +import importlib import json import shutil +import subprocess from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace @@ -674,6 +676,161 @@ def refuse(*args, **kwargs): assert verification.findings == [] and verification.offer is None +class _EstimatingAdapter: + """The smallest adapter that can be asked what a statement will cost. + + Deliberately not a connector's real adapter: the assertion is about the + order dex does things in, and a fake that prices every statement at a fixed + magnitude makes the handshake fire without any warehouse in the picture. + """ + + def __init__(self, connector: str): + from exmergo_dex_core.connect import paradigm_for + from exmergo_dex_core.guards.cost_guard import CostGate + + self.connector = connector + self.dialect = "duckdb" + self.paradigm = paradigm_for(connector, DexConfig(connector=connector)) + self.cost_gate = CostGate( + paradigm=self.paradigm, + ceiling=None, + session_ceiling=None, + session_spent=0.0, + confirmed=False, + connector=connector, + command="transform test", + ) + + def query_estimate(self, sql: str) -> float: + return 1_000.0 + + def close(self) -> None: + pass + + +@pytest.mark.parametrize( + "connector", + ["bigquery", "snowflake", "databricks", "redshift", "postgres", "clickhouse"], +) +def test_mutation_coverage_prices_its_whole_batch_before_running_any_of_it( + connector, dbt_project_dir: Path, monkeypatch +): + """N dbt runs behind one handshake, and none of them before it. + + This is the largest thing dex can be asked to run: one dbt invocation per + mutant, each of them a real query against the dev target. The cost rule is + the same as everywhere else, but the stakes are multiplied, so the ordering + matters more: the batch is priced and confirmed as one number, and an + unconfirmed call must execute nothing at all. A per-mutant ask would be + worse than useless, since the caller would answer twenty times for one + question and could not see the total before the first run. + """ + + from exmergo_dex_core.guards.cost_guard import ConfirmationRequiredError + from exmergo_dex_core.transform import commands as transform_commands + + (dbt_project_dir / "models" / "staging" / "fct.sql").write_text( + "select o.id, o.amount from {{ ref('stg_customers') }} o where o.amount > 1\n", + encoding="utf-8", + ) + build_module = importlib.import_module("exmergo_dex_core.transform.build") + invoked: list[str] = [] + + def fake_runner(timeout, cwd, env=None): + def run(argv): + invoked.append(argv[1]) + target_path = Path(argv[argv.index("--target-path") + 1]) + target_path.mkdir(parents=True, exist_ok=True) + (target_path / "manifest.json").write_text( + json.dumps( + { + "metadata": {"project_name": "dex_test"}, + "nodes": { + "model.dex_test.fct": { + "name": "fct", + "unique_id": "model.dex_test.fct", + "package_name": "dex_test", + "language": "sql", + "config": {"materialized": "ephemeral"}, + "depends_on": {"nodes": []}, + "compiled_code": ( + "select id, amount from raw where amount > 1" + ), + }, + "test.dex_test.not_null_fct_id.abc": { + "name": "not_null_fct_id", + "unique_id": "test.dex_test.not_null_fct_id.abc", + "resource_type": "test", + "attached_node": "model.dex_test.fct", + "depends_on": {"nodes": ["model.dex_test.fct"]}, + "compiled_code": ( + "with __dbt__cte__fct as (select id from raw) " + "select id from __dbt__cte__fct" + ), + }, + }, + "unit_tests": {}, + } + ) + ) + return subprocess.CompletedProcess( + args=argv, returncode=0, stdout="", stderr="" + ) + + return run + + monkeypatch.setattr(build_module, "_default_runner", fake_runner) + monkeypatch.setattr( + importlib.import_module("exmergo_dex_core.transform.dev_target"), + "check", + lambda *a, **k: [], + ) + engine = DexEngine( + connector=connector, + repo_root=str(dbt_project_dir.parent), + store=FilesystemStore(dbt_project_dir.parent), + config=DexConfig( + connector=connector, dbt_target="dev", dbt_project_dir=dbt_project_dir.name + ), + ) + monkeypatch.setattr( + DexEngine, "_adapter", lambda self, command=None: _EstimatingAdapter(connector) + ) + + with pytest.raises(ConfirmationRequiredError): + transform_commands.test_mutations(engine, "fct") + # Compiling to find the defects is free and never executes the model; what + # must not have happened is a test run, which is what spends. + assert "test" not in invoked + + +@pytest.mark.parametrize( + "connector", + ["bigquery", "snowflake", "databricks", "redshift", "postgres", "clickhouse"], +) +def test_mutation_coverage_refuses_a_cap_above_the_engine_ceiling( + connector, dbt_project_dir: Path, monkeypatch +): + """The ceiling is a cost boundary, so a flag may narrow it and never widen + it, which is the same rule `--scope` follows against a committed allowlist. + Refused before anything opens a connection.""" + + from exmergo_dex_core.transform import commands as transform_commands + + def refuse(*args, **kwargs): + raise AssertionError("a refused cap opened a connection") + + monkeypatch.setattr(DexEngine, "_adapter", refuse) + engine = DexEngine( + connector=connector, + repo_root=str(dbt_project_dir.parent), + store=FilesystemStore(dbt_project_dir.parent), + config=DexConfig(connector=connector), + ) + with pytest.raises(ValueError, match="above the engine ceiling"): + transform_commands.test_mutations(engine, "fct", max_mutants=10_000) + + def test_build_verification_findings_never_become_errors( dbt_project_dir: Path, monkeypatch ): @@ -1970,6 +2127,112 @@ def test_changes_are_diffs_not_silent_writes(dbt_project_dir: Path): assert not new_model.exists() +def test_a_mutant_is_never_written_into_the_project(dbt_project_dir: Path, monkeypatch): + """Mutation coverage deliberately produces broken SQL, so where that SQL is + allowed to exist is the whole safety question. + + Every mutant lives in a throwaway copy and dies with it. The project keeps + the bytes it had, and no file anywhere under it carries a mutated statement, + including after a run that was interrupted partway. + """ + + from exmergo_dex_core.transform import commands as transform_commands + + model = dbt_project_dir / "models" / "staging" / "fct.sql" + model.write_text( + "select id, amount from {{ ref('stg_customers') }} where amount > 1\n", + encoding="utf-8", + ) + before = { + path: path.read_bytes() + for path in dbt_project_dir.rglob("*") + if path.is_file() and "target" not in path.parts + } + + build_module = importlib.import_module("exmergo_dex_core.transform.build") + seen_mutants: list[str] = [] + + def fake_runner(timeout, cwd, env=None): + def run(argv): + shadow = Path(argv[argv.index("--project-dir") + 1]) + mutant = shadow / "models" / "staging" / "fct.sql" + if mutant.is_file(): + seen_mutants.append(mutant.read_text()) + target_path = Path(argv[argv.index("--target-path") + 1]) + target_path.mkdir(parents=True, exist_ok=True) + (target_path / "manifest.json").write_text( + json.dumps( + { + "metadata": {"project_name": "dex_test"}, + "nodes": { + "model.dex_test.fct": { + "name": "fct", + "unique_id": "model.dex_test.fct", + "package_name": "dex_test", + "language": "sql", + "config": {"materialized": "ephemeral"}, + "depends_on": {"nodes": []}, + "compiled_code": ( + "select id, amount from raw where amount > 1" + ), + }, + "test.dex_test.not_null_fct_id.abc": { + "name": "not_null_fct_id", + "unique_id": "test.dex_test.not_null_fct_id.abc", + "resource_type": "test", + "attached_node": "model.dex_test.fct", + "depends_on": {"nodes": ["model.dex_test.fct"]}, + }, + }, + "unit_tests": {}, + } + ) + ) + if argv[1] == "test": + (target_path / "run_results.json").write_text( + json.dumps( + { + "results": [ + { + "unique_id": "test.dex_test.not_null_fct_id.abc", + "status": "pass", + "execution_time": 0.0, + } + ] + } + ) + ) + return subprocess.CompletedProcess( + args=argv, returncode=0, stdout="", stderr="" + ) + + return run + + monkeypatch.setattr(build_module, "_default_runner", fake_runner) + monkeypatch.setattr( + importlib.import_module("exmergo_dex_core.transform.dev_target"), + "check", + lambda *a, **k: [], + ) + engine = DexEngine( + connector="duckdb", + repo_root=str(dbt_project_dir.parent), + store=FilesystemStore(dbt_project_dir.parent), + config=DexConfig( + connector="duckdb", dbt_target="dev", dbt_project_dir=dbt_project_dir.name + ), + ) + transform_commands.test_mutations(engine, "fct") + + # The mutants were real and they were written somewhere other than here. + assert any(">=" in text for text in seen_mutants) + for path, content in before.items(): + assert path.read_bytes() == content, f"{path} changed" + assert model.read_text() == ( + "select id, amount from {{ ref('stg_customers') }} where amount > 1\n" + ) + + def test_a_house_convention_warns_and_never_imposes(dbt_project_dir: Path): """The one plan-time check that judges style rather than fact. From 1850457f0a5eedfd4028795ce410d6fc95884d97 Mon Sep 17 00:00:00 2001 From: Marco Ciavarella Date: Thu, 10 Sep 2026 20:07:59 +0200 Subject: [PATCH 4/5] Update agent-facing docs --- AGENTS.md | 4 ++- references/bigquery.md | 14 ++++++++ references/clickhouse.md | 7 ++++ references/command-contract.md | 60 ++++++++++++++++++++++++++++++- references/databricks.md | 7 ++++ references/duckdb.md | 8 +++++ references/methodology.md | 30 ++++++++++++++++ references/postgres.md | 7 ++++ references/redshift.md | 7 ++++ references/snowflake.md | 11 ++++++ skills/transform/SKILL.md | 32 +++++++++++++++++ skills/transform/evals/evals.json | 13 +++++++ 12 files changed, 198 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9b4142a7..bf35d749 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,7 @@ credentials and no network. | `transform remove [--edits-file ]` | the definition removed, and every read verified gone. Same kinds and same refusals as `rename`. dex authors the removal of the **definition** and refuses while any read survives, naming each with a file and a line; it never rewrites a read, because `{% if var('flag') %}` can be dropped or unguarded and `{{ var('x') }}` in an expression has no value dex may invent, and only you know which. Author those read edits yourself and pass them with `--edits-file`: they are validated and stored in this same plan, so the removal stays atomic without dex guessing at semantics | | `transform place --targets --expr "" [--explain]` | where a derived column that several models need should be *defined*: the lowest model in the `ref()` graph that every target descends from and that already projects the inputs the expression reads. The inputs are parsed out of `--expr`, so they cannot disagree with the expression using them. Defines the column there and threads it down every chain, with a `schema.yml` entry at the ancestor and at each target and none at the hops in between. `data.reasoning` names the ancestor, why it is the lowest, which targets descend from it, and the chain, because a proposal you cannot argue with is an imposition. Where there is no common ancestor, where the lowest one lacks an input (dex will not hunt further upstream to pull one down, because that turns one placement into an unbounded rewrite of the graph above it), or where two candidates tie, `data.strategy` is `per_target` and the reason is stated rather than the worse thing being done quietly. `--explain` returns the reasoning and stores no plan. Repo-only and free | | `transform test --scaffold ` | plan a `unit_tests:` skeleton for the model: a `given` block per `ref()`/`source()` input carrying only the columns the model reads, typed from the exploration cache, and an `expect:` stub that fails until it is filled in (dbt-parse-checked; applied with `transform apply`) | +| `transform test --mutate [--max-mutants ]` | measure the tests the model already has. Plants one standard analytics defect at a time in the model's compiled SQL (a flipped boundary, a dropped or negated filter, a swapped join type, a removed `CASE` branch, an inverted ratio, a shifted window frame, `sum` for `max`), runs the model's own tests against each, and reports which defects nothing caught, each with the test that would catch it. Every mutant builds as an ephemeral model in a throwaway copy of the project, so nothing is written to your tree and no relation is created, replaced or dropped. Dev-target only. Capped at 20 mutants, with `--max-mutants` able to narrow and never widen, and whatever the cap cuts reported per defect class. Refused for free, before any connection, when the model has no tests, is a Python model, belongs to a package, or has no mutable SQL. On a metered connector the whole batch is priced as one number and confirmed once, and a budget that runs out stops the run with the remainder reported as `not_run` | | `transform macro [name]` | no name lists the shipped dbt macros; a name proposes scaffolding it into the project's macro directory as a plan (dbt-parse-checked, applied with `transform apply`); re-running diffs the project's copy against the shipped version | | `transform build --target dev [--verify] [--for-plan \|--for-plan-file ] [--no-install-deps]` | prod-looking targets refused outright; then a free dev-target preflight (refuses when `.dex/config.yml` and the rendered `profiles.yml` disagree, or when the dev database does not exist, naming the fix); then the cost preflight, priced upfront by a free `dbt compile` dry-run of each node (a partial floor when a cold dev target has not built a node's inputs yet; degrades to no estimate when dex cannot open its own connection); runs only with `--confirm` and a budget; cwd pinned to the project dir; auto-runs `dbt deps` when packages are declared but not installed. Each node in `data.nodes` carries dbt's `unique_id` beside a readable `name`, because a generic test's id ends in a content hash and a name taken from the last segment reads as `3249b83c15`, which is no use to a caller asking which of seventeen tests warned. `--verify` runs the `maintain verify` sweep over the nodes this build touched and reports it under `data.verification`, in the same envelope: `ran` is always present (a build that did not verify and one that verified and found nothing are different answers), and a run that swept adds `scope`, `findings`, `finding_count` and `suppressed`. Opt-in on every connector, free ones included. Findings never enter `errors` and never change the status: whether a `row_loss` should gate a pipeline is the caller's policy, and a pointer line in `warnings` names the count so a caller reading warnings alone still learns of them. A build that failed partway still reports the build-status half, which is when naming the failed node and what it took down with it is worth most; row population is suppressed there, and `no_relation` is suppressed always, since dbt's run results are a better authority on what it just built than the catalog is. Cost: the row counts a verdict needs are priced during the build's own pricing pass off the compiled manifest and folded into the same estimate as a `(row counts)` entry, so one `--budget` covers both phases; only a relation the warehouse keeps no count for (any view) costs anything, a cold dev target that cannot be priced upfront says so and is priced again after the build as a phase drawn against the reservation it already holds, and a phase that does not fit returns `ok` with the counts in `data.offer` rather than `needs_confirmation` for a build that is finished and billed. `--verify` folds the connector's dev namespace into its own read scope for the length of the command, since that namespace is refused as a source everywhere else and is exactly what this one is verifying; the widening shows in `connection.target` and nothing is written back to `.dex/config.yml``data.outcome` says what the run *established*, which `success` cannot: `success` is dbt's exit code and an empty selection exits zero, so a build that matched no nodes is `empty_selection`, one that built nothing the change required is `unrelated`, one whose tree no longer matches the plan is `stale`, and `validated` is reserved for a run that built and passed every node the change required. `data.evidence` carries the invocation, the typed per-node statuses, what the selection asked for against what it got, the manifest and run-results digests, the relations generated, and the principal errors. `--for-plan` (a stored plan id) or `--for-plan-file` (an exported plan document, which is what a build sandbox holds) is what adds `coverage`; without either there is no coverage to report and the key is absent rather than empty. `--no-install-deps` refuses a declared-but-uninstalled package by name instead of installing it, before any subprocess and before pricing, which is the right outcome in a sandbox with no network and costs nothing | | `transform deps` | install/refresh dbt packages (repo-confined; no warehouse spend) | @@ -157,7 +158,8 @@ enters the drift baseline and neither raises a dangling-reference guard on delete. Note that three separate things are called a test: generic tests declared inside a `schema.yml`, unit tests scaffolded by `transform test --scaffold` into a `unit_tests:` block, and the files under -`test-paths` that `test_sql` authors. The semantic commands take a second, +`test-paths` that `test_sql` authors. `transform test --mutate` measures all +three at once, since a defect has to get past every one of them to ship. The semantic commands take a second, narrower payload instead: `--definitions-file ` with `{"definitions": [{"kind", "path", "content"}, ...]}`, where `kind` is `semantic_model` or `metric` and `content` is that one definition's YAML body. diff --git a/references/bigquery.md b/references/bigquery.md index e3e5f4bc..e0103bce 100644 --- a/references/bigquery.md +++ b/references/bigquery.md @@ -215,6 +215,20 @@ command, because dbt writes the relations it is judging there and that namespace is refused as a source everywhere else. The widening shows in the envelope's `connection.target`; nothing is written back to `.dex/config.yml`. +`transform test --mutate` prices the whole batch upfront and confirms it once. +Each mutant is spliced into each of the model's compiled data tests and dry-run +priced as the statement the warehouse will actually run, which matters here more +than on any other connector: dropping or flipping a predicate on a partitioned +table changes what the scan prunes, so a mutant can legitimately cost more than +the model it came from and pricing the batch at the baseline's cost would +under-report it. The breakdown names `(baseline)` and each mutant by id, so one +`--budget` covers the run and the caller sees where it goes. Unit tests read +fixtures rather than the warehouse and are not priced. If the confirmed budget +runs out partway, the run stops and the remaining mutants come back `not_run`. +Nothing is materialized: mutants build as ephemeral models, so no table or view +is created in `bigquery.dev_dataset` and none of the run needs cleaning up. + + With `--layered-schemas`, the scaffolded `generate_schema_name` override makes each layer build into its own sibling dataset in the profile's project diff --git a/references/clickhouse.md b/references/clickhouse.md index c3434959..1fa1691d 100644 --- a/references/clickhouse.md +++ b/references/clickhouse.md @@ -325,6 +325,13 @@ already holding. A phase that does not fit returns `ok` with the counts in `data.offer`, never `needs_confirmation` for a build that has already run and billed. +`transform test --mutate` prices its whole batch as one number and confirms it +once, then runs one dbt invocation per mutant. Nothing is materialized: a mutant +builds as an ephemeral model, so the dev namespace holds exactly the relations it +held before. A budget that runs out partway stops the run, and the remaining +mutants are reported `not_run` rather than the budget being exceeded. + + `--verify` also folds `clickhouse.dev_database` into its read scope for the length of that one command, because dbt writes the relations it is judging there and that database is refused as a source everywhere else. The widening shows in diff --git a/references/command-contract.md b/references/command-contract.md index 409e1ae1..5e458ffc 100644 --- a/references/command-contract.md +++ b/references/command-contract.md @@ -259,6 +259,13 @@ dex transform test --scaffold -> plan a unit_tests: skeleton for model : ref()/source() input with only the columns reads, typed from the exploration cache; expect: is an empty stub that fails until filled in (dbt-parse-checked; apply like any plan) +dex transform test --mutate -> measure the tests already has: plant standard analytics + defects in its SQL one at a time, run its own tests against each, + and report which defects nothing caught. Dev-target only, and + every mutant builds as an ephemeral model in a throwaway copy, so + nothing is written to the project and nothing is materialized. + Capped at 20 mutants; --max-mutants only narrows. On a metered + connector the whole batch is priced and confirmed as one number dex semantic define|update|plan -> dbt semantic model edits as diffs (fronted by transform); validated up to and including dbt's own parser; applied with transform apply like any other plan @@ -403,7 +410,58 @@ built, and costs nothing. inside a `schema.yml` (`data_tests:` on a model or a column). Unit tests are scaffolded by `transform test --scaffold ` into a `unit_tests:` block, also `schema_yml`. Singular tests and generic test *definitions* are files under -`test-paths`, and those are what `test_sql` authors. +`test-paths`, and those are what `test_sql` authors. `transform test --mutate` +measures all three together, because dbt runs all three and a defect only has to +get past every one of them to ship. + +### `transform test --mutate `: what the tests are worth + +A passing suite says the tests ran. It does not say they would notice if the +model were wrong, and no count of tests distinguishes the two. This plants one +defect at a time in the model's compiled SQL and reports which ones nothing +caught. + +The defect classes are the ones that recur in analytics code: a boundary +comparison flipped to include or exclude its edge, a `WHERE` predicate dropped or +negated, an inner join swapped for a left join or the reverse, a `CASE` branch +removed, a ratio inverted, a window frame bound shifted by one, and `sum` +swapped with `max`. Each finding is written as the defect rather than as a diff, +and carries `suggested_test`, because the reader's next action is to write a +test. + +**Where a mutant lives.** In a throwaway copy of the project, and nowhere else. +Each one is written with `materialized='ephemeral'`, so dbt inlines it into each +test as a CTE and materializes nothing: no relation is created, replaced or +dropped, and there is nothing to clean up afterwards. The run uses `dbt test` +rather than `dbt build`, which is what keeps a failing unit test from skipping +the model and cascading that skip onto every data test attached to it. The +project's `on-run-start` and `on-run-end` hooks are stripped from the copy, +since dbt is invoked once per mutant and a hook that grants or audits should not +fire N+1 times; a note says so when it happens. + +**What the verdicts mean.** `killed` is a defect at least one test caught, named +in `caught_by`. `survived` is one nothing caught. `rejected` is one the warehouse +refused outright, kept separate because a build would have failed on it anyway +and counting it as caught would flatter the suite. `not_run` is a mutant the +budget stopped. Every verdict is relative to the tests that passed against the +unmutated model, and any test that did not is listed in `baseline.excluded` with +the reason, so a suite measured against its own broken tests cannot read as +clean. A run where nothing passes at baseline is an error, not a clean sweep. + +**Cost.** Free on DuckDB. On a metered connector each mutant is priced as the +statements the warehouse will actually run, by splicing it into each test's +compiled SQL, because a mutant that drops a partition predicate scans more than +the model it came from. The whole batch is one estimate and one confirmation: +`per_table_bytes` names `(baseline)` and each mutant, so the caller sees the +total and the breakdown before anything executes. If the confirmed budget runs +out partway, the run stops and the remaining mutants are reported `not_run` +rather than the budget being exceeded. Spend settles per run under +`command: "transform test"` in the ledger. + +**The cap.** 20 mutants, and `--max-mutants` may only lower it. Mutants are +ordered round robin across the defect classes, so a capped run on a model with +forty comparisons and one join still tests the join; what the cap cut is reported +in `data.cap.elided`, per class. `seed_csv` is the first kind that puts **values**, not logic, into a reviewable diff, and a diff goes into git and stays there. So a seed's header is checked diff --git a/references/databricks.md b/references/databricks.md index a4651bc6..8e3e9594 100644 --- a/references/databricks.md +++ b/references/databricks.md @@ -107,6 +107,13 @@ a phase drawn against the reservation the build is already holding. A phase that does not fit returns `ok` with the counts in `data.offer`, never `needs_confirmation` for a build that has already run and billed. +`transform test --mutate` prices its whole batch as one number and confirms it +once, then runs one dbt invocation per mutant. Nothing is materialized: a mutant +builds as an ephemeral model, so the dev namespace holds exactly the relations it +held before. A budget that runs out partway stops the run, and the remaining +mutants are reported `not_run` rather than the budget being exceeded. + + `--verify` also folds `databricks.dev_schema` into its read scope for the length of that one command, because dbt writes the relations it is judging there and that namespace is refused as a source everywhere else. The widening shows in the envelope's diff --git a/references/duckdb.md b/references/duckdb.md index 2761491b..9d838935 100644 --- a/references/duckdb.md +++ b/references/duckdb.md @@ -115,6 +115,14 @@ would judge nothing in a default project. There is no dev-namespace fold to make either, since the dev target is a database file and the dev-target preflight already refuses a build whose profile and config disagree about which one. +`transform test --mutate` is free here too, and it is the connector to learn the +command on: every mutant is a full dbt run, so twenty of them against a local +file cost nothing but a few seconds, and no handshake stands between the caller +and the answer. Nothing is materialized either way, because a mutant builds as +an ephemeral model, so the dev database holds exactly the relations it held +before. + + The dev target being the source file is also why `transform init`'s content preflight skips DuckDB's base namespace: "the file already holds objects" is diff --git a/references/methodology.md b/references/methodology.md index 070ebbe8..bbae2f3b 100644 --- a/references/methodology.md +++ b/references/methodology.md @@ -204,6 +204,36 @@ looks: a warehouse keeps no row count for a view, and a view is dbt's default materialization, so on a metered connector the models this can judge for free and the ones it cannot are split down exactly that line. +## Test strength: measuring the tests rather than the data + +Everything above measures data. A dbt project also carries assertions about that +data, and those assertions are themselves unmeasured: a suite of twenty tests +that all pass proves the tests ran, not that any of them would object if the +model were wrong. The two are routinely confused, because the only number the +ecosystem reports is a count, and a count cannot distinguish a `not_null` on a +surrogate key from a unit test pinning the arithmetic. + +The measurement that does distinguish them is to break the model deliberately and +see whether anything complains. `transform test --mutate` plants one defect at a +time, drawn from the same taxonomy the rest of this document is organised around: +a boundary comparison that now includes or excludes its edge, a filter dropped or +inverted, an inner join where a left join was meant, a missing `CASE` branch, an +inverted ratio, a window frame off by one, a `sum` reporting a `max`. Each is +built and the model's own tests are run against it. A defect nothing catches is +reported as a gap in the suite, described as the defect rather than as a diff, +because the reader's next step is to write a test and not to reread SQL they +already know. + +Three properties keep the result honest. Every verdict is relative to the tests +that passed against the *unmutated* model, so a suite measured against its own +already-failing tests cannot come back looking clean. A mutant the warehouse +refuses outright is reported separately from one the tests caught, since a build +would have failed on it anyway and counting it would flatter the suite. And a +mutant that survives is a statement about detection, not about correctness: some +survivors are defects the current data cannot distinguish at all, such as an +inner join where every key happens to match, which is exactly why the finding +names the test that would catch it rather than claiming the model is wrong. + ## The draft map: composing and persisting `explore map` composes the above into the `.dex/` cache (never the source of diff --git a/references/postgres.md b/references/postgres.md index 91ecb721..72fd2ab7 100644 --- a/references/postgres.md +++ b/references/postgres.md @@ -150,6 +150,13 @@ at all, which is any view (dbt's default materialization); a table's count is free catalog metadata, and a verdict resting on it is reported `exact: false` to say so. +`transform test --mutate` prices its whole batch as one number and confirms it +once, then runs one dbt invocation per mutant. Nothing is materialized: a mutant +builds as an ephemeral model, so the dev namespace holds exactly the relations it +held before. A budget that runs out partway stops the run, and the remaining +mutants are reported `not_run` rather than the budget being exceeded. + + `--verify` also folds `postgres.dev_schema` into its read scope for the length of that one command, because dbt writes the relations it is judging there and that namespace is refused as a source everywhere else. The widening shows in the envelope's diff --git a/references/redshift.md b/references/redshift.md index b83d9921..cda48efb 100644 --- a/references/redshift.md +++ b/references/redshift.md @@ -117,6 +117,13 @@ a phase drawn against the reservation the build is already holding. A phase that does not fit returns `ok` with the counts in `data.offer`, never `needs_confirmation` for a build that has already run and billed. +`transform test --mutate` prices its whole batch as one number and confirms it +once, then runs one dbt invocation per mutant. Nothing is materialized: a mutant +builds as an ephemeral model, so the dev namespace holds exactly the relations it +held before. A budget that runs out partway stops the run, and the remaining +mutants are reported `not_run` rather than the budget being exceeded. + + `--verify` also folds `redshift.dev_schema` into its read scope for the length of that one command, because dbt writes the relations it is judging there and that namespace is refused as a source everywhere else. The widening shows in the envelope's diff --git a/references/snowflake.md b/references/snowflake.md index 2b321078..4b7f640f 100644 --- a/references/snowflake.md +++ b/references/snowflake.md @@ -148,6 +148,17 @@ a phase drawn against the reservation the build is already holding. A phase that does not fit returns `ok` with the counts in `data.offer`, never `needs_confirmation` for a build that has already run and billed. +`transform test --mutate` prices the batch in warehouse-seconds, from the same +heuristic the rest of this connector uses rather than from a dry run, so the +estimate carries `estimate_quality: heuristic` and the resume minimum floors it +like any other billed command here. One estimate and one confirmation cover +every mutant; a budget that runs out partway stops the run and reports the rest +as `not_run`. Settlement is the sum of the per-node execution seconds each run +reports, ledgered under `command: "transform test"`. Nothing is materialized, +because a mutant builds as an ephemeral model, so `snowflake.dev_database` holds +exactly the objects it held before the run. + + `--verify` also folds `snowflake.dev_database` / `snowflake.dev_schema` into its read scope for the length of that one command, because dbt writes the relations it is judging there and that namespace is refused as a source everywhere else. The widening shows in the envelope's diff --git a/skills/transform/SKILL.md b/skills/transform/SKILL.md index 3fe6733c..f19c5c7e 100644 --- a/skills/transform/SKILL.md +++ b/skills/transform/SKILL.md @@ -90,6 +90,8 @@ Generic tests are declared inside a `schema.yml` (`data_tests:` on a model or a column). Unit tests come from `transform test --scaffold `, which writes a `unit_tests:` block, also `schema_yml`. Singular tests and generic test *definitions* are files under `test-paths`, and `test_sql` is the kind for those. +`transform test --mutate ` measures all three at once, since a defect has +to get past every one of them to reach production. **A seed puts values, not logic, into a diff, and a diff goes into git and stays there.** So a seed whose header names a column that looks like personal data is @@ -362,6 +364,36 @@ check" note just means no connection was reachable at init time. back `ok` with a `data.offer`, the build is done and billed and the offer buys only the counts it could not afford; relay the number rather than re-running the build. +- **`transform test --mutate ` answers "are these tests worth + anything".** Writing a test is not the same as writing a test that would catch + something, and nothing else in the dbt ecosystem tells the two apart. This + plants one standard analytics defect at a time in the model's SQL (a flipped + boundary, a dropped or negated filter, a swapped join type, a removed `CASE` + branch, an inverted ratio, a shifted window frame, `sum` for `max`), runs the + model's own tests against each, and reports which ones nothing caught. + + Reach for it right after you author or scaffold tests, and before telling the + user the model is covered. It is also the honest answer when a user asks + whether their tests are any good, which is otherwise unanswerable. + + Read `data.counts` and then the survivors, which are listed first. Each carries + `defect`, a sentence saying what would now be wrong, and `suggested_test`, the + test that would catch it. Relay those two: the user's next action is to write + that test, not to read the SQL. A `score` is reported but it is a ratio of two + small integers over one model, so quote it as context and never as a grade, and + never compare it between models. + + Check `baseline.excluded` before trusting a clean-looking result. Every verdict + is relative to the tests that passed against the unmutated model, so a test + that was already failing is excluded and named there. And read `cap.elided`: + the run is capped at 20 mutants, so a model with more sites than that was + measured on a sample, spread across defect classes. + + It writes nothing. Mutants build as ephemeral models in a throwaway copy, so + the project is untouched and no relation is created or replaced. On a billed + connector the whole batch is one estimate and one `--confirm`, and if the + budget runs out partway the rest come back `not_run`: relay that rather than + reading a short list as a clean bill. - `transform deps` installs dbt packages explicitly (also the refresh path when `dbt_packages/` exists but is stale). No confirmation needed: deps writes only inside the project and never touches the warehouse. diff --git a/skills/transform/evals/evals.json b/skills/transform/evals/evals.json index 0c30ba0e..0821a9b8 100644 --- a/skills/transform/evals/evals.json +++ b/skills/transform/evals/evals.json @@ -74,6 +74,19 @@ "PII flags propagate into emitted dbt (model and column meta); no example values", "Nothing is applied silently; the change is a reviewable diff" ] + }, + { + "id": 4, + "prompt": "I added tests to my orders mart last week. Are they actually any good?", + "expected_output": "Mutation coverage run on that model, reporting which planted defects the tests failed to catch, each with the test that would catch it. Survivors are relayed as the finding; the score is context, not a grade.", + "files": [], + "assertions": [ + "Reaches for `transform test --mutate ` rather than reading the tests and judging them by eye", + "Relays the surviving defects and their suggested tests, not just a count or a score", + "Reports the cap and any excluded baseline tests rather than presenting a partial run as complete", + "Any metered connector surfaces one batch estimate before running (cost before spend)", + "Never claims the project or the warehouse was modified: mutants are ephemeral and confined to a throwaway copy" + ] } ] } From 7f62ea13e58ec03e4ae9934968353b146a59c019 Mon Sep 17 00:00:00 2001 From: Marco Ciavarella Date: Thu, 10 Sep 2026 20:08:11 +0200 Subject: [PATCH 5/5] Update CHANGELOG.md --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4558896b..98815934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,48 @@ tag releases both in lockstep, so entries below are keyed by the engine version. ## [Unreleased] +### Added + +- **`transform test --mutate ` measures whether a model's tests would + catch anything** ([#232]). A suite that passes proves the tests ran, not that + they would object if the model were wrong, and no count distinguishes the two: + a `not_null` on a surrogate key and a unit test pinning the arithmetic both + read as "tested". This plants one standard analytics defect at a time in the + model's compiled SQL (a boundary flipped, a filter dropped or negated, an + inner join swapped for a left join, a `CASE` branch removed, a ratio inverted, + a window frame shifted, `sum` reporting a `max`), runs the model's own generic, + singular and unit tests against each, and reports which defects nothing caught. + Each finding is written as the defect rather than as a diff, and carries the + test that would catch it, because the reader's next action is to write a test. + + **Nothing is written and nothing is materialized.** Every mutant is built in a + throwaway copy of the project as an ephemeral model, so dbt inlines it into + each test and creates no relation: the project is byte-identical afterwards and + the dev namespace holds exactly what it held before. The dogfood checked both + on all three warehouses. The run uses `dbt test` rather than `dbt build`, + which is load-bearing: under a build, one failing unit test marks the model + skipped and that skip cascades onto every data test attached to it, so every + mutant would read as caught and nothing would be learned about the data tests. + + **The batch is priced and confirmed once.** On a metered connector each mutant + is priced as the statement the warehouse will actually run, by splicing it into + each test's compiled SQL rather than multiplying the baseline, because a mutant + that drops a partition predicate scans more than the model it came from. One + estimate names `(baseline)` and each mutant; one `--budget` covers the run. A + budget that runs out partway stops the run and reports the remainder as + `not_run` rather than overspending. Capped at 20 mutants, ordered round robin + across the defect classes so a cap stays representative, with whatever it cut + reported per class. + + **Every verdict is relative to what already passed.** A test failing before + anything was mutated is excluded and named, so a suite measured against its own + broken tests cannot come back looking clean, and a run where nothing passes at + baseline is an error rather than a clean sweep. A mutant the warehouse refuses + outright is reported as `rejected` rather than `killed`, since a build would + have failed on it anyway and counting it would flatter the suite. + + Also available as `DexEngine.test_mutations(model)`. + ## [1.12.2] - 2026-09-09 ### Fixed