From 04e43148b26b24ecb395115efab3d292a8bcbb8e Mon Sep 17 00:00:00 2001 From: itismichael <56938495+itismichael@users.noreply.github.com> Date: Mon, 27 May 2024 14:09:18 -0500 Subject: [PATCH 1/5] add module for checking dbt version --- droughty/droughty/droughty_dbt/dbt_version_util.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 droughty/droughty/droughty_dbt/dbt_version_util.py diff --git a/droughty/droughty/droughty_dbt/dbt_version_util.py b/droughty/droughty/droughty_dbt/dbt_version_util.py new file mode 100644 index 0000000..5c9bdc4 --- /dev/null +++ b/droughty/droughty/droughty_dbt/dbt_version_util.py @@ -0,0 +1,9 @@ +import dbt.version + +def is_version_gte_1_8(): + """ + Checks if the installed dbt version is greater than or equal to 1.8.0. + """ + installed_version = dbt.version.get_installed_version() + target_version = dbt.semver.VersionSpecifier.from_version_string("1.8.0") + return installed_version.compare(target_version) >= 0 \ No newline at end of file From 17d740f41b67249d1730482f724c82929a6b9279 Mon Sep 17 00:00:00 2001 From: itismichael <56938495+itismichael@users.noreply.github.com> Date: Mon, 27 May 2024 14:12:21 -0500 Subject: [PATCH 2/5] import version check util --- droughty/droughty/droughty_dbt/dbt_test_module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/droughty/droughty/droughty_dbt/dbt_test_module.py b/droughty/droughty/droughty_dbt/dbt_test_module.py index 3f6c477..49fe592 100644 --- a/droughty/droughty/droughty_dbt/dbt_test_module.py +++ b/droughty/droughty/droughty_dbt/dbt_test_module.py @@ -18,6 +18,7 @@ IdentifyConfigVariables, ExploresVariables ) +from droughty.droughty_dbt.dbt_version_util import is_version_gte_1_8 import sys import ruamel.yaml From af93adf555cdcf9e97d5c4b6be3e75f1e54d9042 Mon Sep 17 00:00:00 2001 From: itismichael <56938495+itismichael@users.noreply.github.com> Date: Mon, 27 May 2024 14:19:49 -0500 Subject: [PATCH 3/5] add function to get key to use in config --- droughty/droughty/droughty_dbt/dbt_test_module.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/droughty/droughty/droughty_dbt/dbt_test_module.py b/droughty/droughty/droughty_dbt/dbt_test_module.py index 49fe592..abdccb6 100644 --- a/droughty/droughty/droughty_dbt/dbt_test_module.py +++ b/droughty/droughty/droughty_dbt/dbt_test_module.py @@ -25,6 +25,15 @@ import git +def get_test_definition_key(): + """ + Returns the appropriate test definition key based on the dbt version. + """ + if is_version_gte_1_8(): + return "data_tests" + else: + return "tests" + def get_all_values(nested_dictionary): test_overwrite = ExploresVariables.test_overwrite From fc704d2836f8c80b828043b59cc56352cdde9ce1 Mon Sep 17 00:00:00 2001 From: itismichael <56938495+itismichael@users.noreply.github.com> Date: Mon, 27 May 2024 14:25:31 -0500 Subject: [PATCH 4/5] use variable instead of "tests" string --- .../droughty/droughty_dbt/dbt_test_module.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/droughty/droughty/droughty_dbt/dbt_test_module.py b/droughty/droughty/droughty_dbt/dbt_test_module.py index abdccb6..27a29af 100644 --- a/droughty/droughty/droughty_dbt/dbt_test_module.py +++ b/droughty/droughty/droughty_dbt/dbt_test_module.py @@ -35,6 +35,8 @@ def get_test_definition_key(): return "tests" def get_all_values(nested_dictionary): + + tests_config_key = get_test_definition_key() test_overwrite = ExploresVariables.test_overwrite @@ -73,22 +75,22 @@ def get_all_values(nested_dictionary): if "pk" in key1 and "not_null" not in value1 and "unique" not in value1: - elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", "tests": ["not_null","unique"]} + elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", tests_config_key: ["not_null","unique"]} seq.append(elem) elif "fk" in key1: - elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", "tests": ["dbt_utils.at_least_one"]} + elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", tests_config_key: ["dbt_utils.at_least_one"]} seq.append(elem) elif "valid_to" in key1 or "valid_from" in key1: - elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", "tests": ["dbt_utils.expression_is_true"":""expression"":"" valid_from < valid_to","not_null","unique"]} + elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", tests_config_key: ["dbt_utils.expression_is_true"":""expression"":"" valid_from < valid_to","not_null","unique"]} seq.append(elem) elif "pk" not in key1 or "fk" not in key1: - elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", "tests": [""+"dbt_utils.at_least_one"]} + elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", tests_config_key: [""+"dbt_utils.at_least_one"]} seq.append(elem) elif "pk" not in key1 or "fk" not in key1: @@ -100,22 +102,22 @@ def get_all_values(nested_dictionary): if "pk" in key1: - elem = {"name": key1, "tests": ["not_null","unique"]} + elem = {"name": key1, tests_config_key: ["not_null","unique"]} seq.append(elem) elif "fk" in key1: - elem = {"name": key1, "tests": ["dbt_utils.at_least_one"]} + elem = {"name": key1, tests_config_key: ["dbt_utils.at_least_one"]} seq.append(elem) elif "valid_to" in key1 or "valid_from" in key1: - elem = {"name": key1, "tests": ["dbt_utils.expression_is_true"":""expression"":"" valid_from < valid_to","not_null","unique"]} + elem = {"name": key1, tests_config_key: ["dbt_utils.expression_is_true"":""expression"":"" valid_from < valid_to","not_null","unique"]} seq.append(elem) elif "pk" not in key1 or "fk" not in key1: - elem = {"name": key1, "tests": [""+"dbt_utils.at_least_one"]} + elem = {"name": key1, tests_config_key: [""+"dbt_utils.at_least_one"]} seq.append(elem) elif key + "-" + key1 in ignore_test_keys_and_values and key not in ExploresVariables.test_ignore: @@ -123,12 +125,12 @@ def get_all_values(nested_dictionary): if key1 in described_columns_list: - elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", "tests": value1} + elem = {"name": key1, "description": "{{doc("+'"'+key1+'"'+")}}", tests_config_key: value1} seq.append(elem) elif key1 not in described_columns_list: - elem = {"name": key1, "tests": value1} + elem = {"name": key1, tests_config_key: value1} seq.append(elem) res.append([{"name": key, "columns": seq}]) From d7eb73b709e642652b4f4421ebd3f07af24acec9 Mon Sep 17 00:00:00 2001 From: itismichael <56938495+itismichael@users.noreply.github.com> Date: Mon, 27 May 2024 14:51:09 -0500 Subject: [PATCH 5/5] update to correctly get version from string --- droughty/droughty/droughty_dbt/dbt_version_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/droughty/droughty/droughty_dbt/dbt_version_util.py b/droughty/droughty/droughty_dbt/dbt_version_util.py index 5c9bdc4..db27d66 100644 --- a/droughty/droughty/droughty_dbt/dbt_version_util.py +++ b/droughty/droughty/droughty_dbt/dbt_version_util.py @@ -1,9 +1,10 @@ import dbt.version +import dbt_common.semver as semver def is_version_gte_1_8(): """ Checks if the installed dbt version is greater than or equal to 1.8.0. """ installed_version = dbt.version.get_installed_version() - target_version = dbt.semver.VersionSpecifier.from_version_string("1.8.0") + target_version = semver.VersionSpecifier.from_version_string("1.8.0") return installed_version.compare(target_version) >= 0 \ No newline at end of file