From 71df85e257027708bf425b8c3274a546c6ef866d Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 11 Jul 2023 13:18:00 -0700 Subject: [PATCH 1/4] Add support for DuckDB --- integration_tests/ci/sample.profiles.yml | 4 +++- integration_tests/requirements.txt | 3 ++- macros/percentile.sql | 11 ++++++++++- macros/timestamp_add.sql | 6 ++++++ macros/timestamp_diff.sql | 7 ++++++- macros/try_cast.sql | 6 +++++- macros/wrap_in_quotes.sql | 6 +++++- 7 files changed, 37 insertions(+), 6 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index aa45995..404523f 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -50,4 +50,6 @@ integration_tests: schema: fivetran_utils_integration_tests threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks \ No newline at end of file + type: databricks + duckdb: + type: duckdb diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..5221bcc 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 +dbt-duckdb>=1.3.0,<2.0.0 diff --git a/macros/percentile.sql b/macros/percentile.sql index 5bb29a9..a2febaa 100644 --- a/macros/percentile.sql +++ b/macros/percentile.sql @@ -44,6 +44,15 @@ {% endmacro %} +{% macro duckdb__percentile(percentile_field, partition_field, percent) %} + + percentile_cont( + {{ percent }} ) + within group ( order by {{ percentile_field }} ) + /* have to group by partition field */ + +{% endmacro %} + {% macro spark__percentile(percentile_field, partition_field, percent) %} percentile( @@ -52,4 +61,4 @@ over (partition by {{ partition_field }} ) -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/timestamp_add.sql b/macros/timestamp_add.sql index d0fdaf5..f1e28ba 100644 --- a/macros/timestamp_add.sql +++ b/macros/timestamp_add.sql @@ -43,3 +43,9 @@ {{ dbt.dateadd(datepart, interval, from_timestamp) }} {% endmacro %} + +{% macro duckdb__timestamp_add(datepart, interval, from_timestamp) %} + + {{ dbt.dateadd(datepart, interval, from_timestamp) }} + +{% endmacro % diff --git a/macros/timestamp_diff.sql b/macros/timestamp_diff.sql index 824aab1..25f4f73 100644 --- a/macros/timestamp_diff.sql +++ b/macros/timestamp_diff.sql @@ -65,4 +65,9 @@ {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} + + +{% macro duckdb__timestamp_diff(first_date, second_date, datepart) %} + return(postgres__timestamp_diff(first_date, second_date, datepart)) +{% endmacro %} diff --git a/macros/try_cast.sql b/macros/try_cast.sql index d075432..affb3fa 100644 --- a/macros/try_cast.sql +++ b/macros/try_cast.sql @@ -49,4 +49,8 @@ {% macro spark__try_cast(field, type) %} try_cast({{field}} as {{type}}) -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro duckdb__try_cast(field, type) %} + try_cast({{field}} as {{type}}) +{% endmacro %} diff --git a/macros/wrap_in_quotes.sql b/macros/wrap_in_quotes.sql index acb0a4f..74dc911 100644 --- a/macros/wrap_in_quotes.sql +++ b/macros/wrap_in_quotes.sql @@ -19,4 +19,8 @@ {%- macro postgres__wrap_in_quotes(object_to_quote) -%} "{{ object_to_quote }}" -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} + +{%- macro duckdb__wrap_in_quotes(object_to_quote) -%} + "{{ object_to_quote }}" +{%- endmacro -%} From 83f361c14bb18e59126db8fcfb784d04b02bd34e Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 7 Aug 2023 07:20:01 -0700 Subject: [PATCH 2/4] bug fix/wire up integration tests --- .buildkite/pipeline.yml | 194 ++++++++++++++++++++++- integration_tests/ci/sample.profiles.yml | 2 + macros/timestamp_add.sql | 2 +- 3 files changed, 196 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4a32060..4850503 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -73,6 +73,18 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh databricks linkedin apple_search_ads snapchat_ads facebook_ads + - label: ":duck: Run Tests - Ads Packages :bookmark: :one:" + key: "run_dbt_duckdb_ads_1" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/ads1.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb linkedin apple_search_ads snapchat_ads facebook_ads + ## Ads Packages 2 - label: ":snowflake: Run Tests - Ads Packages :bookmark: :two:" key: "run_dbt_snowflake_ads_2" @@ -150,6 +162,18 @@ steps: - step: run_dbt_databricks_ads_1 allow_failure: true + - label: ":duck: Run Tests - Ads Packages :bookmark: :two:" + key: "run_dbt_duckdb_ads_2" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/ads2.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb tiktok_ads twitter_ads amazon_ads reddit_ads + ## Ads Packages 3 - label: ":snowflake: Run Tests - Ads Packages :bookmark: :three:" key: "run_dbt_snowflake_ads_3" @@ -227,6 +251,18 @@ steps: - step: run_dbt_databricks_ads_2 allow_failure: true + - label: ":duck: Run Tests - Ads Packages :bookmark: :three:" + key: "run_dbt_duckdb_ads_3" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/ads3.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb google_ads microsoft_ads pinterest + ## Social Packages - label: ":snowflake: Run Tests - Social Packages :iphone:" key: "run_dbt_snowflake_social" @@ -304,6 +340,18 @@ steps: - step: run_dbt_databricks_ads_3 allow_failure: true + - label: ":duck: Run Tests - Social Packages :iphone:" + key: "run_dbt_duckdb_social" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/social.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb linkedin_pages facebook_pages instagram_business twitter_organic youtube_analytics + ## App Packages - label: ":snowflake: Run Tests - App Packages :video_game:" key: "run_dbt_snowflake_app" @@ -381,6 +429,18 @@ steps: - step: run_dbt_databricks_social allow_failure: true + - label: ":duck: Run Tests - App Packages :video_game:" + key: "run_dbt_duckdb_app" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/app.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb google_play apple_store + ## CRM Packages - label: ":snowflake: Run Tests - CRM Packages :salesforce::hubspot:" key: "run_dbt_snowflake_crm" @@ -458,6 +518,18 @@ steps: - step: run_dbt_databricks_app allow_failure: true + - label: ":duck: Run Tests - CRM Packages :salesforce::hubspot:" + key: "run_dbt_duckdb_crm" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/crm.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb salesforce hubspot + ## Ecommerce and Subscription Packages - label: ":snowflake: Run Tests - Ecommerce and Subscription Packages :shopping_bags:" key: "run_dbt_snowflake_esub" @@ -535,6 +607,18 @@ steps: - step: run_dbt_databricks_crm allow_failure: true + - label: ":duck: Run Tests - Ecommerce and Subscription Packages :shopping_bags:" + key: "run_dbt_duckdb_esub" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/esub.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb shopify recurly stripe zuora + ## Email Marketing Packages - label: ":snowflake: Run Tests - Email Marketing Packages :email:" key: "run_dbt_snowflake_email" @@ -613,6 +697,18 @@ steps: - step: run_dbt_databricks_esub allow_failure: true + - label: ":duck: Run Tests - Email Marketing Packages :email:" + key: "run_dbt_duckdb_email" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/email.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb marketo iterable klaviyo mailchimp pardot + ## Finance Packages - label: ":snowflake: Run Tests - Finance Packages :money_with_wings:" key: "run_dbt_snowflake_finance" @@ -691,6 +787,18 @@ steps: - step: run_dbt_databricks_email allow_failure: true + - label: ":gcloud: Run Tests - Finance Packages :money_with_wings:" + key: "run_dbt_duckdb_finance" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/finance.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb quickbooks xero sage_intacct netsuite + ## Log and People Packages - label: ":snowflake: Run Tests - Log and People Packages :people_hugging:" key: "run_dbt_snowflake_log_people" @@ -769,6 +877,18 @@ steps: - step: run_dbt_databricks_finance allow_failure: true + - label: ":duck: Run Tests - Log and People Packages :people_hugging:" + key: "run_dbt_duckdb_log_people" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/log_people.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb fivetran_log greenhouse lever + ## Product Packages - label: ":snowflake: Run Tests - Product Packages :unicorn_face:" key: "run_dbt_snowflake_product" @@ -846,6 +966,18 @@ steps: - step: run_dbt_databricks_log_people allow_failure: true + - label: ":duck: Run Tests - Product Packages :unicorn_face:" + key: "run_dbt_duckdb_product" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/product.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb amplitude mixpanel pendo + ## Velocity Packages - label: ":snowflake: Run Tests - Velocity Packages :roller_coaster:" key: "run_dbt_snowflake_velocity" @@ -924,6 +1056,18 @@ steps: - step: run_dbt_databricks_product allow_failure: true + - label: ":duck: Run Tests - Velocity Packages :roller_coaster:" + key: "run_dbt_duckdb_velocity" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/velocity.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb intercom github zendesk asana jira + ## Ad Reporting Roll Up Packages - label: ":snowflake: Run Tests - Ad Reporting Roll Up Package :roller_skate:" key: "run_dbt_snowflake_ad_roll" @@ -1001,6 +1145,18 @@ steps: - step: run_dbt_databricks_velocity allow_failure: true + - label: ":duck: Run Tests - Ad Reporting Roll Up Package :roller_skate:" + key: "run_dbt_duckdb_ad_roll" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/ad_reporting.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb ad_reporting + ## App Reporting Roll Up Packages - label: ":snowflake: Run Tests - App Reporting Roll Up Package :roller_skate:" key: "run_dbt_snowflake_app_roll" @@ -1078,6 +1234,18 @@ steps: - step: run_dbt_databricks_ad_roll allow_failure: true + - label: ":duck: Run Tests - App Reporting Roll Up Package :roller_skate:" + key: "run_dbt_duckdb_app_roll" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/app_reporting.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb app_reporting + ## Social Media Reporting Roll Up Packages - label: ":snowflake: Run Tests - Social Reporting Roll Up Package :roller_skate:" key: "run_dbt_snowflake_social_roll" @@ -1155,6 +1323,18 @@ steps: - step: run_dbt_databricks_app_roll allow_failure: true + - label: ":duck: Run Tests - Social Reporting Roll Up Package :roller_skate:" + key: "run_dbt_duckdb_social_roll" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/social_media_reporting.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb social_media_reporting + ## Shopify Holistic Reporting Roll Up Packages - label: ":snowflake: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:" key: "run_dbt_snowflake_shopify_holistic_roll" @@ -1230,4 +1410,16 @@ steps: bash .buildkite/scripts/run_standard_models.sh databricks shopify_holistic_reporting depends_on: - step: run_dbt_databricks_social_roll - allow_failure: true \ No newline at end of file + allow_failure: true + + - label: ":duck: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:" + key: "run_dbt_duckdb_shopify_holistic_roll" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DUCKDB_DBT_PATH=/tmp/shopify_holistic_reporting.duckdb" + commands: | + bash .buildkite/scripts/run_standard_models.sh duckdb shopify_holistic_reporting \ No newline at end of file diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 404523f..a9c48dd 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -53,3 +53,5 @@ integration_tests: type: databricks duckdb: type: duckdb + path: "{{ env_var('CI_DUCKDB_DBT_PATH') }}" + schema: fivetran_utils_integration_tests diff --git a/macros/timestamp_add.sql b/macros/timestamp_add.sql index f1e28ba..3a2d4bf 100644 --- a/macros/timestamp_add.sql +++ b/macros/timestamp_add.sql @@ -48,4 +48,4 @@ {{ dbt.dateadd(datepart, interval, from_timestamp) }} -{% endmacro % +{% endmacro %} From 37695ba913d3eff200e8b646f421400fad29c869 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 7 Aug 2023 09:44:22 -0700 Subject: [PATCH 3/4] fix emoji --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4850503..1d777b3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -787,7 +787,7 @@ steps: - step: run_dbt_databricks_email allow_failure: true - - label: ":gcloud: Run Tests - Finance Packages :money_with_wings:" + - label: ":duck: Run Tests - Finance Packages :money_with_wings:" key: "run_dbt_duckdb_finance" plugins: - docker#v3.13.0: @@ -1422,4 +1422,4 @@ steps: - "BASH_ENV=/tmp/.bashrc" - "CI_DUCKDB_DBT_PATH=/tmp/shopify_holistic_reporting.duckdb" commands: | - bash .buildkite/scripts/run_standard_models.sh duckdb shopify_holistic_reporting \ No newline at end of file + bash .buildkite/scripts/run_standard_models.sh duckdb shopify_holistic_reporting From b875d5dcf6dcc135f810501b6e1a7eff0b8ccc3c Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 7 Aug 2023 09:48:52 -0700 Subject: [PATCH 4/4] README updates for macros that have DuckDB support --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 923ce1d..dbfd9b8 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ BigQuery, Snowflake, Redshift, and Postgres. By default a comma `,` is used as a ---- ### ceiling ([source](macros/ceiling.sql)) This macro allows for cross database use of the ceiling function. The ceiling function returns the smallest integer greater -than, or equal to, the specified numeric expression. The ceiling macro is compatible with BigQuery, Redshift, Postgres, and Snowflake. +than, or equal to, the specified numeric expression. The ceiling macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake. **Usage:** ```sql @@ -189,7 +189,7 @@ This macro returns the value_expression for the first row in the current window ### json_extract ([source](macros/json_extract.sql)) This macro allows for cross database use of the json extract function. The json extract allows the return of data from a json object. -The data is returned by the path you provide as the argument. The json_extract macro is compatible with BigQuery, Redshift, Postgres, and Snowflake. +The data is returned by the path you provide as the argument. The json_extract macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake. **Usage:** ```sql @@ -214,7 +214,7 @@ The data is returned by the path you provide as the list within the `string_path ---- ### max_bool ([source](macros/max_bool.sql)) -This macro allows for cross database use of obtaining the max boolean value of a field. This macro recognizes true = 1 and false = 0. The macro will aggregate the boolean_field and return the max boolean value. The max_bool macro is compatible with BigQuery, Redshift, Postgres, and Snowflake. +This macro allows for cross database use of obtaining the max boolean value of a field. This macro recognizes true = 1 and false = 0. The macro will aggregate the boolean_field and return the max boolean value. The max_bool macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake. **Usage:** ```sql @@ -241,7 +241,7 @@ from your_cte ---- ### pivot_json_extract ([source](macros/pivot_json_extract.sql)) -This macro builds off of the `json_extract` macro in order to extract a list of fields from a json object and pivot the fields out into columns. The `pivot_json_extract` macro is compatible with BigQuery, Redshift, Postgres, and Snowflake. +This macro builds off of the `json_extract` macro in order to extract a list of fields from a json object and pivot the fields out into columns. The `pivot_json_extract` macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake. **Usage:** ```sql @@ -254,7 +254,7 @@ This macro builds off of the `json_extract` macro in order to extract a list of ---- ### string_agg ([source](macros/string_agg.sql)) This macro allows for cross database field aggregation and delimiter customization. Supported database specific field aggregation functions include -BigQuery, Snowflake, Redshift, Postgres, and Spark. +BigQuery, Snowflake, Redshift, Postgres, DuckDB, and Spark. **Usage:** ```sql @@ -265,7 +265,7 @@ BigQuery, Snowflake, Redshift, Postgres, and Spark. * `delimiter` (required): Character you want to be used as the delimiter between aggregates. ---- ### timestamp_add ([source](macros/timestamp_add.sql)) -This macro allows for cross database addition of a timestamp field and a specified datepart and interval for BigQuery, Redshift, Postgres, and Snowflake. +This macro allows for cross database addition of a timestamp field and a specified datepart and interval for BigQuery, Redshift, Postgres, DuckDB, and Snowflake. **Usage:** ```sql @@ -278,7 +278,7 @@ This macro allows for cross database addition of a timestamp field and a specifi ---- ### timestamp_diff ([source](macros/timestamp_diff.sql)) -This macro allows for cross database timestamp difference calculation for BigQuery, Redshift, Postgres, and Snowflake. +This macro allows for cross database timestamp difference calculation for BigQuery, Redshift, Postgres, DuckDB, and Snowflake. **Usage:** ```sql @@ -291,7 +291,7 @@ This macro allows for cross database timestamp difference calculation for BigQue ---- ### try_cast ([source](macros/try_cast.sql)) -This macro allows a field to be cast to a specified datatype. If the datatype is incompatible then a `null` value is provided. This macro is compatible with BigQuery, Redshift, Postgres, Snowflake, and Databricks. +This macro allows a field to be cast to a specified datatype. If the datatype is incompatible then a `null` value is provided. This macro is compatible with BigQuery, Redshift, Postgres, DuckDB, Snowflake, and Databricks. > Please note: For Postgres and Redshift destinations the `numeric` datatype is only supported to try_cast. **Usage:** ```sql