Add DuckDB support

.buildkite/pipeline.yml

@@ -73,6 +73,18 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh databricks linkedin apple_search_ads snapchat_ads facebook_ads 
 
+  - label: ":duck: Run Tests - Ads Packages :bookmark: :one:"
+    key: "run_dbt_duckdb_ads_1"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/ads1.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb linkedin apple_search_ads snapchat_ads facebook_ads 
+
 ## Ads Packages 2
   - label: ":snowflake: Run Tests - Ads Packages :bookmark: :two:"
     key: "run_dbt_snowflake_ads_2"
@@ -150,6 +162,18 @@ steps:
       - step: run_dbt_databricks_ads_1
         allow_failure: true
 
+  - label: ":duck: Run Tests - Ads Packages :bookmark: :two:"
+    key: "run_dbt_duckdb_ads_2"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/ads2.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb tiktok_ads twitter_ads amazon_ads reddit_ads
+
 ## Ads Packages 3
   - label: ":snowflake: Run Tests - Ads Packages :bookmark: :three:"
     key: "run_dbt_snowflake_ads_3"
@@ -227,6 +251,18 @@ steps:
       - step: run_dbt_databricks_ads_2
         allow_failure: true
 
+  - label: ":duck: Run Tests - Ads Packages :bookmark: :three:"
+    key: "run_dbt_duckdb_ads_3"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/ads3.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb google_ads microsoft_ads pinterest
+
 ## Social Packages
   - label: ":snowflake: Run Tests - Social Packages :iphone:"
     key: "run_dbt_snowflake_social"
@@ -304,6 +340,18 @@ steps:
       - step: run_dbt_databricks_ads_3
         allow_failure: true
 
+  - label: ":duck: Run Tests - Social Packages :iphone:"
+    key: "run_dbt_duckdb_social"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/social.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb linkedin_pages facebook_pages instagram_business twitter_organic youtube_analytics
+
 ## App Packages
   - label: ":snowflake: Run Tests - App Packages :video_game:"
     key: "run_dbt_snowflake_app"
@@ -381,6 +429,18 @@ steps:
       - step: run_dbt_databricks_social
         allow_failure: true
 
+  - label: ":duck: Run Tests - App Packages :video_game:"
+    key: "run_dbt_duckdb_app"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/app.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb google_play apple_store
+
 ## CRM Packages
   - label: ":snowflake: Run Tests - CRM Packages :salesforce::hubspot:"
     key: "run_dbt_snowflake_crm"
@@ -458,6 +518,18 @@ steps:
       - step: run_dbt_databricks_app
         allow_failure: true
 
+  - label: ":duck: Run Tests - CRM Packages :salesforce::hubspot:"
+    key: "run_dbt_duckdb_crm"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/crm.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb salesforce hubspot
+
 ## Ecommerce and Subscription Packages
   - label: ":snowflake: Run Tests - Ecommerce and Subscription Packages :shopping_bags:"
     key: "run_dbt_snowflake_esub"
@@ -535,6 +607,18 @@ steps:
       - step: run_dbt_databricks_crm
         allow_failure: true
 
+  - label: ":duck: Run Tests - Ecommerce and Subscription Packages :shopping_bags:"
+    key: "run_dbt_duckdb_esub"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/esub.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb shopify recurly stripe zuora
+
 ## Email Marketing Packages
   - label: ":snowflake: Run Tests - Email Marketing Packages :email:"
     key: "run_dbt_snowflake_email"
@@ -613,6 +697,18 @@ steps:
       - step: run_dbt_databricks_esub
         allow_failure: true
 
+  - label: ":duck: Run Tests - Email Marketing Packages :email:"
+    key: "run_dbt_duckdb_email"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/email.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb marketo iterable klaviyo mailchimp pardot
+
 ## Finance Packages
   - label: ":snowflake: Run Tests - Finance Packages :money_with_wings:"
     key: "run_dbt_snowflake_finance"
@@ -691,6 +787,18 @@ steps:
       - step: run_dbt_databricks_email
         allow_failure: true
 
+  - label: ":duck: Run Tests - Finance Packages :money_with_wings:"
+    key: "run_dbt_duckdb_finance"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/finance.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb quickbooks xero sage_intacct netsuite
+
 ## Log and People Packages
   - label: ":snowflake: Run Tests - Log and People Packages :people_hugging:"
     key: "run_dbt_snowflake_log_people"
@@ -769,6 +877,18 @@ steps:
       - step: run_dbt_databricks_finance
         allow_failure: true
 
+  - label: ":duck: Run Tests - Log and People Packages :people_hugging:"
+    key: "run_dbt_duckdb_log_people"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/log_people.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb fivetran_log greenhouse lever
+
 ## Product Packages
   - label: ":snowflake: Run Tests - Product Packages :unicorn_face:"
     key: "run_dbt_snowflake_product"
@@ -846,6 +966,18 @@ steps:
       - step: run_dbt_databricks_log_people
         allow_failure: true
 
+  - label: ":duck: Run Tests - Product Packages :unicorn_face:"
+    key: "run_dbt_duckdb_product"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/product.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb amplitude mixpanel pendo
+
 ## Velocity Packages
   - label: ":snowflake: Run Tests - Velocity Packages :roller_coaster:"
     key: "run_dbt_snowflake_velocity"
@@ -924,6 +1056,18 @@ steps:
       - step: run_dbt_databricks_product
         allow_failure: true
 
+  - label: ":duck: Run Tests - Velocity Packages :roller_coaster:"
+    key: "run_dbt_duckdb_velocity"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/velocity.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb intercom github zendesk asana jira
+
 ## Ad Reporting Roll Up Packages
   - label: ":snowflake: Run Tests - Ad Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_snowflake_ad_roll"
@@ -1001,6 +1145,18 @@ steps:
       - step: run_dbt_databricks_velocity
         allow_failure: true
 
+  - label: ":duck: Run Tests - Ad Reporting Roll Up Package :roller_skate:"
+    key: "run_dbt_duckdb_ad_roll"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/ad_reporting.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb ad_reporting
+
 ## App Reporting Roll Up Packages
   - label: ":snowflake: Run Tests - App Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_snowflake_app_roll"
@@ -1078,6 +1234,18 @@ steps:
       - step: run_dbt_databricks_ad_roll
         allow_failure: true
 
+  - label: ":duck: Run Tests - App Reporting Roll Up Package :roller_skate:"
+    key: "run_dbt_duckdb_app_roll"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/app_reporting.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb app_reporting
+
 ## Social Media Reporting Roll Up Packages
   - label: ":snowflake: Run Tests - Social Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_snowflake_social_roll"
@@ -1155,6 +1323,18 @@ steps:
       - step: run_dbt_databricks_app_roll
         allow_failure: true
 
+  - label: ":duck: Run Tests - Social Reporting Roll Up Package :roller_skate:"
+    key: "run_dbt_duckdb_social_roll"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/social_media_reporting.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb social_media_reporting
+
 ## Shopify Holistic Reporting Roll Up Packages
   - label: ":snowflake: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_snowflake_shopify_holistic_roll"
@@ -1230,4 +1410,16 @@ steps:
       bash .buildkite/scripts/run_standard_models.sh databricks shopify_holistic_reporting
     depends_on:
       - step: run_dbt_databricks_social_roll
-        allow_failure: true
+        allow_failure: true
+
+  - label: ":duck: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:"
+    key: "run_dbt_duckdb_shopify_holistic_roll"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DUCKDB_DBT_PATH=/tmp/shopify_holistic_reporting.duckdb"
+    commands: |
+      bash .buildkite/scripts/run_standard_models.sh duckdb shopify_holistic_reporting

README.md

@@ -162,7 +162,7 @@ BigQuery, Snowflake, Redshift, and Postgres. By default a comma `,` is used as a
 ----
 ### ceiling ([source](macros/ceiling.sql))
 This macro allows for cross database use of the ceiling function. The ceiling function returns the smallest integer greater 
-than, or equal to, the specified numeric expression. The ceiling macro is compatible with BigQuery, Redshift, Postgres, and Snowflake.
+than, or equal to, the specified numeric expression. The ceiling macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake.
 
 **Usage:**
 ```sql
@@ -189,7 +189,7 @@ This macro returns the value_expression for the first row in the current window
 
 ### json_extract ([source](macros/json_extract.sql))
 This macro allows for cross database use of the json extract function. The json extract allows the return of data from a json object.
-The data is returned by the path you provide as the argument. The json_extract macro is compatible with BigQuery, Redshift, Postgres, and Snowflake.
+The data is returned by the path you provide as the argument. The json_extract macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake.
 
 **Usage:**
 ```sql
@@ -214,7 +214,7 @@ The data is returned by the path you provide as the list within the `string_path
 
 ----
 ### max_bool ([source](macros/max_bool.sql))
-This macro allows for cross database use of obtaining the max boolean value of a field. This macro recognizes true = 1 and false = 0. The macro will aggregate the boolean_field and return the max boolean value. The max_bool macro is compatible with BigQuery, Redshift, Postgres, and Snowflake.
+This macro allows for cross database use of obtaining the max boolean value of a field. This macro recognizes true = 1 and false = 0. The macro will aggregate the boolean_field and return the max boolean value. The max_bool macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake.
 
 **Usage:**
 ```sql
@@ -241,7 +241,7 @@ from your_cte
 
 ----
 ### pivot_json_extract ([source](macros/pivot_json_extract.sql))
-This macro builds off of the `json_extract` macro in order to extract a list of fields from a json object and pivot the fields out into columns. The `pivot_json_extract` macro is compatible with BigQuery, Redshift, Postgres, and Snowflake.
+This macro builds off of the `json_extract` macro in order to extract a list of fields from a json object and pivot the fields out into columns. The `pivot_json_extract` macro is compatible with BigQuery, Redshift, Postgres, DuckDB, and Snowflake.
 
 **Usage:**
 ```sql
@@ -254,7 +254,7 @@ This macro builds off of the `json_extract` macro in order to extract a list of
 ----
 ### string_agg ([source](macros/string_agg.sql))
 This macro allows for cross database field aggregation and delimiter customization. Supported database specific field aggregation functions include 
-BigQuery, Snowflake, Redshift, Postgres, and Spark.
+BigQuery, Snowflake, Redshift, Postgres, DuckDB, and Spark.
 
 **Usage:**
 ```sql
@@ -265,7 +265,7 @@ BigQuery, Snowflake, Redshift, Postgres, and Spark.
 * `delimiter`    (required): Character you want to be used as the delimiter between aggregates.
 ----
 ### timestamp_add ([source](macros/timestamp_add.sql))
-This macro allows for cross database addition of a timestamp field and a specified datepart and interval for BigQuery, Redshift, Postgres, and Snowflake.
+This macro allows for cross database addition of a timestamp field and a specified datepart and interval for BigQuery, Redshift, Postgres, DuckDB, and Snowflake.
 
 **Usage:**
 ```sql
@@ -278,7 +278,7 @@ This macro allows for cross database addition of a timestamp field and a specifi
 
 ----
 ### timestamp_diff ([source](macros/timestamp_diff.sql))
-This macro allows for cross database timestamp difference calculation for BigQuery, Redshift, Postgres, and Snowflake.
+This macro allows for cross database timestamp difference calculation for BigQuery, Redshift, Postgres, DuckDB, and Snowflake.
 
 **Usage:**
 ```sql
@@ -291,7 +291,7 @@ This macro allows for cross database timestamp difference calculation for BigQue
 
 ----
 ### try_cast ([source](macros/try_cast.sql))
-This macro allows a field to be cast to a specified datatype. If the datatype is incompatible then a `null` value is provided. This macro is compatible with BigQuery, Redshift, Postgres, Snowflake, and Databricks.
+This macro allows a field to be cast to a specified datatype. If the datatype is incompatible then a `null` value is provided. This macro is compatible with BigQuery, Redshift, Postgres, DuckDB, Snowflake, and Databricks.
 > Please note: For Postgres and Redshift destinations the `numeric` datatype is only supported to try_cast.
 **Usage:**
 ```sql

integration_tests/ci/sample.profiles.yml

@@ -50,4 +50,8 @@ integration_tests:
       schema: fivetran_utils_integration_tests
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
-      type: databricks
+      type: databricks
+    duckdb:
+      type: duckdb
+      path: "{{ env_var('CI_DUCKDB_DBT_PATH') }}"
+      schema: fivetran_utils_integration_tests

integration_tests/requirements.txt

@@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+dbt-duckdb>=1.3.0,<2.0.0