From b47b61f7610eca8c714c96d746df21ac0fe759dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 20:23:25 +0000 Subject: [PATCH 01/17] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.7 → v0.14.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.7...v0.14.10) - [github.com/pre-commit/mirrors-mypy: v1.19.0 → v1.19.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.19.0...v1.19.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c8e7dae0..5c1bbaa05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.7 + rev: v0.14.10 hooks: - id: ruff language_version: python3.12 @@ -11,7 +11,7 @@ repos: - id: ruff-format language_version: python3.12 - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.19.0' + rev: 'v1.19.1' hooks: - id: mypy args: [--check-untyped-defs, --ignore-missing-imports] From 78f74b0c23917a151d1cbd4fc57efc553b8ad1a1 Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Mon, 2 Jun 2025 14:52:19 -0700 Subject: [PATCH 02/17] [run CI] From f5369f3e7ba58225e4c3a04986dfc16bef8fb023 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 20:42:28 +0000 Subject: [PATCH 03/17] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.10 → v0.15.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.10...v0.15.9) - [github.com/pre-commit/mirrors-mypy: v1.19.1 → v1.20.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.19.1...v1.20.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c1bbaa05..b2b78e893 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.10 + rev: v0.15.9 hooks: - id: ruff language_version: python3.12 @@ -11,7 +11,7 @@ repos: - id: ruff-format language_version: python3.12 - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.19.1' + rev: 'v1.20.0' hooks: - id: mypy args: [--check-untyped-defs, --ignore-missing-imports] From 8e731e7ca6e5f4fcd36a1eef3bcd2c15716218d7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 20:44:14 +0000 Subject: [PATCH 04/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydough/conversion/filter_pushdown.py | 6 ++++-- pydough/sqlglot/override_qualify.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pydough/conversion/filter_pushdown.py b/pydough/conversion/filter_pushdown.py index 27dd67ad8..273782dfc 100644 --- a/pydough/conversion/filter_pushdown.py +++ b/pydough/conversion/filter_pushdown.py @@ -239,8 +239,10 @@ def visit_join(self, join: Join) -> RelationalNode: # they are false if the input is null. pushable_filters, remaining_filters = partition_expressions( remaining_filters, - lambda expr: only_references_columns(expr, input_cols[idx]) - and false_when_null_columns(expr, input_cols[idx]), + lambda expr: ( + only_references_columns(expr, input_cols[idx]) + and false_when_null_columns(expr, input_cols[idx]) + ), ) else: pushable_filters, remaining_filters = partition_expressions( diff --git a/pydough/sqlglot/override_qualify.py b/pydough/sqlglot/override_qualify.py index 606ad6aca..1dd472034 100644 --- a/pydough/sqlglot/override_qualify.py +++ b/pydough/sqlglot/override_qualify.py @@ -364,9 +364,9 @@ def qualify_columns_func( # In Snowflake / Oracle queries that have a CONNECT BY clause, one can use the LEVEL # pseudocolumn, which doesn't belong to a table, so we change it into an identifier scope_expression.transform( - lambda n: n.this - if isinstance(n, exp.Column) and n.name == "LEVEL" - else n, + lambda n: ( + n.this if isinstance(n, exp.Column) and n.name == "LEVEL" else n + ), copy=False, ) scope.clear_cache() From e4c069c2869807fd160a2291f020b6ea05d46be7 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 8 Apr 2026 11:05:37 -0700 Subject: [PATCH 05/17] [RUN CI] From f08df233d8e690d8abc7fd02c43442d647116dc6 Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Tue, 14 Apr 2026 10:24:12 -0700 Subject: [PATCH 06/17] [run CI] --- pydough/user_collections/view_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydough/user_collections/view_collection.py b/pydough/user_collections/view_collection.py index 0d3e47c51..6bd5b03ee 100644 --- a/pydough/user_collections/view_collection.py +++ b/pydough/user_collections/view_collection.py @@ -62,7 +62,7 @@ def __init__( self._is_replace = is_replace self._is_temp = is_temp # Default to all columns as the unique key if not specified - self._unique_columns = ( + self._unique_columns: list[str | list[str]] = ( unique_columns if unique_columns is not None else [columns] ) self._write_path = write_path From a339fdfa4c6a250ee600db485f313cf5e3540f3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 20:33:03 +0000 Subject: [PATCH 07/17] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.15.15 → v0.15.20](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.15...v0.15.20) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85c24d7f8..93315c85b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.15 + rev: v0.15.20 hooks: - id: ruff language_version: python3.12 From 9425ddcee052ed5aea63c8cb6234a80cf33a7d93 Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 24 Jul 2026 14:29:38 -0700 Subject: [PATCH 08/17] [run CI] From d14e996995047f2fab738adba124a7fbf0e6afea Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 24 Jul 2026 14:30:48 -0700 Subject: [PATCH 09/17] [run CI] From 635eccad4ed3a3f9d53bb18b51cb270b48503cd7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:22:55 +0000 Subject: [PATCH 10/17] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.15.20 → v0.16.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.20...v0.16.10) - [github.com/pre-commit/mirrors-mypy: v2.1.0 → v2.4.0](https://github.com/pre-commit/mirrors-mypy/compare/v2.1.0...v2.4.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93315c85b..b86b016e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.20 + rev: v0.16.10 hooks: - id: ruff language_version: python3.12 @@ -11,7 +11,7 @@ repos: - id: ruff-format language_version: python3.12 - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v2.1.0' + rev: 'v2.4.0' hooks: - id: mypy args: [--check-untyped-defs, --ignore-missing-imports] From 3dfec654c733894afbe218fae3337d62ffb13fd4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:23:45 +0000 Subject: [PATCH 11/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 3 +- benchmark/README.md | 8 +- documentation/dsl.md | 14 +- documentation/functions.md | 332 ++++++++++-------- documentation/metadata.md | 7 +- documentation/usage.md | 289 +++++++++------ pydough/conversion/column_bubbler.py | 4 +- pydough/conversion/hybrid_tree.py | 33 +- .../conversion/relational_simplification.py | 51 ++- pydough/exploration/README.md | 8 +- pydough/exploration/explain.py | 2 +- pydough/exploration/term.py | 4 +- pydough/jupyter_extensions/README.md | 3 +- .../metadata/templates/attribute_metadata.py | 6 +- .../type_inference/__init__.py | 1 - pydough/qdag/collections/README.md | 12 +- pydough/relational/README.md | 16 +- .../abstract_expression.py | 3 +- .../literal_expression.py | 4 +- pydough/relational/relational_nodes/README.md | 16 +- .../relational_nodes/abstract_node.py | 2 +- pydough/sqlglot/execute_relational.py | 13 +- pydough/sqlglot/override_simplify.py | 8 +- .../oracle_transform_bindings.py | 2 +- .../sf_transform_bindings.py | 9 +- .../sqlite_transform_bindings.py | 9 +- pydough/unqualified/README.md | 14 +- pydough/unqualified/unqualified_node.py | 2 +- pydough/unqualified/unqualified_transform.py | 6 +- pydough/user_collections/README.md | 6 +- .../user_collections/dataframe_collection.py | 4 +- .../user_collections/user_collection_apis.py | 4 +- pydough/user_collections/user_collections.py | 6 +- tests/gen_data/databricks_task.py | 22 +- tests/test_documentation.py | 2 +- tests/test_logging.py | 2 +- tests/test_masked_sf.py | 52 +-- tests/test_masked_sqlite.py | 49 ++- tests/test_pipeline_oracle.py | 4 +- tests/test_pipeline_tpch_custom.py | 4 +- tests/test_qdag_collection.py | 2 - tests/test_types.py | 2 +- tests/test_unqualified_node.py | 5 - 43 files changed, 565 insertions(+), 480 deletions(-) diff --git a/README.md b/README.md index 3522cee9c..003b5a9fb 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,7 @@ The following PyDough snippet solves this problem: ```py result = People.CALCULATE( - name, - net_income = SUM(jobs.income_earned) - SUM(schools.tuition_paid) + name, net_income=SUM(jobs.income_earned) - SUM(schools.tuition_paid) ) pydough.to_df(result) ``` diff --git a/benchmark/README.md b/benchmark/README.md index 40d7423d4..2a1b1e64d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -26,10 +26,10 @@ The generated `.csv` file has the format described in `Metrics` section. A new f from benchmarker import Benchmarker benchmarker = Benchmarker( - postgres_conn=conn, # psycopg2 (or compatible) connection - questions_path="./", # directory containing the questions file - filename="questions", # questions filename (without .csv extension) - export_metrics=False, # write metrics CSV on completion + postgres_conn=conn, # psycopg2 (or compatible) connection + questions_path="./", # directory containing the questions file + filename="questions", # questions filename (without .csv extension) + export_metrics=False, # write metrics CSV on completion ) ``` diff --git a/documentation/dsl.md b/documentation/dsl.md index 972314466..4d4b98246 100644 --- a/documentation/dsl.md +++ b/documentation/dsl.md @@ -397,10 +397,7 @@ So far, many different kinds of expressions have been noted in the examples for ```py # Referencing scalar properties of the current collection -People.CALCULATE( - first_name, - last_name -) +People.CALCULATE(first_name, last_name) # Referencing scalar properties of a singular sub-collection People.CALCULATE( @@ -411,8 +408,8 @@ People.CALCULATE( # Referencing properties from the CALCULATE an ancestor collection # (see down-streaming for more details) Addresses.CALCULATE(zip_code).current_occupants.CALCULATE(email).packages.CALCULATE( - email, # <- refers to the `email` from `current_occupants` - zip_code, # <- refers to the `zip_code` from `Addresses` + email, # <- refers to the `email` from `current_occupants` + zip_code, # <- refers to the `zip_code` from `Addresses` ) # Invoking normal functions/operations on other singular data @@ -435,6 +432,7 @@ customers.CALCULATE( from pandas import Timestamp from datetime import date from decimal import Decimal + customers.CALCULATE( a=0, b=3.14, @@ -1960,9 +1958,7 @@ Template definition: ```python customers.CALCULATE( key, - n_orders=orders_filter_count( - (total_price > 3000) & (YEAR(order_date) == 1996) - ), + n_orders=orders_filter_count((total_price > 3000) & (YEAR(order_date) == 1996)), ).TOP_K(5, by=(n_orders.DESC(), key.ASC())) ``` diff --git a/documentation/functions.md b/documentation/functions.md index c5ad51ddc..9f3cb9ff5 100644 --- a/documentation/functions.md +++ b/documentation/functions.md @@ -116,7 +116,9 @@ Below is each binary operator currently supported in PyDough. Supported mathematical operations: addition (`+`), subtraction (`-`), multiplication (`*`), division (`/`), exponentiation (`**`). ```py -Lineitems.CALCULATE(value = (extended_price * (1 - (discount ** 2)) + 1.0) / part.retail_price) +Lineitems.CALCULATE( + value=(extended_price * (1 - (discount**2)) + 1.0) / part.retail_price +) ``` > [!WARNING] @@ -130,12 +132,12 @@ Expression values can be compared using standard comparison operators: `<=`, `<` ```py customers.CALCULATE( - in_debt = acctbal < 0, - at_most_12_orders = COUNT(orders) <= 12, - is_european = nation.region.name == "EUROPE", - non_german = nation.name != "GERMANY", - non_empty_acct = acctbal > 0, - at_least_5_orders = COUNT(orders) >= 5, + in_debt=acctbal < 0, + at_most_12_orders=COUNT(orders) <= 12, + is_european=nation.region.name == "EUROPE", + non_german=nation.name != "GERMANY", + non_empty_acct=acctbal > 0, + at_least_5_orders=COUNT(orders) >= 5, ) ``` @@ -153,9 +155,9 @@ is_asian = nation.region.name == "ASIA" is_european = nation.region.name == "EUROPE" in_debt = acctbal < 0 customers.CALCULATE( - is_eurasian = is_asian | is_european, - is_not_eurasian = ~(is_asian | is_european), - is_european_in_debt = is_european & in_debt + is_eurasian=is_asian | is_european, + is_not_eurasian=~(is_asian | is_european), + is_european_in_debt=is_european & in_debt, ) ``` @@ -175,7 +177,7 @@ Below is each unary operator currently supported in PyDough. A numerical expression's sign can be flipped by prefixing it with the `-` operator: ```py -Lineitems.CALCULATE(lost_value = extended_price * (-discount)) +Lineitems.CALCULATE(lost_value=extended_price * (-discount)) ``` @@ -218,7 +220,7 @@ Below is each function currently supported in PyDough that operates on strings. Calling `LOWER` on a string converts its characters to lowercase: ```py -customers.CALCULATE(lowercase_name = LOWER(name)) +customers.CALCULATE(lowercase_name=LOWER(name)) ``` @@ -228,7 +230,7 @@ customers.CALCULATE(lowercase_name = LOWER(name)) Calling `UPPER` on a string converts its characters to uppercase: ```py -customers.CALCULATE(uppercase_name = UPPER(name)) +customers.CALCULATE(uppercase_name=UPPER(name)) ``` @@ -238,7 +240,7 @@ customers.CALCULATE(uppercase_name = UPPER(name)) Calling `length` on a string returns the number of characters it contains: ```py -Suppliers.CALCULATE(n_chars_in_comment = LENGTH(comment)) +Suppliers.CALCULATE(n_chars_in_comment=LENGTH(comment)) ``` @@ -248,7 +250,7 @@ Suppliers.CALCULATE(n_chars_in_comment = LENGTH(comment)) The `STARTSWITH` function checks if its first argument begins with its second argument as a string prefix: ```py -Parts.CALCULATE(begins_with_yellow = STARTSWITH(name, "yellow")) +Parts.CALCULATE(begins_with_yellow=STARTSWITH(name, "yellow")) ``` @@ -258,7 +260,7 @@ Parts.CALCULATE(begins_with_yellow = STARTSWITH(name, "yellow")) The `ENDSWITH` function checks if its first argument ends with its second argument as a string suffix: ```py -Parts.CALCULATE(ends_with_chocolate = ENDSWITH(name, "chocolate")) +Parts.CALCULATE(ends_with_chocolate=ENDSWITH(name, "chocolate")) ``` @@ -268,7 +270,7 @@ Parts.CALCULATE(ends_with_chocolate = ENDSWITH(name, "chocolate")) The `CONTAINS` function checks if its first argument contains its second argument as a substring: ```py -Parts.CALCULATE(is_green = CONTAINS(name, "green")) +Parts.CALCULATE(is_green=CONTAINS(name, "green")) ``` @@ -278,7 +280,7 @@ Parts.CALCULATE(is_green = CONTAINS(name, "green")) The `LIKE` function checks if the first argument matches the SQL pattern text of the second argument, where `_` is a 1 character wildcard and `%` is an 0+ character wildcard. ```py -Orders.CALCULATE(is_special_request = LIKE(comment, "%special%requests%")) +Orders.CALCULATE(is_special_request=LIKE(comment, "%special%requests%")) ``` [This link](https://www.w3schools.com/sql/sql_like.asp) explains how these SQL pattern strings work and provides some examples. @@ -290,12 +292,10 @@ Orders.CALCULATE(is_special_request = LIKE(comment, "%special%requests%")) The `JOIN_STRINGS` function concatenates all its string arguments, using the first argument as a delimiter between each of the following arguments (like the `.join` method in Python): ```py -Regions.CALCULATE( - region_name=name -).nations.CALCULATE( - nation_name=name +Regions.CALCULATE(region_name=name).nations.CALCULATE( + nation_name=name ).customers.CALCULATE( - fully_qualified_name = JOIN_STRINGS("-", region_name, nation_name, name) + fully_qualified_name=JOIN_STRINGS("-", region_name, nation_name, name) ) ``` @@ -319,7 +319,7 @@ The function behaves as follows: - If the padding argument is not a single character, it raises an error. ```py -customers.CALCULATE(left_padded_name = LPAD(name, 30, "*")) +customers.CALCULATE(left_padded_name=LPAD(name, 30, "*")) ``` Here are examples on how it pads on string literals: @@ -349,7 +349,7 @@ The function behaves as follows: - If the padding argument is not a single character, it raises an error ```py -customers.CALCULATE(right_padded_name = RPAD(name, 30, "*")) +customers.CALCULATE(right_padded_name=RPAD(name, 30, "*")) ``` Here are examples on how it pads on string literals: @@ -391,8 +391,12 @@ This function is equivalent to python's `str.strip()` method. Note: This function is case-sensitive. ```py -customers.CALCULATE(stripped_name = STRIP(name)) # removes all leading and trailing whitespace -customers.CALCULATE(stripped_name = STRIP(name, "aeiou")) # removes all leading and trailing vowels +customers.CALCULATE( + stripped_name=STRIP(name) +) # removes all leading and trailing whitespace +customers.CALCULATE( + stripped_name=STRIP(name, "aeiou") +) # removes all leading and trailing vowels ``` | **Input String (X)** | **STRIP(X, Y)** | **Result** | @@ -412,8 +416,10 @@ This function is equivalent to Python’s `str.replace()` method with the defaul Note: This function is case-sensitive. ```py -Customers.CALCULATE(updated_name= REPLACE(name, "xy", "..")) # replaces all `xy` with `..` -Customers.CALCULATE(updated_name= REPLACE(name, "xy")) # removes all `xy` appearances. +Customers.CALCULATE( + updated_name=REPLACE(name, "xy", "..") +) # replaces all `xy` with `..` +Customers.CALCULATE(updated_name=REPLACE(name, "xy")) # removes all `xy` appearances. ``` | **Input String (X)** | **REPLACE(X, Y, Z)** | **Result** | @@ -430,9 +436,13 @@ This function requires the two arguments, and is equivalent to the Python string Note: This function is case-sensitive and if one or both of the arguments are an empty string returns 0. ```py -Customers.CALCULATE(count_substring= STRCOUNT(name, "e")) # counts how many 'e's are in name -Customers.CALCULATE(count_substring= STRCOUNT(name, "Alex")) # counts how many 'Alex's are in name -Customers.CALCULATE(count_substring= STRCOUNT(name, "")) # returns 0 by default +Customers.CALCULATE( + count_substring=STRCOUNT(name, "e") +) # counts how many 'e's are in name +Customers.CALCULATE( + count_substring=STRCOUNT(name, "Alex") +) # counts how many 'Alex's are in name +Customers.CALCULATE(count_substring=STRCOUNT(name, "")) # returns 0 by default ``` | **Input String (X)** | **STRCOUNT(X, Y)** | **Result** | @@ -455,13 +465,13 @@ If the index is out of range, `GETPART` returns `""`. If the delimiter is an emp ```py # Extracts the first name from a full name -Customers.CALCULATE(first_name = GETPART(name, " ", 1)) +Customers.CALCULATE(first_name=GETPART(name, " ", 1)) # Extracts the last name from a full name -Customers.CALCULATE(last_name = GETPART(name, " ", -1)) +Customers.CALCULATE(last_name=GETPART(name, " ", -1)) # Extracts the second part from a hyphen-separated string -Parts.CALCULATE(second_code = GETPART(code, "-", 2)) +Parts.CALCULATE(second_code=GETPART(code, "-", 2)) ``` | **Input String** | **Delimiter** | **Index** | **GETPART Result** | @@ -526,16 +536,16 @@ If there are multiple modifiers, they operate left-to-right. # 5. The current day, at midnight # 6. The first day after the start of the current quarter TPCH.CALCULATE( - ts_1=DATETIME('now'), - ts_2=DATETIME('NoW', 'start of month'), - ts_3=DATETIME(' CURRENT_DATE ', '12 hours'), - ts_4=DATETIME('Current Timestamp', 'start of y', '- 1 D'), - ts_5=DATETIME('NOW', ' Start of Day '), - ts_6=DATETIME('now', 'start of quarter', '+1 d'), + ts_1=DATETIME("now"), + ts_2=DATETIME("NoW", "start of month"), + ts_3=DATETIME(" CURRENT_DATE ", "12 hours"), + ts_4=DATETIME("Current Timestamp", "start of y", "- 1 D"), + ts_5=DATETIME("NOW", " Start of Day "), + ts_6=DATETIME("now", "start of quarter", "+1 d"), ) # For each order, truncates the order date to the first day of the year -Orders.CALCULATE(order_year=DATETIME(order_year, 'START OF Y')) +Orders.CALCULATE(order_year=DATETIME(order_year, "START OF Y")) ``` @@ -565,7 +575,7 @@ Orders.WHERE(QUARTER(order_date) == 1) Calling `MONTH` on a date/timestamp extracts the month of the year it belongs to: ```py -Orders.CALCULATE(is_summer = (MONTH(order_date) >= 6) & (MONTH(order_date) <= 8)) +Orders.CALCULATE(is_summer=(MONTH(order_date) >= 6) & (MONTH(order_date) <= 8)) ``` @@ -575,7 +585,7 @@ Orders.CALCULATE(is_summer = (MONTH(order_date) >= 6) & (MONTH(order_date) <= 8) Calling `DAY` on a date/timestamp extracts the day of the month it belongs to: ```py -Orders.CALCULATE(is_first_of_month = DAY(order_date) == 1) +Orders.CALCULATE(is_first_of_month=DAY(order_date) == 1) ``` @@ -586,7 +596,7 @@ Calling `HOUR` on a date/timestamp extracts the hour it belongs to. The range of is from 0-23: ```py -Orders.CALCULATE(is_12pm = HOUR(order_date) == 12) +Orders.CALCULATE(is_12pm=HOUR(order_date) == 12) ``` @@ -597,7 +607,7 @@ Calling `MINUTE` on a date/timestamp extracts the minute. The range of output is from 0-59: ```py -Orders.CALCULATE(is_half_hour = MINUTE(order_date) == 30) +Orders.CALCULATE(is_half_hour=MINUTE(order_date) == 30) ``` @@ -608,7 +618,7 @@ Calling `SECOND` on a date/timestamp extracts the second. The range of output is from 0-59: ```py -Orders.CALCULATE(is_lt_30_seconds = SECOND(order_date) < 30) +Orders.CALCULATE(is_lt_30_seconds=SECOND(order_date) < 30) ``` @@ -634,9 +644,7 @@ Calling `DATEDIFF` between 2 timestamps returns the difference in one of `years` ```py # Calculates, for each order, the number of days since January 1st 1992 # that the order was placed: -Orders.CALCULATE( - days_since=DATEDIFF("days",datetime.date(1992, 1, 1), order_date) -) +Orders.CALCULATE(days_since=DATEDIFF("days", datetime.date(1992, 1, 1), order_date)) ``` The first argument in the `DATEDIFF` function supports the following aliases for each unit of time. The argument is **case-insensitive**, and if a unit is not one of the provided options, an error will be thrown. See [`DATETIME`](#datetime) for the supported units and their aliases. Invalid or unrecognized units will result in an error. @@ -651,7 +659,7 @@ In other words, `DAYOFWEEK` returns which day of the week is the given date/time ```py # Returns the day of the week for the order date -Orders.CALCULATE(day_of_week = DAYOFWEEK(order_date)) +Orders.CALCULATE(day_of_week=DAYOFWEEK(order_date)) ``` The following table shows the day of the week for a given date/timestamp, where the first day of the give date/timestamp is decided by the `start_of_week` config and if the week starts at 0 or 1 decided by the `start_week_as_zero` config. @@ -675,7 +683,7 @@ The `DAYNAME` function returns the name of the day of the week for a given date/ ```py # Returns the name of the day of the week for the order date -Orders.CALCULATE(day_name = DAYNAME(order_date)) +Orders.CALCULATE(day_name=DAYNAME(order_date)) ``` @@ -685,7 +693,7 @@ The `MONTHNAME` function returns the name of the month of the year for a given d ```py # Retuns the 3-letter abbrivation of the month for the order date -Orders.CALCULATE(day_name = MONTHNAME(order_date)) +Orders.CALCULATE(day_name=MONTHNAME(order_date)) ``` @@ -703,7 +711,7 @@ The `IFF` function cases on the True/False value of its first argument. If it is ```py qty_from_germany = IFF(supplier.nation.name == "GERMANY", quantity, 0) customers.CALCULATE( - total_quantity_shipped_from_germany = SUM(lines.CALCULATE(q=qty_from_germany).q) + total_quantity_shipped_from_germany=SUM(lines.CALCULATE(q=qty_from_germany).q) ) ``` @@ -724,7 +732,7 @@ Parts.WHERE(ISIN(size, (10, 11, 17, 19, 45))) The `DEFAULT_TO` function returns the first of its arguments that is non-null (e.g. the same as the `COALESCE` function in SQL): ```py -Lineitems.CALCULATE(adj_tax = DEFAULT_TO(tax, 0)) +Lineitems.CALCULATE(adj_tax=DEFAULT_TO(tax, 0)) ``` @@ -734,7 +742,7 @@ Lineitems.CALCULATE(adj_tax = DEFAULT_TO(tax, 0)) The `PRESENT` function checks if its argument is non-null (e.g. the same as `IS NOT NULL` in SQL): ```py -Lineitems.CALCULATE(has_tax = PRESENT(tax)) +Lineitems.CALCULATE(has_tax=PRESENT(tax)) ``` @@ -744,7 +752,7 @@ Lineitems.CALCULATE(has_tax = PRESENT(tax)) The `ABSENT` function checks if its argument is null (e.g. the same as `IS NULL` in SQL): ```py -Lineitems.CALCULATE(no_tax = ABSENT(tax)) +Lineitems.CALCULATE(no_tax=ABSENT(tax)) ``` @@ -754,7 +762,11 @@ Lineitems.CALCULATE(no_tax = ABSENT(tax)) The `KEEP_IF` function returns the first function if the second arguments is True, otherwise it returns a null value. In other words, `KEEP_IF(a, b)` is equivalent to the SQL expression `CASE WHEN b THEN a END`. ```py -TPCH.CALCULATE(avg_non_debt_balance = AVG(customers.CALCULATE(no_debt_bal = KEEP_IF(acctbal, acctbal > 0)).no_debt_bal)) +TPCH.CALCULATE( + avg_non_debt_balance=AVG( + customers.CALCULATE(no_debt_bal=KEEP_IF(acctbal, acctbal > 0)).no_debt_bal + ) +) ``` @@ -780,9 +792,9 @@ Below is each numerical function currently supported in PyDough. The `ABS` function returns the absolute value of its input. The Python builtin `abs()` function can also be used to accomplish the same thing. ```py -customers.CALCULATE(acct_magnitude = ABS(acctbal)) +customers.CALCULATE(acct_magnitude=ABS(acctbal)) # The below statement is equivalent to above. -customers.CALCULATE(acct_magnitude = abs(acctbal)) +customers.CALCULATE(acct_magnitude=abs(acctbal)) ``` @@ -792,14 +804,14 @@ customers.CALCULATE(acct_magnitude = abs(acctbal)) The `ROUND` function rounds its first argument to the precision of its second argument. The rounding rules used depend on the database's round function. The second argument is optional, and if not provided, the first argument is rounded to 0 decimal places (to match the Python builtin `round()` function). The Python builtin `round()` function can also be used to accomplish the same thing. ```py -Parts.CALCULATE(rounded_price = ROUND(retail_price, 1)) +Parts.CALCULATE(rounded_price=ROUND(retail_price, 1)) # The below statement is equivalent to above. -Parts.CALCULATE(rounded_price = round(retail_price, 1)) +Parts.CALCULATE(rounded_price=round(retail_price, 1)) # The below statement takes the default precision as 0. -Parts.CALCULATE(rounded_price = ROUND(retail_price)) +Parts.CALCULATE(rounded_price=ROUND(retail_price)) # The below statement is equivalent to above. -Parts.CALCULATE(rounded_price = ROUND(retail_price,0)) +Parts.CALCULATE(rounded_price=ROUND(retail_price, 0)) ``` @@ -809,7 +821,7 @@ Parts.CALCULATE(rounded_price = ROUND(retail_price,0)) The `CEIL` function rounds its argument up to the nearest integer. It returns the smallest integer value that is greater than or equal to the input. This is equivalent to Python's `math.ceil()` function and corresponds to the SQL `CEIL` or `CEILING` function. ```py -parts.CALCULATE(ceiled_price = CEIL(retail_price)) +parts.CALCULATE(ceiled_price=CEIL(retail_price)) ``` Here are examples on how `CEIL` works: | Input | Output | @@ -824,7 +836,7 @@ Note: `CEIL` only accepts a single numeric argument and always returns an intege The `FLOOR` function rounds its argument down to the nearest integer. It returns the greatest integer value that is less than or equal to the input. This is equivalent to Python's `math.floor()` function and the SQL `FLOOR` function. ```py -parts.CALCULATE(floored_price = FLOOR(retail_price)) +parts.CALCULATE(floored_price=FLOOR(retail_price)) ``` Here are examples on how `FLOOR` works: @@ -840,7 +852,7 @@ Note: `FLOOR` only accepts a single numeric argument and always returns an integ The `POWER` function exponentiates its first argument to the power of its second argument. ```py -Parts.CALCULATE(powered_price = POWER(retail_price, 2)) +Parts.CALCULATE(powered_price=POWER(retail_price, 2)) ``` @@ -850,7 +862,7 @@ Parts.CALCULATE(powered_price = POWER(retail_price, 2)) The `SQRT` function takes the square root of its input. It's equivalent to `POWER(x,0.5)`. ```py -Parts.CALCULATE(sqrt_price = SQRT(retail_price)) +Parts.CALCULATE(sqrt_price=SQRT(retail_price)) ``` @@ -860,7 +872,7 @@ Parts.CALCULATE(sqrt_price = SQRT(retail_price)) The `SIGN` function returns the sign of its input. It returns 1 if the input is positive, -1 if the input is negative, and 0 if the input is zero. ```py -Suppliers.CALCULATE(sign_of_acctbal = SIGN(account_balance)) +Suppliers.CALCULATE(sign_of_acctbal=SIGN(account_balance)) ``` @@ -871,16 +883,18 @@ The `SMALLEST` function returns the smallest value from the set of values it is ```py TPCH.CALCULATE( - s1=SMALLEST(20,10,10,-1,-2,100,-200), # Returns -200 - s2=SMALLEST(-0.001,-0.01,-0.0001), # Returns -0.0001 - s3=SMALLEST(datetime.datetime(2025,1,1),datetime.datetime(2024,1,1)), # Returns 2024-01-01 - s4=SMALLEST(1,None,3,4), # Returns NULL + s1=SMALLEST(20, 10, 10, -1, -2, 100, -200), # Returns -200 + s2=SMALLEST(-0.001, -0.01, -0.0001), # Returns -0.0001 + s3=SMALLEST( + datetime.datetime(2025, 1, 1), datetime.datetime(2024, 1, 1) + ), # Returns 2024-01-01 + s4=SMALLEST(1, None, 3, 4), # Returns NULL ) # Average gap, in days, for shipments between when they were ordered # versus when they were expected to arrive (or when they actually arrived, # if they were early), for shipments done via rail. delay_info = Lineitems.WHERE(HAS(order) & (ship_mode == "RAIL")).CALCULATE( - day_gap=DATEDIFF("days", order.order_date, SMALLEST(commit_date, receipt_date)) + day_gap=DATEDIFF("days", order.order_date, SMALLEST(commit_date, receipt_date)) ) return TPCH.CALCULATE(avg_gap=AVG(delay_info.day_gap)) ``` @@ -893,16 +907,18 @@ The `LARGEST` function returns the largest value from the set of values it is ca ```py TPCH.CALCULATE( - l1=LARGEST(20,10,10,-1,-2,100,-200), # Returns 100 - l2=LARGEST(-0.001,-0.01,-0.0001), # Returns -0.01 - l3=LARGEST(datetime.datetime(2025,1,1),datetime.datetime(2024,1,1)), # Returns 2025-01-01 - l4=LARGEST(1,None,3,4), # Returns NULL + l1=LARGEST(20, 10, 10, -1, -2, 100, -200), # Returns 100 + l2=LARGEST(-0.001, -0.01, -0.0001), # Returns -0.01 + l3=LARGEST( + datetime.datetime(2025, 1, 1), datetime.datetime(2024, 1, 1) + ), # Returns 2025-01-01 + l4=LARGEST(1, None, 3, 4), # Returns NULL ) # For each region, what is the average account balance of all # customers in a hypothetical scenario where all debt was erased Regions.CALCULATE( - region_name=name, - avg_bal_without_debt_erasure=AVG(LARGEST(nations.customers.acctbal, 0)), + region_name=name, + avg_bal_without_debt_erasure=AVG(LARGEST(nations.customers.acctbal, 0)), ) ``` @@ -921,7 +937,7 @@ Aggregation functions are a special set of functions that, when called on their The `SUM` function returns the sum of the plural set of numerical values it is called on. ```py -Nations.CALCULATE(total_consumer_wealth = SUM(customers.acctbal)) +Nations.CALCULATE(total_consumer_wealth=SUM(customers.acctbal)) ``` @@ -931,7 +947,7 @@ Nations.CALCULATE(total_consumer_wealth = SUM(customers.acctbal)) The `AVG` function takes the average of the plural set of numerical values it is called on. ```py -Parts.CALCULATE(average_shipment_size = AVG(lines.quantity)) +Parts.CALCULATE(average_shipment_size=AVG(lines.quantity)) ``` @@ -942,10 +958,7 @@ The `MEDIAN` function takes the median of the plural set of numerical values it Note: absent records are ignored when deriving the median. ```py -customers.CALCULATE( - name, - median_order_price = MEDIAN(orders.total_price) -) +customers.CALCULATE(name, median_order_price=MEDIAN(orders.total_price)) ``` @@ -955,7 +968,7 @@ customers.CALCULATE( The `MIN` function returns the smallest value from the set of values it is called on. ```py -Suppliers.CALCULATE(cheapest_part_supplied = MIN(supply_records.supply_cost)) +Suppliers.CALCULATE(cheapest_part_supplied=MIN(supply_records.supply_cost)) ``` @@ -965,7 +978,7 @@ Suppliers.CALCULATE(cheapest_part_supplied = MIN(supply_records.supply_cost)) The `MAX` function returns the largest value from the set of values it is called on. ```py -Suppliers.CALCULATE(most_expensive_part_supplied = MAX(supply_records.supply_cost)) +Suppliers.CALCULATE(most_expensive_part_supplied=MAX(supply_records.supply_cost)) ``` @@ -980,10 +993,10 @@ The `QUANTILE` function returns the value at a specified quantile from the set o ```py # Returns the value at the 90th percentile of supply costs for each supplier -Suppliers.CALCULATE(ninetieth_percentile_cost = QUANTILE(supply_records.supply_cost, 0.9)) +Suppliers.CALCULATE(ninetieth_percentile_cost=QUANTILE(supply_records.supply_cost, 0.9)) # Returns the median (50th percentile, discrete) supply cost for each supplier -Suppliers.CALCULATE(median_cost = QUANTILE(supply_records.supply_cost, 0.5)) +Suppliers.CALCULATE(median_cost=QUANTILE(supply_records.supply_cost, 0.5)) ``` - The first argument is the plural set of values to aggregate. @@ -1007,7 +1020,7 @@ Suppliers.CALCULATE(median_cost = QUANTILE(supply_records.supply_cost, 0.5)) The `ANYTHING` function returns an arbitrary value from the set of values it is called on. ```py -Suppliers.CALCULATE(chosen_part_name = ANYTHING(supply_records.part.name)) +Suppliers.CALCULATE(chosen_part_name=ANYTHING(supply_records.part.name)) ``` @@ -1017,13 +1030,13 @@ Suppliers.CALCULATE(chosen_part_name = ANYTHING(supply_records.part.name)) The `COUNT` function returns how many non-null records exist on the set of plural values it is called on. ```py -customers.CALCULATE(num_taxed_purchases = COUNT(orders.lines.tax)) +customers.CALCULATE(num_taxed_purchases=COUNT(orders.lines.tax)) ``` The `COUNT` function can also be called on a sub-collection, in which case it will return how many records from that sub-collection exist. ```py -Nations.CALCULATE(num_customers_in_debt = COUNT(customers.WHERE(acctbal < 0))) +Nations.CALCULATE(num_customers_in_debt=COUNT(customers.WHERE(acctbal < 0))) ``` @@ -1033,7 +1046,7 @@ Nations.CALCULATE(num_customers_in_debt = COUNT(customers.WHERE(acctbal < 0))) The `NDISTINCT` function returns how many distinct values of its argument exist. ```py -customers.CALCULATE(num_unique_parts_purchased = NDISTINCT(orders.lines.parts.key)) +customers.CALCULATE(num_unique_parts_purchased=NDISTINCT(orders.lines.parts.key)) ``` @@ -1067,10 +1080,10 @@ The `VAR` function returns the variance of the set of numerical values it is cal ```py # Compute the population variance -Parts.CALCULATE(variance = VAR(supply_records.supply_cost)) +Parts.CALCULATE(variance=VAR(supply_records.supply_cost)) # Compute the sample variance -Parts.CALCULATE(variance = VAR(supply_records.supply_cost, type="sample")) +Parts.CALCULATE(variance=VAR(supply_records.supply_cost, type="sample")) ``` @@ -1084,10 +1097,10 @@ The `STD` function returns the standard deviation of the set of numerical values ```py # Compute the population standard deviation -Parts.CALCULATE(std = STD(supply_records.supply_cost)) +Parts.CALCULATE(std=STD(supply_records.supply_cost)) # Compute the sample standard deviation -Parts.CALCULATE(std = STD(supply_records.supply_cost, type="sample")) +Parts.CALCULATE(std=STD(supply_records.supply_cost, type="sample")) ``` @@ -1105,7 +1118,7 @@ Regions.CALCULATE(region_name=name, nation_names=LISTOF(nations.name)) # For each customer, list the three largest quantities purchased made by that # customer. -selected_lines = orders.lines.best(by=quantity.DESC(), per='Customers', n_best=3) +selected_lines = orders.lines.best(by=quantity.DESC(), per="Customers", n_best=3) Customers.CALCULATE(customer_name=name, quantities=LISTOF(selected_lines.quantity)) ``` @@ -1177,11 +1190,13 @@ The `RANKING` function returns ordinal position of the current record when all r ```py # Rank customers per-nation by their account balance # (highest = rank #1, no ties) -Nations.customers.CALCULATE(r = RANKING(by=acctbal.DESC(), per="Nations")) +Nations.customers.CALCULATE(r=RANKING(by=acctbal.DESC(), per="Nations")) # For every customer, finds their most recent order # (ties allowed) -customers.orders.WHERE(RANKING(by=order_date.DESC(), per="customers", allow_ties=True) == 1) +customers.orders.WHERE( + RANKING(by=order_date.DESC(), per="customers", allow_ties=True) == 1 +) ``` @@ -1218,27 +1233,27 @@ The `PREV` function returns the value of an expression from a preceding record i # Find the 10 customers with at least 5 orders with the largest average time # gap between their orders, in days. order_info = orders.CALCULATE( - day_diff=DATEDIFF("days", PREV(order_date, by=order_date.ASC(), per="customers"), order_date) + day_diff=DATEDIFF( + "days", PREV(order_date, by=order_date.ASC(), per="customers"), order_date + ) ) customers.WHERE(COUNT(orders) > 5).CALCULATE( - name, - average_order_gap=AVG(order_info.day_diff) + name, average_order_gap=AVG(order_info.day_diff) ).TOP_K(10, by=average_order_gap.DESC()) # For every year/month, calculate the percent change in the number of # orders made in that month from the previous month. PARTITION( - Orders(year=YEAR(order_date), month=MONTH(order_date)), - name="orders", - by=(year, month) + Orders(year=YEAR(order_date), month=MONTH(order_date)), + name="orders", + by=(year, month), ).CALCULATE( - year, - month, - n_orders=COUNT(orders), - pct_change= - 100.0 - * (COUNT(orders) - PREV(COUNT(orders), by=(year.ASC(), month.ASC()))) - / PREV(COUNT(orders), by=(year.ASC(), month.ASC())) + year, + month, + n_orders=COUNT(orders), + pct_change=100.0 + * (COUNT(orders) - PREV(COUNT(orders), by=(year.ASC(), month.ASC()))) + / PREV(COUNT(orders), by=(year.ASC(), month.ASC())), ) ``` @@ -1287,11 +1302,15 @@ Nations.customers.CALCULATE(ratio=acctbal / RELSUM(acctbal, per="Nations")) # Finds, for each customer, the wealth of that customer combined with all # poorer customers. -Customers.CALCULATE(cumulative_wealth=RELSUM(acctbal, by=acctbal.ASC(), cumulative=True)) +Customers.CALCULATE( + cumulative_wealth=RELSUM(acctbal, by=acctbal.ASC(), cumulative=True) +) # Finds, for each customer's orders, the sum of the total price of that # order and the two most recent orders before it. -Customers.orders.CALCULATE(sliding_sum=RELSUM(total_price, by=order_date.ASC(), per="Customers", frame=(-2, 0))) +Customers.orders.CALCULATE( + sliding_sum=RELSUM(total_price, by=order_date.ASC(), per="Customers", frame=(-2, 0)) +) ``` @@ -1325,12 +1344,18 @@ Nations.customers.WHERE(acctbal > RELAVG(acctbal, per="Nations")) # Finds the cumulative average of the total price of orders when sorted by # order date (breaking ties by the order key). -Orders.CALCULATE(average_price_so_far=RELAVG(total_price, by=(order_date.ASC(), key.ASC()), cumulative=True)) +Orders.CALCULATE( + average_price_so_far=RELAVG( + total_price, by=(order_date.ASC(), key.ASC()), cumulative=True + ) +) # For each order, finds the average of the total price of that order and the 5 # orders before/after it when sorted by order date (breaking ties by the order # key). -Orders.CALCULATE(window_average=RELAVG(total_price, by=(order_date.ASC(), key.ASC()), frame=(-5, 5))) +Orders.CALCULATE( + window_average=RELAVG(total_price, by=(order_date.ASC(), key.ASC()), frame=(-5, 5)) +) ``` @@ -1357,19 +1382,30 @@ The `RELCOUNT` function returns the number of non-null records in multiple rows ```py # Divides each customer's account balance by the total number of positive # account balances globally. -customers.CALCULATE(ratio = acctbal / RELCOUNT(KEEP_IF(acctbal, acctbal > 0.0))) +customers.CALCULATE(ratio=acctbal / RELCOUNT(KEEP_IF(acctbal, acctbal > 0.0))) # Divides each customer's account balance by the total number of positive # account balances in the same nation. -Nations.customers.CALCULATE(ratio = acctbal / RELCOUNT(KEEP_IF(acctbal, acctbal > 0.0), per="Nations")) +Nations.customers.CALCULATE( + ratio=acctbal / RELCOUNT(KEEP_IF(acctbal, acctbal > 0.0), per="Nations") +) # For each customer, count how many customers are poorer than them but are not # in debt. -Customers.CALCULATE(n_poorer_non_debt=RELCOUNT(KEEP_IF(acctbal, acctbal >= 0), by=(acctbal.ASC()), cumulative=True) - (acctbal >= 0)) +Customers.CALCULATE( + n_poorer_non_debt=RELCOUNT( + KEEP_IF(acctbal, acctbal >= 0), by=(acctbal.ASC()), cumulative=True + ) + - (acctbal >= 0) +) # Same as previous example, but using frames to exclude the current record # instead of subtracting (acctbal >= 0) -Customers.CALCULATE(n_poorer_non_debt=RELCOUNT(KEEP_IF(acctbal, acctbal >= 0), by=(acctbal.ASC()), frame=(None, -1))) +Customers.CALCULATE( + n_poorer_non_debt=RELCOUNT( + KEEP_IF(acctbal, acctbal >= 0), by=(acctbal.ASC()), frame=(None, -1) + ) +) ``` @@ -1395,18 +1431,20 @@ The `RELSIZE` function returns the number of total records, either globally or t ```py # Divides each customer's account balance by # the number of total customers. -customers.CALCULATE(ratio = acctbal / RELSIZE()) +customers.CALCULATE(ratio=acctbal / RELSIZE()) # Divides each customer's account balance by the # number of total customers in that nation. -Nations.customers.CALCULATE(ratio = acctbal / RELSIZE(per="Nations")) +Nations.customers.CALCULATE(ratio=acctbal / RELSIZE(per="Nations")) # For each customer, returns the number of customers poorer than them. Customers.CALCULATE(customers_poorer=RELSIZE(by=(acctbal.ASC()), cumulative=True) - 1) # For each customer's orders, count how many orders the customer made AFTER the # current order. -Customers.orders.CALCULATE(n_orders_after=RELSIZE(by=(order_date.ASC(), key.ASC()), frame=(1, None))) +Customers.orders.CALCULATE( + n_orders_after=RELSIZE(by=(order_date.ASC(), key.ASC()), frame=(1, None)) +) ``` @@ -1422,14 +1460,14 @@ The `STRING` function casts the first argument to a string data type. The first ```py Orders.CALCULATE( - # Casts the key column (numeric type) to a string. - key_string=STRING(key), - # Casts the order_date column (date type) to a string - # with the format YYYY-MM-DD. - # Please refer to your underlying database's documentation - # for the format strings it supports. - # In this case, the database used is SQLite. - order_date_string=STRING(order_date, "%Y-%m-%d"), + # Casts the key column (numeric type) to a string. + key_string=STRING(key), + # Casts the order_date column (date type) to a string + # with the format YYYY-MM-DD. + # Please refer to your underlying database's documentation + # for the format strings it supports. + # In this case, the database used is SQLite. + order_date_string=STRING(order_date, "%Y-%m-%d"), ) ``` @@ -1453,10 +1491,10 @@ The `INTEGER` function casts the argument to an integer. ```py Orders.CALCULATE( - # Casts the total_price column (decimal type) to an integer. - total_price_int=INTEGER(total_price), - # Casts the string "2" to an integer. - discount = INTEGER("2") + # Casts the total_price column (decimal type) to an integer. + total_price_int=INTEGER(total_price), + # Casts the string "2" to an integer. + discount=INTEGER("2"), ) ``` @@ -1468,10 +1506,10 @@ The `FLOAT` function casts the argument to a float. ```py Orders.CALCULATE( - # Casts the ship_priority column (integer type) to a float. - ship_priority_float=FLOAT(ship_priority), - # Casts the string "2" to a float. - discount = FLOAT("-2.71") + # Casts the ship_priority column (integer type) to a float. + ship_priority_float=FLOAT(ship_priority), + # Casts the string "2" to a float. + discount=FLOAT("-2.71"), ) ``` @@ -1488,17 +1526,17 @@ The `__bool__` magic method is not supported in PyDough. PyDough code cannot be ```py # Not allowed - will raise PyDoughUnqualifiedException if Customer and Order: - print("Available") + print("Available") customers.WHERE((acctbal > 0) and (nation.name == "GERMANY")) # Use &`instead of `and`: # customers.WHERE((acctbal > 0) & (nation.name == "GERMANY")) Orders.WHERE((discount > 0.05) or (tax > 0.08)) -# Use `|` instead of `or` +# Use `|` instead of `or` # Orders.WHERE((discount > 0.05) | (tax > 0.08)) -Parts.WHERE(not(retail_price > 1000)) +Parts.WHERE(not (retail_price > 1000)) # Use `~` instead of `not` # Parts.WHERE(~(retail_price > 1000)) ``` @@ -1631,7 +1669,7 @@ Using the `in` operator calls the `__contains__` magic method, which is not supp ```py # Not allowed currently as it would need to return a boolean instead of a PyDough object -Orders('discount' in order.details) +Orders("discount" in order.details) ``` @@ -1642,7 +1680,7 @@ Assigning to an index calls the `__setitem__` magic method, which is not support ```py # Not allowed currently as PyDough objects cannot support item assignment. -Order.details['discount'] = True +Order.details["discount"] = True ``` @@ -1654,7 +1692,7 @@ Iterating over an object calls the `__iter__` magic method, which is not support ```py # Not allowed currently as implementation has to return an iterator instead of a PyDough object. for item in customer: - print(item) + print(item) [item for item in customer] diff --git a/documentation/metadata.md b/documentation/metadata.md index a1e3e33b3..70fa6fa34 100644 --- a/documentation/metadata.md +++ b/documentation/metadata.md @@ -772,10 +772,9 @@ def cumulative_orders_counter(base_year: int, last_year: int): if base_year == last_year: result = COUNT(orders.WHERE(YEAR(order_date) == base_year)) else: - result = ( - COUNT(orders.WHERE(YEAR(order_date) == base_year)) - + cumulative_orders_counter(base_year + 1, last_year) - ) + result = COUNT( + orders.WHERE(YEAR(order_date) == base_year) + ) + cumulative_orders_counter(base_year + 1, last_year) return result ``` diff --git a/documentation/usage.md b/documentation/usage.md index 3926b319d..5d21c70c6 100644 --- a/documentation/usage.md +++ b/documentation/usage.md @@ -75,6 +75,7 @@ A new `PyDoughSession` object can be created as follows: ```py from pydough.configs import PyDoughSession + session = PyDoughSession() ``` @@ -83,6 +84,7 @@ There exists a special session object `pydough.active_session` used to describe ```py import pydough from pydough.configs import PyDoughSession + # Create a new session session_a = PyDoughSession() # Capture the original active session @@ -133,6 +135,7 @@ The configurations of a session object can be accessed with the `.config` field ```py import pydough from pydough.configs import PyDoughConfigs + # Create a new session new_configs = PyDoughConfigs() # Capture the original configs of the active session @@ -164,6 +167,7 @@ Not every nation has at least 1 customer with a name containing the substring `2 ```py import pydough + # Capture the configs of the active session configs = pydough.active_session.config # Access the original value of sum_default_zero (which should be True) @@ -276,24 +280,24 @@ configs = pydough.active_session.config configs.start_of_week = DayOfWeek.MONDAY # Set start of week to be 1 configs.start_week_as_zero = False -# This would return dow = 4, as March 20, 2025 is a Thursday and is the 4th day +# This would return dow = 4, as March 20, 2025 is a Thursday and is the 4th day # of the week when the start of week is considered to be Monday -TPCH.calculate(dow = DAYOFWEEK("2025-03-20")) # dow = 4 -# If start_week_as_zero is set to True (configs.start_week_as_zero = True), +TPCH.calculate(dow=DAYOFWEEK("2025-03-20")) # dow = 4 +# If start_week_as_zero is set to True (configs.start_week_as_zero = True), # then DAYOFWEEK("2025-03-20") = 3 configs.start_week_as_zero = True -TPCH.calculate(dow = DAYOFWEEK("2025-03-20")) # dow = 3 +TPCH.calculate(dow=DAYOFWEEK("2025-03-20")) # dow = 3 # Now, start_of_week is set to Thursday, configs.start_of_week = DayOfWeek.Thursday configs.start_week_as_zero = False # This would return dow = 1, as March 20, 2025 is a Thursday and is the 1st day # of the week when the start of week is considered to be Thursday. -TPCH.calculate(dow = DAYOFWEEK("2025-03-20")) # dow = 1 -# If start_week_as_zero is set to True (configs.start_week_as_zero = True), +TPCH.calculate(dow=DAYOFWEEK("2025-03-20")) # dow = 1 +# If start_week_as_zero is set to True (configs.start_week_as_zero = True), # then DAYOFWEEK("2025-03-20") = 0 configs.start_week_as_zero = True -TPCH.calculate(dow = DAYOFWEEK("2025-03-20")) # dow = 0 +TPCH.calculate(dow=DAYOFWEEK("2025-03-20")) # dow = 0 # Understanding the behavior of DATETIME with `start of week` @@ -301,34 +305,34 @@ TPCH.calculate(dow = DAYOFWEEK("2025-03-20")) # dow = 0 configs.start_of_week = DayOfWeek.MONDAY # This would set `dt` to the start of the week for March 20, 2025 (Wednesday) # to be March 17, 2025 (recent Monday) -TPCH.calculate(dt = DATETIME("2025-03-20", "start of week")) # dt = 2025-03-17 +TPCH.calculate(dt=DATETIME("2025-03-20", "start of week")) # dt = 2025-03-17 # start_of_week is set to Thursday, configs.start_of_week = DayOfWeek.Thursday # This would set `dt` to the start of the week for March 20, 2025 (Thursday) # to be March 20, 2025 (recent Thursday) -TPCH.calculate(dt = DATETIME("2025-03-20", "start of week")) # dt = 2025-03-20 -# Note: DATETIME function related to week does not depend on the +TPCH.calculate(dt=DATETIME("2025-03-20", "start of week")) # dt = 2025-03-20 +# Note: DATETIME function related to week does not depend on the # `start_week_as_zero` config. # Understanding the behavior of DATEDIFF with `start of week` # Set the start of the week to Monday configs.start_of_week = DayOfWeek.MONDAY -# This would return 0, as the difference between March 20, 2025(Thursday) and -# March 17, 2025(Monday) is 0 weeks when the start of week is considered to be +# This would return 0, as the difference between March 20, 2025(Thursday) and +# March 17, 2025(Monday) is 0 weeks when the start of week is considered to be # Monday. This is because both these dates fall on the same week. -TPCH.calculate(diff = DATEDIFF("2025-03-17", "2025-03-20", "week")) # diff = 0 +TPCH.calculate(diff=DATEDIFF("2025-03-17", "2025-03-20", "week")) # diff = 0 # Set the start of the week to Thursday configs.start_of_week = DayOfWeek.Thursday # This would return 1, as the difference between March 20, 2025 (Thursday) -# and March 17, 2025 (Monday) is 1 week when the start of week is considered +# and March 17, 2025 (Monday) is 1 week when the start of week is considered # to be Thursday. This is because March 17, 2025 belongs to the week starting -# from March 13, 2025 (the recent Thursday for it) and March 20, 2025 belongs +# from March 13, 2025 (the recent Thursday for it) and March 20, 2025 belongs # to the week starting from March 20, 2025 (the recent Thursday). -TPCH.calculate(diff = DATEDIFF("2025-03-17", "2025-03-20", "week")) # diff = 1 -# Note: DATEDIFF function related to week does not depend on the +TPCH.calculate(diff=DATEDIFF("2025-03-17", "2025-03-20", "week")) # diff = 1 +# Note: DATEDIFF function related to week does not depend on the # `start_week_as_zero` config. ``` @@ -399,7 +403,9 @@ education_context = load_database_context("sqlite", database="db_files/education pydough.active_session.database = education_context # Same but for another database & with a different method -shakespeare_context = pydough.active_session.load_database("sqlite", database="db_files/education.db") +shakespeare_context = pydough.active_session.load_database( + "sqlite", database="db_files/education.db" +) ``` Notice that both APIs `load_database_context` and `sesion.load_database` take in the name of the database type first and all the connection keyword arguments, and also return the context object. @@ -413,90 +419,109 @@ It is important to ensure that the correct database context is being used for se Password authentication: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/snowflake_sample_graphs.json", "TPCH") - pydough.active_session.connect_database("snowflake", - user=snowflake_username, - password=snowflake_password, - account=snowflake_account, - warehouse=snowflake_warehouse, - database=snowflake_database, - schema=snowflake_schema - ) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/snowflake_sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database( + "snowflake", + user=snowflake_username, + password=snowflake_password, + account=snowflake_account, + warehouse=snowflake_warehouse, + database=snowflake_database, + schema=snowflake_schema, + ) ``` Key pair (JWT) authentication The private key file can be an unencrypted or encrypted PEM PKCS#8 file; pass `private_key_file_pwd` only if the key is encrypted: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/snowflake_sample_graphs.json", "TPCH") - pydough.active_session.connect_database("snowflake", - user=snowflake_username, - account=snowflake_account, - authenticator="snowflake_jwt", - private_key_file=snowflake_private_key_file_path, - private_key_file_pwd=snowflake_private_key_file_pwd, # omit if the key is unencrypted - warehouse=snowflake_warehouse, - database=snowflake_database, - schema=snowflake_schema - ) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/snowflake_sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database( + "snowflake", + user=snowflake_username, + account=snowflake_account, + authenticator="snowflake_jwt", + private_key_file=snowflake_private_key_file_path, + private_key_file_pwd=snowflake_private_key_file_pwd, # omit if the key is unencrypted + warehouse=snowflake_warehouse, + database=snowflake_database, + schema=snowflake_schema, + ) ``` You can find a full example of using Snowflake database with PyDough in [this usage guide](./../demos/notebooks/SF_TPCH_q1.ipynb). - MySQL: You can connect to a mysql database using `load_metadata_graph` and `connect_database` APIs. For example: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - pydough.active_session.connect_database("mysql", - user=mysql_username, - password=mysql_password, - database=mysql_tpch_db, - host=mysql_host, - ) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database( + "mysql", + user=mysql_username, + password=mysql_password, + database=mysql_tpch_db, + host=mysql_host, + ) ``` You can find a full example of using MySQL database with PyDough in [this usage guide](./../demos/notebooks/MySQL_TPCH.ipynb). - Postgres: You can connect to a postgres database using `load_metadata_graph` and `connect_database` APIs. For example: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - pydough.active_session.connect_database("postgres", - user=postgres_username, - password=postgres_password, - database=postgres_db, - host=postgres_host, - ) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database( + "postgres", + user=postgres_username, + password=postgres_password, + database=postgres_db, + host=postgres_host, + ) ``` Example with a connection object ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - postgres_conn: psycopg2.extensions.connection = psycopg2.connect( - dbname=postgres_db, - user=postgres_user, - password=postgres_password, - host=postgres_host, - port=postgres_port, - ) - pydough.active_session.connect_database("postgres", connection=postgres_conn) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + postgres_conn: psycopg2.extensions.connection = psycopg2.connect( + dbname=postgres_db, + user=postgres_user, + password=postgres_password, + host=postgres_host, + port=postgres_port, + ) + pydough.active_session.connect_database("postgres", connection=postgres_conn) ``` You can find a full example of using Postgres database with PyDough in [this usage guide](./../demos/notebooks/PG_TPCH.ipynb). - Postgres: You can connect to a postgres database using `load_metadata_graph` and `connect_database` APIs. For example: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - pydough.active_session.connect_database("trino", - host=TRINO_HOST, - port=TRINO_PORT, - user=TRINO_USER, - ) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database( + "trino", + host=TRINO_HOST, + port=TRINO_PORT, + user=TRINO_USER, + ) ``` Example with a connection object ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - trino_conn: trino.dbapi.Connection = trino.dbapi.connect( - host=TRINO_HOST, - port=TRINO_PORT, - user=TRINO_USER, - ) - pydough.active_session.connect_database("trino", connection=trino_conn) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + trino_conn: trino.dbapi.Connection = trino.dbapi.connect( + host=TRINO_HOST, + port=TRINO_PORT, + user=TRINO_USER, + ) + pydough.active_session.connect_database("trino", connection=trino_conn) ``` You can find a full example of using Trino database with PyDough in [this usage guide](./../demos/notebooks/Trino_TPCH.ipynb). @@ -515,16 +540,18 @@ You can find a full example of using Trino database with PyDough in [this usage Example with a connection object ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - oracle_conn: oracledb.connection = oracledb.connect( - dbname=oracle_db, - user=oracle_user, - password=oracle_password, - host=oracle_host, - port=oracle_port, - service_name=oracle_service_name, - ) - pydough.active_session.connect_database("oracle", connection=oracle_conn) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + oracle_conn: oracledb.connection = oracledb.connect( + dbname=oracle_db, + user=oracle_user, + password=oracle_password, + host=oracle_host, + port=oracle_port, + service_name=oracle_service_name, + ) + pydough.active_session.connect_database("oracle", connection=oracle_conn) ``` You can find a full example of using an Oracle database with PyDough in [this usage guide](./../demos/notebooks/Oracle_TPCH.ipynb). @@ -536,37 +563,48 @@ literals. This ensures consistent behavior across DATE expressions. But sub-seco - Databricks: You can connect to a Databricks database using `load_metadata_graph` and `connect_database` APIs. For example: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/databricks_sample_graphs.json", "TPCH") - pydough.active_session.connect_database("databricks", - server_hostname=databricks_host, - http_path=databricks_http_path, - access_token=databricks_token - ) + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/databricks_sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database( + "databricks", + server_hostname=databricks_host, + http_path=databricks_http_path, + access_token=databricks_token, + ) ``` Example with a connection object ```py - from databricks import sql - pydough.active_session.load_metadata_graph("../../tests/test_metadata/databricks_sample_graphs.json", "TPCH") - databricks_conn = sql.connect( - server_hostname=databricks_host, - http_path=databricks_http_path, - access_token=databricks_token - ) - pydough.active_session.connect_database("databricks", connection=databricks_conn) + from databricks import sql + + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/databricks_sample_graphs.json", "TPCH" + ) + databricks_conn = sql.connect( + server_hostname=databricks_host, + http_path=databricks_http_path, + access_token=databricks_token, + ) + pydough.active_session.connect_database("databricks", connection=databricks_conn) ``` You can find a full example of using a Databricks database with PyDough in [this usage guide](./../demos/notebooks/Databricks_TPCH.ipynb). - DuckDB: You can connect to a DuckDB database using `load_metadata_graph` and `connect_database` APIs. For example: ```py - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - pydough.active_session.connect_database("duckdb", database=":memory:") + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + pydough.active_session.connect_database("duckdb", database=":memory:") ``` Example with a connection object ```py - import duckdb - pydough.active_session.load_metadata_graph("../../tests/test_metadata/sample_graphs.json", "TPCH") - duckdb_conn = duckdb.connect(database=":memory:") - pydough.active_session.connect_database("duckdb", connection=duckdb_conn) + import duckdb + + pydough.active_session.load_metadata_graph( + "../../tests/test_metadata/sample_graphs.json", "TPCH" + ) + duckdb_conn = duckdb.connect(database=":memory:") + pydough.active_session.connect_database("duckdb", connection=duckdb_conn) ``` You can find a full example of using a DuckDB database with PyDough in [this usage guide](./../demos/notebooks/DuckDB_TPCH.ipynb). @@ -844,13 +882,17 @@ This first example is of Python code using `pydough.from_string` to generate SQL import pydough # Setup demo metadata. Make sure you have the TPC-H dataset downloaded locally. -graph = pydough.active_session.load_metadata_graph("demos/metadata/tpch_demo_graph.json", "TPCH") +graph = pydough.active_session.load_metadata_graph( + "demos/metadata/tpch_demo_graph.json", "TPCH" +) pydough.active_session.connect_database("sqlite", database="tpch.db") # Example of a single line pydough code snippet -pydough_code = "pydough_query = TPCH.CALCULATE(n=COUNT(customers.WHERE(market_segment == SEG)))" +pydough_code = ( + "pydough_query = TPCH.CALCULATE(n=COUNT(customers.WHERE(market_segment == SEG)))" +) # Transform the pydough code and get the result from pydough_query -query = pydough.from_string(pydough_code, "pydough_query", graph, {"SEG":"AUTOMOBILE"}) +query = pydough.from_string(pydough_code, "pydough_query", graph, {"SEG": "AUTOMOBILE"}) sql = pydough.to_sql(query) ``` @@ -869,9 +911,13 @@ This next example is of Python code to generate SQL to get the top 5 suppliers w nation_name = "JAPAN" ship_mode = "TRUCK" ship_year = 1996 -env = {"TARGET_NATION" : nation_name, "DESIRED_SHIP_MODE" : ship_mode, "REQUESTED_SHIP_YEAR" : ship_year} +env = { + "TARGET_NATION": nation_name, + "DESIRED_SHIP_MODE": ship_mode, + "REQUESTED_SHIP_YEAR": ship_year, +} -pydough_code=""" +pydough_code = """ # The supply records for the supplier that were from a medium part selected_records = supply_records.WHERE(STARTSWITH(part.name, "coral")).CALCULATE(supply_cost) @@ -931,8 +977,8 @@ LIMIT 5 This final example is of Python code to generate an SQL query, using 'datetime.date' passed in through the environment. ```py -# For every customer, how many urgent orders have they made in year 1996 with a -# total price over 100000, and what is the sum of the total prices of all such +# For every customer, how many urgent orders have they made in year 1996 with a +# total price over 100000, and what is the sum of the total prices of all such # orders they made? Sort the result by the sum from highest to lowest, and only # include customers with at least one such order @@ -940,9 +986,10 @@ This final example is of Python code to generate an SQL query, using 'datetime.d # and date function from datetime. Optionally, we could import datetime.date # from inside the pydough code string import datetime -env = {"date" : datetime.date, "YEAR" : 1996} -pydough_code=""" +env = {"date": datetime.date, "YEAR": 1996} + +pydough_code = """ selected_orders=orders.WHERE((order_priority == '1-URGENT') & (total_price > 100000) & (order_date >= date(YEAR, 1, 1)) & (order_date < date(YEAR + 1, 1, 1))) @@ -1040,7 +1087,9 @@ Using the `orders_revenue_by` template and the `years` / `order_dimensions` attr ```python import pydough -graph = pydough.active_session.load_metadata_graph("demos/metadata/tpch_demo_graph.json", "TPCH") +graph = pydough.active_session.load_metadata_graph( + "demos/metadata/tpch_demo_graph.json", "TPCH" +) pydough.active_session.connect_database("sqlite", database="tpch.db") result = pydough.call_template( @@ -1068,6 +1117,7 @@ Below is an example that displays information about the TPC-H schema, truncated ```py import pydough + graph = pydough.parse_json_metadata_from_file("insert_path_here.json", "TPCH") pydough.explain_structure(graph) ``` @@ -1127,6 +1177,7 @@ Below are examples of each of these behaviors, using a knowledge graph for the T ```py import pydough + graph = pydough.parse_json_metadata_from_file("insert_path_here.json", "TPCH") pydough.explain(graph, verbose=True) ``` @@ -1150,6 +1201,7 @@ Call pydough.explain_structure(graph) to see how all of the collections in the g ```py import pydough + graph = pydough.parse_json_metadata_from_file("insert_path_here.json", "TPCH") pydough.explain(graph["nations"], verbose=True) ``` @@ -1174,6 +1226,7 @@ Call pydough.explain(graph['nations'][property_name]) to learn more about any of ```py import pydough + graph = pydough.parse_json_metadata_from_file("insert_path_here.json", "TPCH") pydough.explain(graph["nations"]["name"], verbose=True) ``` @@ -1188,6 +1241,7 @@ Data type: string ```py import pydough + graph = pydough.parse_json_metadata_from_file("insert_path_here.json", "TPCH") pydough.explain(graph["nations"]["customers"], verbose=True) ``` @@ -1475,8 +1529,8 @@ The output always has a consistent shape regardless of whether the expression is { "error": False, "query_summary": "...", # deterministic plain-English sentence - "steps": [...], # ordered list of operation steps - "schema": {...} # source collection, output columns, ordering, limit + "steps": [...], # ordered list of operation steps + "schema": {...}, # source collection, output columns, ordering, limit } # Failure (unrecognised term, wrong type, etc.) @@ -1484,7 +1538,7 @@ The output always has a consistent shape regardless of whether the expression is "error": True, "message": "Unrecognised term 'typo'. Did you mean: name?", "steps": [], - "schema": None + "schema": None, } ``` @@ -1676,6 +1730,7 @@ Here is an example of basic usage. We have not set the environment variable, hen ```py from pydough import get_logger + pyd_logger = get_logger(__name__) logger.info("This is an info message.") diff --git a/pydough/conversion/column_bubbler.py b/pydough/conversion/column_bubbler.py index 880c79da9..5d3c796ac 100644 --- a/pydough/conversion/column_bubbler.py +++ b/pydough/conversion/column_bubbler.py @@ -42,7 +42,7 @@ def name_sort_key(name: str) -> tuple[bool, bool, str]: comparison). """ return ( - name.startswith("expr") or name.startswith("agg"), + name.startswith(("expr", "agg")), any(char.isdigit() for char in name), name, ) @@ -87,7 +87,7 @@ def generate_cleaner_names(expr: RelationalExpression, current_name: str) -> lis if len(expr.inputs) == 0 and expr.op.function_name.lower() == "count": result.append("n_rows") - if not (current_name.startswith("agg") or current_name.startswith("expr")): + if not (current_name.startswith(("agg", "expr"))): if re.match(r"^(.*)_[0-9]+$", current_name): result.append(re.findall(r"^(.*)_[0-9]+$", current_name)[0]) return result diff --git a/pydough/conversion/hybrid_tree.py b/pydough/conversion/hybrid_tree.py index 679ff9e88..0b929096d 100644 --- a/pydough/conversion/hybrid_tree.py +++ b/pydough/conversion/hybrid_tree.py @@ -526,24 +526,21 @@ def add_child( if min_steps >= existing_connection.max_steps: if connection_type.is_anti: continue - if connection_type.is_semi: - if not ( - always_exists or existing_connection.connection_type.is_semi - ): - # Special case: When applying a SEMI join: - # - If the child is an AGGREGATION, add a COUNT to - # the aggregation and filter in the parent tree - # to check that the count is greater than zero. - # - If the child is SINGULAR, do the same but - # use a PRESENT filter to check that a value - # exists. - if is_singular: - self.insert_presence_filter( - idx, connection_type.is_semi - ) - else: - self.insert_count_filter(idx, True) - return idx + if connection_type.is_semi and not ( + always_exists or existing_connection.connection_type.is_semi + ): + # Special case: When applying a SEMI join: + # - If the child is an AGGREGATION, add a COUNT to + # the aggregation and filter in the parent tree + # to check that the count is greater than zero. + # - If the child is SINGULAR, do the same but + # use a PRESENT filter to check that a value + # exists. + if is_singular: + self.insert_presence_filter(idx, connection_type.is_semi) + else: + self.insert_count_filter(idx, True) + return idx # If combining a semi/anti with an existing non-semi/anti # and filters are banned, keep the existing connection type # and insert a count/presence filter into the tree so that it diff --git a/pydough/conversion/relational_simplification.py b/pydough/conversion/relational_simplification.py index 2eccb5b8a..ae9a2691a 100644 --- a/pydough/conversion/relational_simplification.py +++ b/pydough/conversion/relational_simplification.py @@ -860,19 +860,17 @@ def simplify_function_call( # Return None if any of the inputs are None and the operator is # guaranteed to return NULL if any of its inptus are NULL. - if expr.op in NULL_IF_INPUT_NULL_OPS: - if any( - isinstance(arg, LiteralExpression) and arg.value is None - for arg in expr.inputs - ): - self.stack.append(output_predicates) - return LiteralExpression(None, expr.data_type) + if expr.op in NULL_IF_INPUT_NULL_OPS and any( + isinstance(arg, LiteralExpression) and arg.value is None + for arg in expr.inputs + ): + self.stack.append(output_predicates) + return LiteralExpression(None, expr.data_type) # If the call has null propagating rules, all of the arguments are # non-null, the output is guaranteed to be non-null. - if expr.op in NULLABLE_IF_INPUT_NULLABLE_OPS: - if intersect_set.not_null: - output_predicates.not_null = True + if expr.op in NULLABLE_IF_INPUT_NULLABLE_OPS and intersect_set.not_null: + output_predicates.not_null = True match expr.op: case pydop.COUNT | pydop.NDISTINCT: @@ -915,24 +913,23 @@ def simplify_function_call( not_negative=True, positive=True, ) - if expr.op == pydop.SUM: + if expr.op == pydop.SUM and ( + isinstance(expr.inputs[0], CallExpression) + and expr.inputs[0].op == pydop.IFF + ): + # SUM(IFF(cond, 1, 0)) -> SUM(cond) + cond_arg: RelationalExpression = expr.inputs[0].inputs[0] + first_arg: RelationalExpression = expr.inputs[0].inputs[1] + second_arg: RelationalExpression = expr.inputs[0].inputs[2] if ( - isinstance(expr.inputs[0], CallExpression) - and expr.inputs[0].op == pydop.IFF + isinstance(first_arg, LiteralExpression) + and first_arg.value in (1, 1.0, True) + and isinstance(second_arg, LiteralExpression) + and second_arg.value in (0, 0.0, False) ): - # SUM(IFF(cond, 1, 0)) -> SUM(cond) - cond_arg: RelationalExpression = expr.inputs[0].inputs[0] - first_arg: RelationalExpression = expr.inputs[0].inputs[1] - second_arg: RelationalExpression = expr.inputs[0].inputs[2] - if ( - isinstance(first_arg, LiteralExpression) - and first_arg.value in (1, 1.0, True) - and isinstance(second_arg, LiteralExpression) - and second_arg.value in (0, 0.0, False) - ): - output_expr = CallExpression( - pydop.SUM, expr.data_type, [cond_arg] - ) + output_expr = CallExpression( + pydop.SUM, expr.data_type, [cond_arg] + ) # INTEGER(x) -> x if x is a literal integer. Also simplify for # booleans. @@ -1149,7 +1146,7 @@ def simplify_function_call( # MONOTONIC(x, y, z), where x/y/z are all literals # -> True if x <= y <= z, False otherwise if v0 is not None and v1 is not None and v2 is not None: - monotonic_result = (v0 <= v1) and (v1 <= v2) + monotonic_result = v0 <= v1 <= v2 output_expr = LiteralExpression(monotonic_result, expr.data_type) if monotonic_result: output_predicates.positive = True diff --git a/pydough/exploration/README.md b/pydough/exploration/README.md index 641ec9ea6..bbc27396f 100644 --- a/pydough/exploration/README.md +++ b/pydough/exploration/README.md @@ -153,8 +153,8 @@ The output always has a consistent shape: { "error": False, "query_summary": "...", # deterministic plain-English sentence - "steps": [...], # ordered list of operation steps - "schema": {...} # source collection, output columns, ordering, limit + "steps": [...], # ordered list of operation steps + "schema": {...}, # source collection, output columns, ordering, limit } # Failure (e.g. unrecognised term, expression instead of collection) @@ -162,7 +162,7 @@ The output always has a consistent shape: "error": True, "message": "Unrecognised term 'typo'. Did you mean: name?", "steps": [], - "schema": None + "schema": None, } ``` @@ -189,10 +189,12 @@ from pydough.metadata import parse_json_metadata_from_file graph = parse_json_metadata_from_file("path/to/metadata.json", "example_graph") + @pydough.init_pydough_context(graph) def my_query(): return nations.WHERE(region.name == "ASIA").CALCULATE(key, name) + node = my_query() # JSON dict — for pipeline machinery diff --git a/pydough/exploration/explain.py b/pydough/exploration/explain.py index c0d0cefb5..a29fd86be 100644 --- a/pydough/exploration/explain.py +++ b/pydough/exploration/explain.py @@ -288,7 +288,7 @@ def explain_unqualified( ) elif "Unrecognized term" in str(error): lines.append( - f"{str(error)}\n" + f"{error!s}\n" "This could mean you accessed a property using a name that does not exist, or\n" "that you need to place your PyDough code into a context for it to make sense.\n" "Did you mean to use pydough.explain_term?" diff --git a/pydough/exploration/term.py b/pydough/exploration/term.py index cf5570b03..e374b43d5 100644 --- a/pydough/exploration/term.py +++ b/pydough/exploration/term.py @@ -158,12 +158,12 @@ def explain_term( if "Unrecognized term" in str(e): lines.append( f"Invalid first argument to pydough.explain_term: {display_raw(node)}" - f" {str(e)}" + f" {e!s}" "This could mean you accessed a property using a name that does not exist, or\n" "that you need to place your PyDough code into a context for it to make sense." ) else: - raise e + raise if isinstance(qualified_node, PyDoughExpressionQDAG): lines.append( diff --git a/pydough/jupyter_extensions/README.md b/pydough/jupyter_extensions/README.md index 32611e69a..75ad7a58e 100644 --- a/pydough/jupyter_extensions/README.md +++ b/pydough/jupyter_extensions/README.md @@ -38,12 +38,13 @@ The transformed code will look like this: ```python from pydough.unqualified import UnqualifiedRoot + _ROOT = UnqualifiedRoot(pydough.active_session.metadata) result = _ROOT.Nations.CALCULATE( nation_name=_ROOT.name, region_name=_ROOT.region.name, - num_customers=_ROOT.COUNT(_ROOT.customers) + num_customers=_ROOT.COUNT(_ROOT.customers), ) print(pydough.to_df(result)) ``` diff --git a/pydough/metadata/templates/attribute_metadata.py b/pydough/metadata/templates/attribute_metadata.py index d4c95ee2a..4dfcfdd5f 100644 --- a/pydough/metadata/templates/attribute_metadata.py +++ b/pydough/metadata/templates/attribute_metadata.py @@ -111,7 +111,7 @@ def path(self) -> str: def create_error_name(name: str, graph_error_name: str): return f"Template attribute {name!r} in {graph_error_name}" - def add_attribute_option(self, label: str, value: str | int | float) -> None: + def add_attribute_option(self, label: str, value: str | float) -> None: """ Add an option to the list of options """ @@ -210,9 +210,7 @@ def parse_from_json( graph_labels: dict[str, str] = graph.get_all_labels() if label in graph_labels or label in new_attribute.options: - by_attribute: str = ( - graph_labels[label] if label in graph_labels else attribute_name - ) + by_attribute: str = graph_labels.get(label, attribute_name) raise PyDoughMetadataException( f"Duplicate option label: {label!r} for attribute {attribute_name!r}. " f"The label is already in use by attribute {by_attribute!r} " diff --git a/pydough/pydough_operators/type_inference/__init__.py b/pydough/pydough_operators/type_inference/__init__.py index 24672b96a..38d4d1374 100644 --- a/pydough/pydough_operators/type_inference/__init__.py +++ b/pydough/pydough_operators/type_inference/__init__.py @@ -9,7 +9,6 @@ "ConstantType", "ExpressionTypeDeducer", "RequireArgRange", - "RequireArgRange", "RequireCollection", "RequireMinArgs", "RequireNumArgs", diff --git a/pydough/qdag/collections/README.md b/pydough/qdag/collections/README.md index cb00a1f67..539d3c2a2 100644 --- a/pydough/qdag/collections/README.md +++ b/pydough/qdag/collections/README.md @@ -59,9 +59,7 @@ Below is an example of a PyDough snippet and the corresponding tree string repre ```python Nations.CALCULATE( nation_name=name, -).WHERE( - region.name == "EUROPE" -).suppliers.CALCULATE( +).WHERE(region.name == "EUROPE").suppliers.CALCULATE( supplier_name=name, nation_name=nation_name, ) @@ -84,12 +82,8 @@ And below is another such example: german_suppliers = supply_records.WHERE(supplier.nation == "GERMANY") selected_parts = parts.WHERE(HAS(german_suppliers)) PARTITION(selected_parts, name="p", by=size).CALCULATE( - size, - n_parts_with_german_supplier=COUNT(p) -).TOP_K( - 10, - by=n_parts_with_german_supplier.DESC() -) + size, n_parts_with_german_supplier=COUNT(p) +).TOP_K(10, by=n_parts_with_german_supplier.DESC()) ``` ``` diff --git a/pydough/relational/README.md b/pydough/relational/README.md index b7b5316ba..51d4a4ef1 100644 --- a/pydough/relational/README.md +++ b/pydough/relational/README.md @@ -117,14 +117,18 @@ from pydough.relational.relational_expressions import ( from pydough.types import NumericType, BooleanType # Create a scan node -scan_node = Scan("table_name", {"column_name": ColumnReference("column_name", NumericType())}) +scan_node = Scan( + "table_name", {"column_name": ColumnReference("column_name", NumericType())} +) # Create a filter node filter_condition = LiteralExpression(True, BooleanType()) filter_node = Filter(scan_node, filter_condition, scan_node.columns) # Create a project node -project_node = Project(filter_node, {"column_name": ColumnReference("column_name", NumericType())}) +project_node = Project( + filter_node, {"column_name": ColumnReference("column_name", NumericType())} +) # Create a limit node limit_expression = LiteralExpression(10, NumericType()) @@ -132,10 +136,14 @@ limit_node = Limit(project_node, limit_expression, project_node.columns) # Create a join node join_condition = LiteralExpression(True, BooleanType()) -join_node = Join([scan_node, filter_node], [join_condition], [JoinType.INNER], scan_node.columns) +join_node = Join( + [scan_node, filter_node], [join_condition], [JoinType.INNER], scan_node.columns +) # Create a relational root node -root_node = RelationalRoot(limit_node, [("column_name", ColumnReference("column_name", NumericType()))]) +root_node = RelationalRoot( + limit_node, [("column_name", ColumnReference("column_name", NumericType()))] +) # Convert the root node to a string root_node_str = root_node.to_string() diff --git a/pydough/relational/relational_expressions/abstract_expression.py b/pydough/relational/relational_expressions/abstract_expression.py index 3a002259d..89828f3b5 100644 --- a/pydough/relational/relational_expressions/abstract_expression.py +++ b/pydough/relational/relational_expressions/abstract_expression.py @@ -6,7 +6,6 @@ """ from abc import ABC, abstractmethod -from typing import Any __all__ = ["RelationalExpression"] @@ -68,7 +67,7 @@ def equals(self, other: "RelationalExpression") -> bool: and self.data_type == other.data_type ) - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: return self.equals(other) @abstractmethod diff --git a/pydough/relational/relational_expressions/literal_expression.py b/pydough/relational/relational_expressions/literal_expression.py index 599aa9938..ff5b4afe1 100644 --- a/pydough/relational/relational_expressions/literal_expression.py +++ b/pydough/relational/relational_expressions/literal_expression.py @@ -36,9 +36,9 @@ def value(self) -> object: def to_string(self, compact: bool = False) -> str: if compact: - return f"{repr(self.value)}:{self.data_type.json_string}" + return f"{self.value!r}:{self.data_type.json_string}" else: - return f"Literal(value={repr(self.value)}, type={self.data_type})" + return f"Literal(value={self.value!r}, type={self.data_type})" def equals(self, other: object) -> bool: return ( diff --git a/pydough/relational/relational_nodes/README.md b/pydough/relational/relational_nodes/README.md index dfddb65cc..9d8ca58fc 100644 --- a/pydough/relational/relational_nodes/README.md +++ b/pydough/relational/relational_nodes/README.md @@ -98,14 +98,18 @@ from pydough.relational.relational_expressions import ( from pydough.types import NumericType, BooleanType # Create a scan node -scan_node = Scan("table_name", {"column_name": ColumnReference("column_name", NumericType())}) +scan_node = Scan( + "table_name", {"column_name": ColumnReference("column_name", NumericType())} +) # Create a filter node filter_condition = LiteralExpression(True, BooleanType()) filter_node = Filter(scan_node, filter_condition, scan_node.columns) # Create a project node -project_node = Project(filter_node, {"column_name": ColumnReference("column_name", NumericType())}) +project_node = Project( + filter_node, {"column_name": ColumnReference("column_name", NumericType())} +) # Create a limit node limit_expression = LiteralExpression(10, NumericType()) @@ -113,10 +117,14 @@ limit_node = Limit(project_node, limit_expression, project_node.columns) # Create a join node join_condition = LiteralExpression(True, BooleanType()) -join_node = Join([scan_node, filter_node], [join_condition], [JoinType.INNER], scan_node.columns) +join_node = Join( + [scan_node, filter_node], [join_condition], [JoinType.INNER], scan_node.columns +) # Create a relational root node -root_node = RelationalRoot(limit_node, [("column_name", ColumnReference("column_name", NumericType()))]) +root_node = RelationalRoot( + limit_node, [("column_name", ColumnReference("column_name", NumericType()))] +) # Convert the root node to a string root_node_str = root_node.to_string() diff --git a/pydough/relational/relational_nodes/abstract_node.py b/pydough/relational/relational_nodes/abstract_node.py index c3e093f4e..d62e3c939 100644 --- a/pydough/relational/relational_nodes/abstract_node.py +++ b/pydough/relational/relational_nodes/abstract_node.py @@ -88,7 +88,7 @@ def equals(self, other: "RelationalNode") -> bool: """ return self.node_equals(other) and self.columns == other.columns - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: return isinstance(other, RelationalNode) and self.equals(other) def make_column_string(self, columns: dict[str, Any], compact: bool) -> str: diff --git a/pydough/sqlglot/execute_relational.py b/pydough/sqlglot/execute_relational.py index ee125e700..5e8a2c5bc 100644 --- a/pydough/sqlglot/execute_relational.py +++ b/pydough/sqlglot/execute_relational.py @@ -338,7 +338,7 @@ def replace_keys_with_indices( expr_idx = expressions.index(order_expr.this) else: expr_idx = aliases.index(order_expr.this.name.lower()) - order_list[idx].set( + order_expr.set( "this", sqlglot_expressions.convert(expr_idx + 1), ) @@ -356,7 +356,7 @@ def replace_keys_with_indices( expr_idx = aliases.index(collate.this.this.name.lower()) # Remove the COLLATE from the order expression, but change # the original expression to include the collate. - order_list[idx].set( + order_expr.set( "this", sqlglot_expressions.convert(expr_idx + 1), ) @@ -498,11 +498,12 @@ def remove_table_aliases_conditional(expr: SQLGlotExpression) -> None: if isinstance(expr, Select) and expr.args.get("expressions") is not None: for i in range(len(expr.expressions)): cur_expr = expr.expressions[i] - if isinstance(cur_expr, Alias) and isinstance( - cur_expr.args.get("this"), Column + if ( + isinstance(cur_expr, Alias) + and isinstance(cur_expr.args.get("this"), Column) + and cur_expr.alias == cur_expr.this.name ): - if cur_expr.alias == cur_expr.this.name: - expr.expressions[i] = cur_expr.this + expr.expressions[i] = cur_expr.this # Recursively visit the AST. for arg in expr.args.values(): diff --git a/pydough/sqlglot/override_simplify.py b/pydough/sqlglot/override_simplify.py index 208124b18..8c7b4f9cd 100644 --- a/pydough/sqlglot/override_simplify.py +++ b/pydough/sqlglot/override_simplify.py @@ -191,12 +191,12 @@ def simplify_datetrunc(expression: exp.Expression, dialect: Dialect) -> exp.Expr return expression if isinstance(expression, exp.Binary): - l, r = expression.left, expression.right # noqa: E741 + l, r = expression.left, expression.right if not _is_datetrunc_predicate(l, r): return expression - l = t.cast(exp.DateTrunc, l) # noqa: E741 + l = t.cast(exp.DateTrunc, l) trunc_arg = l.this unit = l.unit.name.lower() date = extract_date(r) @@ -212,11 +212,11 @@ def simplify_datetrunc(expression: exp.Expression, dialect: Dialect) -> exp.Expr ) if isinstance(expression, exp.In): - l = expression.this # noqa: E741 + l = expression.this rs = expression.expressions if rs and all(_is_datetrunc_predicate(l, r) for r in rs): - l = t.cast(exp.DateTrunc, l) # noqa: E741 + l = t.cast(exp.DateTrunc, l) unit = l.unit.name.lower() ranges = [] diff --git a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py index a290bb4bf..61d09a545 100644 --- a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py @@ -562,7 +562,7 @@ def convert_get_part( ELSE REGEXP_SUBSTR( str, - '(.*?)(' || REGEXP_REPLACE(delim, '([][(){}.*+?^$|\#-])', '\\\1') || '|$)', + '(.*?)(' || REGEXP_REPLACE(delim, '([][(){}.*+?^$|\\#-])', '\\\1') || '|$)', 1, CASE WHEN diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index a18f41ac9..c09de79bb 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -204,11 +204,10 @@ def convert_datediff( types: list[PyDoughType], ) -> SQLGlotExpression: assert len(args) == 3 - if not isinstance(args[0], sqlglot_expressions.Literal): - raise ValueError( - f"Unsupported argument {args[0]} for DATEDIFF.It should be a string." - ) - elif not args[0].is_string: + if ( + not isinstance(args[0], sqlglot_expressions.Literal) + or not args[0].is_string + ): raise ValueError( f"Unsupported argument {args[0]} for DATEDIFF.It should be a string." ) diff --git a/pydough/sqlglot/transform_bindings/sqlite_transform_bindings.py b/pydough/sqlglot/transform_bindings/sqlite_transform_bindings.py index bbfe9dd8f..cca23bc8c 100644 --- a/pydough/sqlglot/transform_bindings/sqlite_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sqlite_transform_bindings.py @@ -97,11 +97,10 @@ def convert_datediff( types: list[PyDoughType], ) -> SQLGlotExpression: assert len(args) == 3 - if not isinstance(args[0], sqlglot_expressions.Literal): - raise PyDoughSQLException( - f"Unsupported argument {args[0]} for DATEDIFF.It should be a string." - ) - elif not args[0].is_string: + if ( + not isinstance(args[0], sqlglot_expressions.Literal) + or not args[0].is_string + ): raise PyDoughSQLException( f"Unsupported argument {args[0]} for DATEDIFF.It should be a string." ) diff --git a/pydough/unqualified/README.md b/pydough/unqualified/README.md index c2253f404..f7d91c31f 100644 --- a/pydough/unqualified/README.md +++ b/pydough/unqualified/README.md @@ -44,15 +44,15 @@ from pydough.metadata import parse_json_metadata_from_file # Define a graph metadata object by reading from a JSON file graph = parse_json_metadata_from_file("path/to/metadata.json", "example_graph") + # Define a function with the init_pydough_context decorator @init_pydough_context(graph) def example_function(): return Nations.CALCULATE( - nation_name=name, - region_name=region.name, - num_customers=COUNT(customers) + nation_name=name, region_name=region.name, num_customers=COUNT(customers) ) + # Transform the source code of the function source_code = """ def example_function(): @@ -68,6 +68,7 @@ transformed_ast = transform_code(source_code, graph_dict, known_names) # Display the transformed Python code import ast + print(ast.unparse(transformed_ast)) # Transform a Jupyter cell @@ -88,13 +89,15 @@ The transformed Python code for the function will look like this: ```python from pydough.unqualified import UnqualifiedRoot + _ROOT = UnqualifiedRoot(example_graph) + def example_function(): return _ROOT.Nations.CALCULATE( nation_name=_ROOT.name, region_name=_ROOT.region.name, - num_customers=_ROOT.COUNT(_ROOT.customers) + num_customers=_ROOT.COUNT(_ROOT.customers), ) ``` @@ -102,12 +105,13 @@ The transformed Python code for the Jupyter cell will look like this: ```python from pydough.unqualified import UnqualifiedRoot + _ROOT = UnqualifiedRoot(example_graph) result = _ROOT.Nations.CALCULATE( nation_name=_ROOT.name, region_name=_ROOT.region.name, - num_customers=_ROOT.COUNT(_ROOT.customers) + num_customers=_ROOT.COUNT(_ROOT.customers), ) ``` diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 5bd3209ca..b8f791716 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -563,7 +563,7 @@ def __init__(self, graph: GraphMetadata): ) def __getattribute__(self, name: str) -> Any: - graph, func_map, templates = super(UnqualifiedNode, self).__getattribute__( + _graph, func_map, templates = super(UnqualifiedNode, self).__getattribute__( "_parcel" ) if name in func_map: diff --git a/pydough/unqualified/unqualified_transform.py b/pydough/unqualified/unqualified_transform.py index bd4a4ce2a..efb2f60b1 100644 --- a/pydough/unqualified/unqualified_transform.py +++ b/pydough/unqualified/unqualified_transform.py @@ -391,7 +391,7 @@ def transform_code( """ assert len(graph_dict) == 1, "Expected exactly one key in the graph_dict" visitor: ast.NodeTransformer = AddRootVisitor( - list(graph_dict.keys())[0], known_names + next(iter(graph_dict.keys())), known_names ) source = source.lstrip("\n") n_strip = len(source) - len(source.lstrip()) @@ -549,7 +549,7 @@ def _execute_source( tree: ast.AST = ast.parse(source) except SyntaxError as e: raise ValueError( - f"Syntax error in source PyDough code:\n{source}\n{str(e)}" + f"Syntax error in source PyDough code:\n{source}\n{e!s}" ) from e assert isinstance(tree, ast.AST) new_tree: ast.AST = ast.fix_missing_locations(visitor.visit(tree)) @@ -560,7 +560,7 @@ def _execute_source( try: compile_ast = compile(transformed_code, filename="", mode="exec") except SyntaxError as e: - raise ValueError(f"Syntax error in transformed PyDough code:\n{str(e)}") from e + raise ValueError(f"Syntax error in transformed PyDough code:\n{e!s}") from e execution_context: dict[str, Any] = environment | { graph_name: metadata, "pydough": pydough, diff --git a/pydough/user_collections/README.md b/pydough/user_collections/README.md index c82bcc2c6..59c1b93b0 100644 --- a/pydough/user_collections/README.md +++ b/pydough/user_collections/README.md @@ -66,11 +66,7 @@ You can access user collections through `pydough` and call them with the require ```python import pydough -my_range = pydough.range_collection( - "simple_range", - "col1", - 1, 10, 2 - ) +my_range = pydough.range_collection("simple_range", "col1", 1, 10, 2) ``` Output: ``` diff --git a/pydough/user_collections/dataframe_collection.py b/pydough/user_collections/dataframe_collection.py index d37f17842..a9fcd8d9e 100644 --- a/pydough/user_collections/dataframe_collection.py +++ b/pydough/user_collections/dataframe_collection.py @@ -51,8 +51,10 @@ def __init__( name: str, dataframe: pd.DataFrame, unique_column_names: list[str | list[str]], - column_subset: list[str] = [], + column_subset: list[str] | None = None, ) -> None: + if column_subset is None: + column_subset = [] dataframe = self.filter_dataframe(dataframe, column_subset) super().__init__( name=name, diff --git a/pydough/user_collections/user_collection_apis.py b/pydough/user_collections/user_collection_apis.py index 64f62c378..d70a27643 100644 --- a/pydough/user_collections/user_collection_apis.py +++ b/pydough/user_collections/user_collection_apis.py @@ -58,7 +58,7 @@ def dataframe_collection( name: str, dataframe: pd.DataFrame, unique_column_names: list[str | list[str]], - column_subset: list[str] = [], + column_subset: list[str] | None = None, ) -> UnqualifiedGeneratedCollection: """ Implementation of the `pydough.dataframe_collection` function, which provides @@ -77,6 +77,8 @@ def dataframe_collection( A collection with the given dataframe. """ # Validate name + if column_subset is None: + column_subset = [] if not isinstance(name, str): raise TypeError(f"Expected 'name' to be a string, got {type(name).__name__}") diff --git a/pydough/user_collections/user_collections.py b/pydough/user_collections/user_collections.py index 6c862d029..6c171dd8d 100644 --- a/pydough/user_collections/user_collections.py +++ b/pydough/user_collections/user_collections.py @@ -98,8 +98,10 @@ def to_explanation(self, verbose: bool) -> list[str]: A list of human-readable explanation strings. """ lines = [ - f"This node accesses user-generated collection {self.name!r}.\n" - f"Columns: {', '.join(sorted(self.columns))}", + ( + f"This node accesses user-generated collection {self.name!r}.\n" + f"Columns: {', '.join(sorted(self.columns))}" + ), ] if verbose: unique = self.unique_column_names[0] diff --git a/tests/gen_data/databricks_task.py b/tests/gen_data/databricks_task.py index 17815a0e7..610f7d2d1 100644 --- a/tests/gen_data/databricks_task.py +++ b/tests/gen_data/databricks_task.py @@ -668,16 +668,18 @@ def main(force: bool = False) -> None: from databricks import sql as dbsql - with dbsql.connect( - server_hostname=host, - http_path=http_path, - access_token=token, - ) as conn: - with conn.cursor() as cursor: - if init_only: - setup_tables(cursor) - else: - run_with_cursor(cursor, force=force) + with ( + dbsql.connect( + server_hostname=host, + http_path=http_path, + access_token=token, + ) as conn, + conn.cursor() as cursor, + ): + if init_only: + setup_tables(cursor) + else: + run_with_cursor(cursor, force=force) if __name__ == "__main__": diff --git a/tests/test_documentation.py b/tests/test_documentation.py index 6265f4c47..4327ba8a3 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -28,7 +28,7 @@ def test_function_list(): # Identify every section header in the function documentation headers: set[str] = set() with open("documentation/functions.md") as f: - for line in f.readlines(): + for line in f: if line.startswith("#"): headers.add(line.strip("#").strip()) # Remove any function name that is in the headers, and fail if there are diff --git a/tests/test_logging.py b/tests/test_logging.py index 61bb87b59..88b802465 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -95,7 +95,7 @@ def test_get_logger_propagation(monkeypatch): buf = StringIO() custom_handler = logging.StreamHandler(buf) # Create the pydough logger with the custom handler (default name is pydough) - logger1 = get_logger(default_level=logging.WARN, handlers=[custom_handler]) + logger1 = get_logger(default_level=logging.WARNING, handlers=[custom_handler]) # Avoid interference from root handlers logger1.propagate = False # Create new logger that will propagate messages to logger1 handlers diff --git a/tests/test_masked_sf.py b/tests/test_masked_sf.py index 9dd875784..6dc563009 100644 --- a/tests/test_masked_sf.py +++ b/tests/test_masked_sf.py @@ -975,7 +975,7 @@ "selected_patients = patients.WHERE(" + " | ".join( [ - f"ISIN(last_name, {repr(names)})" + f"ISIN(last_name, {names!r})" for names in [ ["Bailey", "Baker", "Ball", "Banks", "Barajas", "Barnett"], ["Barrera", "Barron", "Barton", "Cabrera", "Calderon"], @@ -1436,28 +1436,30 @@ def test_pipeline_hard_limit_variations( cases with the PYDOUGH_MASK_SERVER_HARD_LIMIT with fewer other parameters. """ sf_data = sf_masked_context("BODO", test_data.graph_name, "FULL") - with temp_env_override( - { - "PYDOUGH_MASK_SERVER_HARD_LIMIT": str(hard_limit), - "PYDOUGH_ENABLE_MASK_REWRITES": "1", - } + with ( + temp_env_override( + { + "PYDOUGH_MASK_SERVER_HARD_LIMIT": str(hard_limit), + "PYDOUGH_ENABLE_MASK_REWRITES": "1", + } + ), + redirect_stdout(io.StringIO()), ): - with redirect_stdout(io.StringIO()): - if execute: - test_data.run_e2e_test( - get_sf_masked_graphs, - sf_masked_context("BODO", test_data.graph_name, "FULL"), - coerce_types=True, - mask_server=true_mask_server_info, - ) - else: - file_path: str = get_sql_test_filename( - f"{test_data.test_name}_rewrite", sf_data.dialect - ) - test_data.run_sql_test( - get_sf_masked_graphs, - file_path, - update_tests, - sf_data, - mask_server=true_mask_server_info, - ) + if execute: + test_data.run_e2e_test( + get_sf_masked_graphs, + sf_masked_context("BODO", test_data.graph_name, "FULL"), + coerce_types=True, + mask_server=true_mask_server_info, + ) + else: + file_path: str = get_sql_test_filename( + f"{test_data.test_name}_rewrite", sf_data.dialect + ) + test_data.run_sql_test( + get_sf_masked_graphs, + file_path, + update_tests, + sf_data, + mask_server=true_mask_server_info, + ) diff --git a/tests/test_masked_sqlite.py b/tests/test_masked_sqlite.py index 0dea9ab83..e4be86332 100644 --- a/tests/test_masked_sqlite.py +++ b/tests/test_masked_sqlite.py @@ -390,14 +390,7 @@ pytest.param( PyDoughPandasTest( "selected_accounts = accounts.WHERE(" - + " & ".join( - [ - "((account_type == 'retirement') | (account_type == 'savings'))", - "(balance >= 5000)", - "(CONTAINS(account_holder.email, 'outlook') | CONTAINS(account_holder.email, 'gmail'))", - "(YEAR(creation_timestamp) < 2020)", - ] - ) + + "((account_type == 'retirement') | (account_type == 'savings')) & (balance >= 5000) & (CONTAINS(account_holder.email, 'outlook') | CONTAINS(account_holder.email, 'gmail')) & (YEAR(creation_timestamp) < 2020)" + ")\n" "result = CRYPTBANK.CALCULATE(n=COUNT(selected_accounts))", "CRYPTBANK", @@ -1203,14 +1196,7 @@ def test_pipeline_e2e_cryptbank( ), pytest.param( "selected_accounts = accounts.WHERE(" - + " & ".join( - [ - "((account_type == 'retirement') | (account_type == 'savings'))", - "(balance >= 5000)", - "(CONTAINS(account_holder.email, 'outlook') | CONTAINS(account_holder.email, 'gmail'))", - "(YEAR(creation_timestamp) < 2020)", - ] - ) + + "((account_type == 'retirement') | (account_type == 'savings')) & (balance >= 5000) & (CONTAINS(account_holder.email, 'outlook') | CONTAINS(account_holder.email, 'gmail')) & (YEAR(creation_timestamp) < 2020)" + ")\n" "result = CRYPTBANK.CALCULATE(n=COUNT(selected_accounts))", [ @@ -1505,16 +1491,26 @@ def test_pipeline_e2e_cryptbank( + ")", [ { - "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'a', " - "'CONTAINS', 2, '__col__', 'e']", - "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'a', " - "'CONTAINS', 2, '__col__', 'u']", - "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'e', " - "'CONTAINS', 2, '__col__', 'i']", - "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'i', " - "'CONTAINS', 2, '__col__', 'o']", - "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'o', " - "'CONTAINS', 2, '__col__', 'u']", + ( + "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'a', " + "'CONTAINS', 2, '__col__', 'e']" + ), + ( + "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'a', " + "'CONTAINS', 2, '__col__', 'u']" + ), + ( + "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'e', " + "'CONTAINS', 2, '__col__', 'i']" + ), + ( + "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'i', " + "'CONTAINS', 2, '__col__', 'o']" + ), + ( + "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'o', " + "'CONTAINS', 2, '__col__', 'u']" + ), "CRBNK/CUSTOMERS/c_fname: ['CONTAINS', 2, '__col__', 'a']", "CRBNK/CUSTOMERS/c_fname: ['CONTAINS', 2, '__col__', 'e']", "CRBNK/CUSTOMERS/c_fname: ['CONTAINS', 2, '__col__', 'i']", @@ -1638,7 +1634,6 @@ def test_pipeline_e2e_cryptbank( "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'a', 'CONTAINS', 2, '__col__', 'e']", "CRBNK/CUSTOMERS/c_fname: ['AND', 2, 'CONTAINS', 2, '__col__', 'o', 'CONTAINS', 2, '__col__', 'u']", "CRBNK/CUSTOMERS/c_fname: ['AND', 3, 'CONTAINS', 2, '__col__', 'a', 'CONTAINS', 2, '__col__', 'e', 'CONTAINS', 2, '__col__', 'i']", - "CRBNK/CUSTOMERS/c_fname: ['AND', 3, 'CONTAINS', 2, '__col__', 'a', 'CONTAINS', 2, '__col__', 'e', 'CONTAINS', 2, '__col__', 'i']", "CRBNK/CUSTOMERS/c_fname: ['AND', 3, 'CONTAINS', 2, '__col__', 'e', 'CONTAINS', 2, '__col__', 'i', 'CONTAINS', 2, '__col__', 'o']", "CRBNK/CUSTOMERS/c_fname: ['CONTAINS', 2, '__col__', 'e']", "CRBNK/CUSTOMERS/c_fname: ['CONTAINS', 2, '__col__', 'i']", diff --git a/tests/test_pipeline_oracle.py b/tests/test_pipeline_oracle.py index 285086da6..f403b4506 100644 --- a/tests/test_pipeline_oracle.py +++ b/tests/test_pipeline_oracle.py @@ -452,7 +452,7 @@ def defog_custom_oracle_test_data( @pytest.fixture def custom_functions_oracle_test_data( - custom_functions_test_data: PyDoughPandasTest, # noqa: F811 + custom_functions_test_data: PyDoughPandasTest, ) -> PyDoughPandasTest: """ Modify reference solution data for some custom queries. @@ -509,7 +509,7 @@ def test_pipeline_e2e_oracle_custom_functions( @pytest.mark.oracle @pytest.mark.execute def test_pipeline_e2e_oracle_defog_custom( - defog_custom_oracle_test_data: PyDoughPandasTest, # noqa: F811 + defog_custom_oracle_test_data: PyDoughPandasTest, get_oracle_defog_graphs: graph_fetcher, defog_config: PyDoughConfigs, oracle_conn_db_context: Callable[[str], DatabaseContext], diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index c2b4f9b07..c1b330439 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -8574,9 +8574,7 @@ def test_pipeline_to_table_ddl( # that reference attached databases. # SQLite the only one that supports temporary views. # So the view will be created as TEMPORARY. - if db_context.dialect == DatabaseDialect.SQLITE and as_view and not temp: - expected_create_statement += " TEMPORARY" - elif temp: + if db_context.dialect == DatabaseDialect.SQLITE and as_view and not temp or temp: expected_create_statement += " TEMPORARY" expected_create_statement += table_or_view diff --git a/tests/test_qdag_collection.py b/tests/test_qdag_collection.py index 8e38f601b..439477536 100644 --- a/tests/test_qdag_collection.py +++ b/tests/test_qdag_collection.py @@ -392,7 +392,6 @@ def region_intra_pct() -> tuple[CollectionTestInfo, str, str]: "nation", "lines", "supply_records", - "lines", }, id="parts_suppliers", ), @@ -427,7 +426,6 @@ def region_intra_pct() -> tuple[CollectionTestInfo, str, str]: "comment", "lines", "supply_records", - "lines", "nation", "good_comment", "negative_balance", diff --git a/tests/test_types.py b/tests/test_types.py index 2f33d74cf..d1b1657ad 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -5,7 +5,7 @@ import pytest # Used so eval() has access to all the type classes -from pydough.types import * # noqa +from pydough.types import * from pydough.types import parse_type_from_string diff --git a/tests/test_unqualified_node.py b/tests/test_unqualified_node.py index 8924d7bba..df9834b8d 100644 --- a/tests/test_unqualified_node.py +++ b/tests/test_unqualified_node.py @@ -186,11 +186,6 @@ def verify_pydough_code_exec_match_unqualified( "((1.5 * x) - 1)", id="arithmetic_03", ), - pytest.param( - "answer = ((1.5 * _ROOT.x) - 1)", - "((1.5 * x) - 1)", - id="arithmetic_03", - ), pytest.param( "answer = (_ROOT.STARTSWITH(_ROOT.x, 'hello') | _ROOT.ENDSWITH(_ROOT.x, 'world')) & _ROOT.CONTAINS(_ROOT.x, ' ')", "((STARTSWITH(x, 'hello') | ENDSWITH(x, 'world')) & CONTAINS(x, ' '))", From 8d17de7d50433ed554c5a359db57cdb168b1f3df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:21:02 +0000 Subject: [PATCH 12/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../relational_expressions/abstract_expression.py | 3 +-- pydough/unqualified/qualification.py | 2 +- tests/test_relational_nodes_to_sqlglot.py | 7 +------ 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/pydough/relational/relational_expressions/abstract_expression.py b/pydough/relational/relational_expressions/abstract_expression.py index 3a002259d..89828f3b5 100644 --- a/pydough/relational/relational_expressions/abstract_expression.py +++ b/pydough/relational/relational_expressions/abstract_expression.py @@ -6,7 +6,6 @@ """ from abc import ABC, abstractmethod -from typing import Any __all__ = ["RelationalExpression"] @@ -68,7 +67,7 @@ def equals(self, other: "RelationalExpression") -> bool: and self.data_type == other.data_type ) - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: return self.equals(other) @abstractmethod diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index 5cef3fa70..e4c27709c 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -684,7 +684,7 @@ def names_defined_in_child_chain( and not isinstance(node, GlobalContext) ): if isinstance(node, Calculate): - defined.update(name for name in node.calc_term_values.keys()) + defined.update(name for name in node.calc_term_values) # Operators (CALCULATE, WHERE, SINGULAR, ...) step back through # their preceding context; child accesses step up to their parent. if node.preceding_context is not None: diff --git a/tests/test_relational_nodes_to_sqlglot.py b/tests/test_relational_nodes_to_sqlglot.py index fc8f47259..b2c72fb00 100644 --- a/tests/test_relational_nodes_to_sqlglot.py +++ b/tests/test_relational_nodes_to_sqlglot.py @@ -135,12 +135,7 @@ def mkglot(expressions: list[Expression], _from: Expression, **kwargs) -> Select assert from_alias is None, "Alias is only allowed for subqueries" from_result = from_value from_result = From(this=from_result) - query: Select = Select( - **{ - "expressions": expressions, - "from_": from_result, - } - ) + query: Select = Select(expressions=expressions, from_=from_result) if "where" in kwargs: query = query.where(kwargs.pop("where")) if "group_by" in kwargs: From 4adcebde4aadd396f03c5b626edbe592e592bb47 Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 16 Oct 2026 10:23:29 -0700 Subject: [PATCH 13/17] [run all] From 114a5710e879a63dd693e40a58587272b630eb2a Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 16 Oct 2026 10:23:49 -0700 Subject: [PATCH 14/17] [run all] From 2e1643c1ed4b80fb15074a4e56d577c24265ae72 Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 16 Oct 2026 10:47:03 -0700 Subject: [PATCH 15/17] [run all] fix failures --- .../abstract_expression.py | 2 +- pydough/sqlglot/override_simplify.py | 181 ++---------------- pyproject.toml | 9 + tests/test_types.py | 2 +- 4 files changed, 28 insertions(+), 166 deletions(-) diff --git a/pydough/relational/relational_expressions/abstract_expression.py b/pydough/relational/relational_expressions/abstract_expression.py index 89828f3b5..bde4c70cb 100644 --- a/pydough/relational/relational_expressions/abstract_expression.py +++ b/pydough/relational/relational_expressions/abstract_expression.py @@ -68,7 +68,7 @@ def equals(self, other: "RelationalExpression") -> bool: ) def __eq__(self, other: object) -> bool: - return self.equals(other) + return isinstance(other, RelationalExpression) and self.equals(other) @abstractmethod def to_string(self, compact: bool = False) -> str: diff --git a/pydough/sqlglot/override_simplify.py b/pydough/sqlglot/override_simplify.py index cc8754f12..cee7f69a6 100644 --- a/pydough/sqlglot/override_simplify.py +++ b/pydough/sqlglot/override_simplify.py @@ -283,171 +283,24 @@ def simplify( """ simplifier = PyDoughSimplifier(dialect=dialect) - dialect = Dialect.get_or_raise(dialect) - - def _simplify(expression, root=True): - if ( - max_depth - and isinstance(expression, exp.Connector) - and not isinstance(expression.parent, exp.Connector) - ): - depth = connector_depth(expression) - if depth > max_depth: - logger.info( - f"Skipping simplification because connector depth {depth} exceeds max {max_depth}" - ) - return expression - - if expression.meta.get(FINAL): - return expression - - # group by expressions cannot be simplified, for example - # select x + 1 + 1 FROM y GROUP BY x + 1 + 1 - # the projection must exactly match the group by key - group = expression.args.get("group") - - if group and hasattr(expression, "selects"): - groups = set(group.expressions) - group.meta[FINAL] = True - - for e in expression.selects: - for node in e.walk(): - if node in groups: - e.meta[FINAL] = True - break - - having = expression.args.get("having") - if having: - for node in having.walk(): - if node in groups: - having.meta[FINAL] = True - break - - # Pre-order transformations - node = expression - node = rewrite_between(node) - node = uniq_sort(node, root) - node = absorb_and_eliminate(node, root) - node = simplify_concat(node) - node = simplify_conditionals(node) - - # PyDough Change: new pre-order transformations - node = rewrite_case_to_nullif(node) - node = rewrite_coalesce_nullif(node) - node = rewrite_sum_nullif(node) - node = rewrite_coalesce_count(node) - node = rewrite_length_literal(node) - node = rewrite_division(node) - - if constant_propagation: - node = propagate_constants(node, root) - - exp.replace_children(node, lambda e: _simplify(e, False)) - - # Post-order transformations - node = simplify_not(node) - node = flatten(node) - node = simplify_connectors(node, root) - node = remove_complements(node, root) - node = simplify_coalesce(node) - node.parent = expression.parent - node = simplify_literals(node, root) - node = simplify_equality(node) - node = simplify_parens(node) - node = simplify_datetrunc(node, dialect) - node = sort_comparison(node) - node = simplify_startswith(node) - - # PyDough Change: new post-order transformations - node = rewrite_nullif_coalesce(node) - # Length is rewritten both before and after since LENGTH(literal) can - # be generated by the PyDough->SQLGlot generation, so doing it in - # pre-order may enable other optimizations. However, other rules may - # reduce terms into string literals, thus generating more LENGTH calls - # on a literal. - node = rewrite_length_literal(node) - - if root: - expression.replace(node) - return node - - expression = while_changing(expression, _simplify) - remove_where_true(expression) - return expression - - -@catch(ModuleNotFoundError, UnsupportedUnit) -def simplify_datetrunc(expression: exp.Expression, dialect: Dialect) -> exp.Expression: - """Simplify expressions like `DATE_TRUNC('year', x) >= CAST('2021-01-01' AS DATE)`""" - comparison = expression.__class__ - - if isinstance(expression, DATETRUNCS): - this = expression.this - trunc_type = extract_type(this) - date = extract_date(this) - #### Start of PyDough Change #### - # If date is datetime.datetime, it should NOT enter the if statement - # because `date_floor` only works correctly on datetime.date - if date and not isinstance(date, datetime.datetime) and expression.unit: - return date_literal( - date_floor(date, expression.unit.name.lower(), dialect), trunc_type - ) - #### End of PyDough Change #### - elif comparison not in DATETRUNC_COMPARISONS: - return expression - - if isinstance(expression, exp.Binary): - l, r = expression.left, expression.right - - if not _is_datetrunc_predicate(l, r): - return expression - - l = t.cast(exp.DateTrunc, l) - trunc_arg = l.this - unit = l.unit.name.lower() - date = extract_date(r) - - if not date: - return expression - - return ( - DATETRUNC_BINARY_COMPARISONS[comparison]( - trunc_arg, date, unit, dialect, extract_type(r) - ) - or expression + #### Start of PyDough Change #### + # `simplify()` can converge one iteration too early for expressions + # reached via our `_simplify` patch above, leaving behind invalid + # intermediate SQL (e.g. a `COALESCE(x)` missing its default argument). + # Re-running cleans this up, and is a no-op once truly converged. + previous_sql: str | None = None + for _ in range(5): + expression = simplifier.simplify( + expression, + constant_propagation=constant_propagation, + # PyDough always ran the coalesce-simplification rule (it wasn't + # optional prior to sqlglot introducing this flag). + coalesce_simplification=True, ) - - if isinstance(expression, exp.In): - l = expression.this - rs = expression.expressions - - if rs and all(_is_datetrunc_predicate(l, r) for r in rs): - l = t.cast(exp.DateTrunc, l) - unit = l.unit.name.lower() - - ranges = [] - for r in rs: - date = extract_date(r) - if not date: - return expression - drange = _datetrunc_range(date, unit, dialect) - if drange: - ranges.append(drange) - - if not ranges: - return expression - - ranges = merge_ranges(ranges) - target_type = extract_type(*rs) - - return exp.or_( - *[ - _datetrunc_eq_expression(l, drange, target_type) - for drange in ranges - ], - copy=False, - ) - + current_sql = expression.sql(dialect=dialect) + if current_sql == previous_sql: + break + previous_sql = current_sql return expression #### End of PyDough Change #### diff --git a/pyproject.toml b/pyproject.toml index 33319d5cf..344f70f6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,15 @@ source = "vcs" [tool.ruff] unsafe-fixes = true +# Ruff 0.16 expanded its default rule set from 59 to 413 rules. Pin `select` +# back to the pre-0.16 defaults so `extend-select` below continues to extend +# the same small baseline instead of the much larger new default. +lint.select = [ + "E4", + "E7", + "E9", + "F", +] lint.extend-select = [ "I", # isort "UP", # pyupgrade diff --git a/tests/test_types.py b/tests/test_types.py index d1b1657ad..06889f4dc 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -5,7 +5,7 @@ import pytest # Used so eval() has access to all the type classes -from pydough.types import * +from pydough.types import * # noqa: F403 from pydough.types import parse_type_from_string From 85aa3a760c2f2bd3df92ad5fa123fe3c9746aed3 Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 16 Oct 2026 10:54:55 -0700 Subject: [PATCH 16/17] fix astype --- tests/test_pipeline_tpch_custom.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 64f2b4a6f..179fb0803 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4403,16 +4403,16 @@ { "idx": [1, 2, 3, 4], "arr_d": [ - [datetime(2020, 1, 1, 0, 0, 0, tzinfo=UTC)], + [datetime(2020, 1, 1, 0, 0, 0)], [], [ - datetime(2021, 6, 15, 0, 0, 0, tzinfo=UTC), - datetime(2022, 12, 31, 0, 0, 0, tzinfo=UTC), + datetime(2021, 6, 15, 0, 0, 0), + datetime(2022, 12, 31, 0, 0, 0), ], [ - datetime(1999, 7, 4, 0, 0, 0, tzinfo=UTC), + datetime(1999, 7, 4, 0, 0, 0), None, - datetime(2000, 1, 1, 0, 0, 0, tzinfo=UTC), + datetime(2000, 1, 1, 0, 0, 0), ], ], "arr_t": [ From 3d9d9fcc7aeeab3a27c2011e9cd43a3caf5a938c Mon Sep 17 00:00:00 2001 From: Hadia Ahmed Date: Fri, 16 Oct 2026 10:57:52 -0700 Subject: [PATCH 17/17] [run all] --- tests/test_pipeline_tpch_custom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 179fb0803..36d4066ba 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -8,7 +8,7 @@ import re import sys from collections.abc import Callable -from datetime import UTC, date, datetime +from datetime import date, datetime from decimal import Decimal import numpy as np @@ -6744,7 +6744,7 @@ "month_name_col": ["May", "Jul", "Feb", "Feb", "Jan"], "month_col_added": ["Aug", "Oct", "May", "May", "Apr"], "month_str_subs": ["May", "May", "May", "May", "May"], - "month_now": [datetime.now(tz=UTC).strftime("%b")] * 5, + "month_now": [datetime.now().strftime("%b")] * 5, } ), "monthname_function_1",