From 80a98a644a9a302c82fe7b402a877b11aee26616 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Tue, 29 Sep 2026 17:40:53 -0600 Subject: [PATCH 01/14] adding bug test --- tests/test_pipeline_tpch_custom.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 32f6f0ec8..cdf8a4942 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -6737,6 +6737,20 @@ ), id="monthname_function_1", ), + pytest.param( + PyDoughPandasTest( + "# 'top' calculates 'cust_name' on customers\n" + "top = customers.CALCULATE(cust_name=name)\n" + "# Referencing 'customer.cust_name' from orders causes a lowering error\n" + "result = top.orders.CALCULATE(\n" + " x=customer.cust_name\n" + ").PARTITION(name='g', by=x)\n", + "TPCH", + lambda: pd.DataFrame({}), + "parent_calc_access_error", + ), + id="parent_calc_access_error", + ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: From 0443d8fc03d043dba8241a7aae0a10791cb2fa04 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Wed, 30 Sep 2026 14:30:52 -0600 Subject: [PATCH 02/14] wip: preparing solution --- pydough/conversion/relational_converter.py | 1 + .../child_operator_child_access.py | 4 ++ pydough/qdag/collections/collection_access.py | 1 + pydough/qdag/collections/sub_collection.py | 5 +++ pydough/unqualified/qualification.py | 1 + tests/test_masked_sqlite.py | 2 + tests/test_pipeline_tpch_custom.py | 37 ++++++++++++------- .../simple_pydough_functions.py | 12 ++++++ 8 files changed, 49 insertions(+), 14 deletions(-) diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index 263f634c8..9546cb5cd 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -1918,6 +1918,7 @@ def convert_ast_to_relational( of `node` are included in the root in the correct order, and if it has an ordering then the relational root stores that information. """ + # breakpoint() # Pre-process the QDAG node so the final CALCULATE includes any ordering # keys. rel_translator: RelTranslation = RelTranslation(session) diff --git a/pydough/qdag/collections/child_operator_child_access.py b/pydough/qdag/collections/child_operator_child_access.py index 64abf77cd..dedb06477 100644 --- a/pydough/qdag/collections/child_operator_child_access.py +++ b/pydough/qdag/collections/child_operator_child_access.py @@ -70,6 +70,10 @@ def get_expression_position(self, expr_name: str) -> int: return self.child_access.get_expression_position(expr_name) def get_term(self, term_name: str) -> PyDoughQDAG: + breakpoint() + # TODO: Trying some verification to make sure that + # the term is access correctly when is inherited and when is through a + # join cannot be accessed like customer.inherited_term term = self.child_access.get_term(term_name) if isinstance(term, ChildAccess): term = term.clone_with_parent(self) diff --git a/pydough/qdag/collections/collection_access.py b/pydough/qdag/collections/collection_access.py index d6d78b529..f9185f5d8 100644 --- a/pydough/qdag/collections/collection_access.py +++ b/pydough/qdag/collections/collection_access.py @@ -105,6 +105,7 @@ def get_expression_position(self, expr_name: str) -> int: return self._calc_property_order[expr_name] def get_term(self, term_name: str) -> PyDoughQDAG: + # breakpoint() self.verify_term_exists(term_name) # Special handling of terms down-streamed from an ancestor CALCULATE diff --git a/pydough/qdag/collections/sub_collection.py b/pydough/qdag/collections/sub_collection.py index 4a0a4b0b0..bdf962d17 100644 --- a/pydough/qdag/collections/sub_collection.py +++ b/pydough/qdag/collections/sub_collection.py @@ -94,6 +94,11 @@ def standalone_string(self) -> str: def tree_item_string(self) -> str: return f"SubCollection[{self.standalone_string}]" + @property + def all_terms(self) -> set[str]: + breakpoint() + return self._all_property_names - self._ancestral_mapping.keys() + def equals(self, other: object) -> bool: return ( isinstance(other, SubCollection) diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index 87cf2017f..6940a0d72 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -612,6 +612,7 @@ def qualify_access( # child is just a de-sugared invocation of the global context. return qualified_parent else: + # breakpoint() # Identify whether the access is an expression or a collection term = qualified_parent.get_term(name) if isinstance(term, PyDoughCollectionQDAG): diff --git a/tests/test_masked_sqlite.py b/tests/test_masked_sqlite.py index 0dea9ab83..051d0e458 100644 --- a/tests/test_masked_sqlite.py +++ b/tests/test_masked_sqlite.py @@ -23,6 +23,8 @@ transform_and_exec_pydough, ) +pytestmark = pytest.mark.skip(reason="Getting stuck") + @pytest.fixture( params=[ diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 006be90b8..59d0178e9 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -124,6 +124,7 @@ order_info_per_priority, order_quarter_test, orders_versus_first_orders, + parent_calc_access_error, part_reduced_size, partition_by_child_reference, partition_by_literal, @@ -6789,20 +6790,6 @@ ), id="compound_ref_correlation_extraction", ), - pytest.param( - PyDoughPandasTest( - "# 'top' calculates 'cust_name' on customers\n" - "top = customers.CALCULATE(cust_name=name)\n" - "# Referencing 'customer.cust_name' from orders causes a lowering error\n" - "result = top.orders.CALCULATE(\n" - " x=customer.cust_name\n" - ").PARTITION(name='g', by=x)\n", - "TPCH", - lambda: pd.DataFrame({}), - "parent_calc_access_error", - ), - id="parent_calc_access_error", - ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: @@ -7594,6 +7581,28 @@ def test_pipeline_e2e_simple_week( ), id="partition_by_child_reference", ), + pytest.param( + parent_calc_access_error, + None, + re.escape( + "Unrecognized term of " + "TPCH.customers.CALCULATE(cust_name=name).orders.customer: " + "'cust_name'. Did you mean: name, phone, key?" + ), + id="parent_calc_access_error", + ), + # pytest.param( + # PyDoughPandasTest( + # "main_context = orders.CALCULATE(or_clerk=clerk)\n" + # "result = main_context.lines.CALCULATE(\n" + # " x=order.or_clerk\n" + # ")", + # "TPCH", + # lambda: pd.DataFrame({}), + # "parent_calc_access_error_2", + # ), + # id="parent_calc_access_error_2", + # ), ], ) def test_pipeline_e2e_errors( diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 15d8be491..9a917567a 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -3896,3 +3896,15 @@ def partition_by_child_reference(): return orders.PARTITION(name="by_cust_key", by=customer.key).CALCULATE( total=COUNT(orders) ) + + +def parent_calc_access_error(): + """ + TODO + """ + # 'top' calculates 'cust_name' on customers + top = customers.CALCULATE(cust_name=name) + + # Referencing 'customer.cust_name' from orders causes a lowering error + result = top.orders.CALCULATE(x=customer.cust_name) + return result From d64a79dd9e01dd31cd0bb91a5279e7f3c563857b Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Thu, 1 Oct 2026 08:35:24 -0600 Subject: [PATCH 03/14] adding extra error handling for mock server setup --- tests/conftest.py | 9 ++++++++- tests/test_masked_sqlite.py | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e5be60b9b..6213eccce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3208,6 +3208,12 @@ def mock_server_setup(): # Wait until server is ready for _ in range(20): + if proc.poll() is not None: + output, _ = proc.communicate() + raise RuntimeError( + f"Mock server exited with code {proc.returncode}:\n{output}" + ) + try: r: httpx.Response = httpx.get(server_url + "/health", timeout=1) if r.status_code == 200: @@ -3216,7 +3222,8 @@ def mock_server_setup(): time.sleep(0.5) else: proc.terminate() - raise RuntimeError("Mock server failed to start") + output, _ = proc.communicate() + raise RuntimeError(f"Mock server failed to start:\n{output}") yield server_url diff --git a/tests/test_masked_sqlite.py b/tests/test_masked_sqlite.py index 051d0e458..0dea9ab83 100644 --- a/tests/test_masked_sqlite.py +++ b/tests/test_masked_sqlite.py @@ -23,8 +23,6 @@ transform_and_exec_pydough, ) -pytestmark = pytest.mark.skip(reason="Getting stuck") - @pytest.fixture( params=[ From 6ee0ac68b855866b4c7115205436b7b98cecba38 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 10:18:08 -0600 Subject: [PATCH 04/14] adding fix for accessed term incorrectly through a child collection [run ci][run all] --- .../child_operator_child_access.py | 4 - pydough/qdag/collections/collection_access.py | 1 - pydough/qdag/collections/sub_collection.py | 5 -- pydough/unqualified/qualification.py | 86 ++++++++++++++++++- tests/test_pipeline_tpch_custom.py | 36 ++++---- .../shadowed_calc_term_exe.txt | 7 ++ .../simple_pydough_functions.py | 26 +++++- .../shadowed_calc_term_exe_ansi.sql | 19 ++++ .../shadowed_calc_term_exe_databricks.sql | 19 ++++ .../shadowed_calc_term_exe_duckdb.sql | 19 ++++ .../shadowed_calc_term_exe_mysql.sql | 19 ++++ .../shadowed_calc_term_exe_oracle.sql | 19 ++++ .../shadowed_calc_term_exe_postgres.sql | 19 ++++ .../shadowed_calc_term_exe_snowflake.sql | 19 ++++ .../shadowed_calc_term_exe_sqlite.sql | 19 ++++ .../shadowed_calc_term_exe_trino.sql | 19 ++++ 16 files changed, 308 insertions(+), 28 deletions(-) create mode 100644 tests/test_plan_refsols/shadowed_calc_term_exe.txt create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql create mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql diff --git a/pydough/qdag/collections/child_operator_child_access.py b/pydough/qdag/collections/child_operator_child_access.py index dedb06477..64abf77cd 100644 --- a/pydough/qdag/collections/child_operator_child_access.py +++ b/pydough/qdag/collections/child_operator_child_access.py @@ -70,10 +70,6 @@ def get_expression_position(self, expr_name: str) -> int: return self.child_access.get_expression_position(expr_name) def get_term(self, term_name: str) -> PyDoughQDAG: - breakpoint() - # TODO: Trying some verification to make sure that - # the term is access correctly when is inherited and when is through a - # join cannot be accessed like customer.inherited_term term = self.child_access.get_term(term_name) if isinstance(term, ChildAccess): term = term.clone_with_parent(self) diff --git a/pydough/qdag/collections/collection_access.py b/pydough/qdag/collections/collection_access.py index f9185f5d8..d6d78b529 100644 --- a/pydough/qdag/collections/collection_access.py +++ b/pydough/qdag/collections/collection_access.py @@ -105,7 +105,6 @@ def get_expression_position(self, expr_name: str) -> int: return self._calc_property_order[expr_name] def get_term(self, term_name: str) -> PyDoughQDAG: - # breakpoint() self.verify_term_exists(term_name) # Special handling of terms down-streamed from an ancestor CALCULATE diff --git a/pydough/qdag/collections/sub_collection.py b/pydough/qdag/collections/sub_collection.py index bdf962d17..4a0a4b0b0 100644 --- a/pydough/qdag/collections/sub_collection.py +++ b/pydough/qdag/collections/sub_collection.py @@ -94,11 +94,6 @@ def standalone_string(self) -> str: def tree_item_string(self) -> str: return f"SubCollection[{self.standalone_string}]" - @property - def all_terms(self) -> set[str]: - breakpoint() - return self._all_property_names - self._ancestral_mapping.keys() - def equals(self, other: object) -> bool: return ( isinstance(other, SubCollection) diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index 6940a0d72..b948a1337 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -612,7 +612,6 @@ def qualify_access( # child is just a de-sugared invocation of the global context. return qualified_parent else: - # breakpoint() # Identify whether the access is an expression or a collection term = qualified_parent.get_term(name) if isinstance(term, PyDoughCollectionQDAG): @@ -639,6 +638,11 @@ def qualify_access( typ: PyDoughType = context.get_expr(name).pydough_type return self.builder.build_reference(context, name, typ) else: + # Accessing `child.name`: forbid reaching a down-streamed + # (inherited) term through a child collection. + self.verify_not_inherited_through_child( + unqualified, unqualified_parent, qualified_parent, context, name + ) # Otherwise, the access is a reference to a scalar attribute of # a child collection node of the current context. Add this new # child to the list of children, unless already present, then @@ -653,6 +657,86 @@ def qualify_access( children, ref_num, name ) + def names_defined_in_child_chain(self, unqualified: UnqualifiedNode) -> set[str]: + """ + Walks the unqualified collection chain of a child access (e.g. + `customer.WHERE(...).CALCULATE(customer_name=name).orders.BEST(...)`) + down to the root, collecting every term name defined by a CALCULATE + that is part of the chain itself. + + Args: + `unqualified`: the unqualified collection node to walk. + + Returns: + The set of term names defined by CALCULATEs within the chain. + """ + defined: set[str] = set() + node: UnqualifiedNode = unqualified + while not isinstance(node, UnqualifiedRoot): + if isinstance(node, UnqualifiedCalculate): + for term_name, _ in node._parcel[1]: + defined.add(term_name) + predecessor = node._parcel[0] + if not isinstance(predecessor, UnqualifiedNode): + break + node = predecessor + return defined + + def verify_not_inherited_through_child( + self, + unqualified: UnqualifiedAccess, + unqualified_parent: UnqualifiedNode, + qualified_parent: PyDoughCollectionQDAG, + context: PyDoughCollectionQDAG, + name: str, + ) -> None: + """ + Verifies that an expression accessed as `child.name` is not a term + that the child only has because it was down-streamed from the context + the child is being evaluated in. For example, in + `customers.CALCULATE(cust_name=name).orders.CALCULATE(x=customer.cust_name)` + `customer` only sees `cust_name` because it inherits it from the + enclosing context, so the access is invalid (use `cust_name` directly). + + Accesses are still valid when the term comes from the child's own + data, e.g. a CALCULATE inside the child expression + (`customer.CALCULATE(customer_name=name).orders.customer_name`) or + inside the data of a PARTITION child (`pricing_collection.price`). + + Raises: + `PyDoughUnqualifiedException` if the term is inherited from the + enclosing context through the child collection. + """ + # Only terms the child inherits (not its own columns/calcs) matter. + is_inherited: bool = ( + name in qualified_parent.ancestral_mapping + or name in qualified_parent.inherited_downstreamed_terms + ) + if not is_inherited or name in qualified_parent.calc_terms: + return + + # Only invalid if the term comes from the enclosing context. If the + # context itself does not inherit it, it originated inside the + # child's own data (e.g. a PARTITION child's partitioned data). + context_inherits: bool = ( + name in context.ancestral_mapping + or name in context.inherited_downstreamed_terms + ) + if not context_inherits: + return + + # The child chain may redefine (shadow) the same name; then the + # value comes from inside the child, so it is valid. + if name in self.names_defined_in_child_chain(unqualified_parent): + return + + raise PyDoughUnqualifiedException( + f"Cannot access inherited term {name!r} through child collection " + f"{qualified_parent.standalone_string!r} in {unqualified!r}. " + f"Terms down-streamed from an ancestor CALCULATE can only be " + f"referenced directly, e.g. {name!r}." + ) + def qualify_calculate( self, unqualified: UnqualifiedCalculate, diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 59d0178e9..8bce86eb4 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -150,6 +150,7 @@ regional_first_order_best_line_part, regional_suppliers_percentile, richest_customer_per_region, + shadowed_calc_term, simple_cross_1, simple_cross_2, simple_cross_3, @@ -6790,6 +6791,15 @@ ), id="compound_ref_correlation_extraction", ), + pytest.param( + PyDoughPandasTest( + shadowed_calc_term, + "TPCH", + lambda: pd.DataFrame({}), + "shadowed_calc_term_exe", + ), + id="shadowed_calc_term_exe", + ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: @@ -7585,24 +7595,20 @@ def test_pipeline_e2e_simple_week( parent_calc_access_error, None, re.escape( - "Unrecognized term of " - "TPCH.customers.CALCULATE(cust_name=name).orders.customer: " - "'cust_name'. Did you mean: name, phone, key?" + "Unrecognized term of customer: 'cust_name'. Did you mean: name, phone, key?" ), id="parent_calc_access_error", ), - # pytest.param( - # PyDoughPandasTest( - # "main_context = orders.CALCULATE(or_clerk=clerk)\n" - # "result = main_context.lines.CALCULATE(\n" - # " x=order.or_clerk\n" - # ")", - # "TPCH", - # lambda: pd.DataFrame({}), - # "parent_calc_access_error_2", - # ), - # id="parent_calc_access_error_2", - # ), + pytest.param( + shadowed_calc_term, + None, + re.escape( + "Cannot access inherited term 'cust_name' through child collection " + "'customer' in customer.cust_name. Terms down-streamed from an " + "ancestor CALCULATE can only be referenced directly, e.g. 'cust_name'." + ), + id="shadowed_calc_term", + ), ], ) def test_pipeline_e2e_errors( diff --git a/tests/test_plan_refsols/shadowed_calc_term_exe.txt b/tests/test_plan_refsols/shadowed_calc_term_exe.txt new file mode 100644 index 000000000..7ccea377d --- /dev/null +++ b/tests/test_plan_refsols/shadowed_calc_term_exe.txt @@ -0,0 +1,7 @@ +ROOT(columns=[('key', o_orderkey), ('x', max_c_phone)], orderings=[(max_c_phone):asc_first], limit=5:numeric) + JOIN(condition=t0.o_custkey == t1.c_custkey, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=PLURAL_FILTER, columns={'max_c_phone': t1.max_c_phone, 'o_orderkey': t0.o_orderkey}) + SCAN(table=tpch.ORDERS, columns={'o_custkey': o_custkey, 'o_orderkey': o_orderkey}) + AGGREGATE(keys={'c_custkey': c_custkey}, aggregations={'max_c_phone': MAX(c_phone)}) + JOIN(condition=t0.c_custkey == t1.o_custkey, type=INNER, cardinality=PLURAL_FILTER, reverse_cardinality=SINGULAR_ACCESS, columns={'c_custkey': t0.c_custkey, 'c_phone': t0.c_phone}) + SCAN(table=tpch.CUSTOMER, columns={'c_custkey': c_custkey, 'c_phone': c_phone}) + SCAN(table=tpch.ORDERS, columns={'o_custkey': o_custkey}) diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 9a917567a..13e6e7317 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -3900,11 +3900,33 @@ def partition_by_child_reference(): def parent_calc_access_error(): """ - TODO + Test that a child collection's CALCULATE cannot reference a term + defined in a parent collection's CALCULATE, and that doing so raises a + descriptive error message instead of silently qualifying and producing an + error later. """ # 'top' calculates 'cust_name' on customers top = customers.CALCULATE(cust_name=name) # Referencing 'customer.cust_name' from orders causes a lowering error - result = top.orders.CALCULATE(x=customer.cust_name) + result = top.orders.CALCULATE( + correct_usage=cust_name, incorrect_usage=customer.cust_name + ) return result + + +def shadowed_calc_term(): + """ + The child `customer.CALCULATE(cust_name=phone).orders` redefines `cust_name` + as `phone`, and `.orders.cust_name` then reads it. Within that child, the nearest + definition is the inner one, so the value really comes from the child's own + data. + """ + return ( + customers.CALCULATE(cust_name=name) + .orders.CALCULATE( + key, + x=MAX(customer.CALCULATE(cust_name=phone).orders.cust_name), + ) + .TOP_K(5, by=x) + ) diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql new file mode 100644 index 000000000..d5ac40870 --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql new file mode 100644 index 000000000..d5ac40870 --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql new file mode 100644 index 000000000..a6ddae08f --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 NULLS FIRST +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql new file mode 100644 index 000000000..37bf3aac8 --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + CUSTOMER.c_custkey, + MAX(CUSTOMER.c_phone) AS max_c_phone + FROM tpch.CUSTOMER AS CUSTOMER + JOIN tpch.ORDERS AS ORDERS + ON CUSTOMER.c_custkey = ORDERS.o_custkey + GROUP BY + 1 +) +SELECT + ORDERS.o_orderkey AS `key`, + _s3.max_c_phone COLLATE utf8mb4_bin AS x +FROM tpch.ORDERS AS ORDERS +LEFT JOIN _s3 AS _s3 + ON ORDERS.o_custkey = _s3.c_custkey +ORDER BY + 2 +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql new file mode 100644 index 000000000..baba13f49 --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql @@ -0,0 +1,19 @@ +WITH "_S3" AS ( + SELECT + CUSTOMER.c_custkey AS C_CUSTKEY, + MAX(CUSTOMER.c_phone) AS MAX_C_PHONE + FROM TPCH.CUSTOMER CUSTOMER + JOIN TPCH.ORDERS ORDERS + ON CUSTOMER.c_custkey = ORDERS.o_custkey + GROUP BY + CUSTOMER.c_custkey +) +SELECT + ORDERS.o_orderkey AS "key", + "_S3".MAX_C_PHONE AS x +FROM TPCH.ORDERS ORDERS +LEFT JOIN "_S3" "_S3" + ON ORDERS.o_custkey = "_S3".C_CUSTKEY +ORDER BY + 2 NULLS FIRST +FETCH FIRST 5 ROWS ONLY diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql new file mode 100644 index 000000000..a6ddae08f --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 NULLS FIRST +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql new file mode 100644 index 000000000..a6ddae08f --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 NULLS FIRST +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql new file mode 100644 index 000000000..d5ac40870 --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql new file mode 100644 index 000000000..a6ddae08f --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 2 NULLS FIRST +LIMIT 5 From 6180ecd1061f5b0da49b849f1c0b3de01ee94ae1 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 10:27:53 -0600 Subject: [PATCH 05/14] fixing test --- tests/test_pipeline_tpch_custom.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 8bce86eb4..b38241fb3 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -6795,7 +6795,12 @@ PyDoughPandasTest( shadowed_calc_term, "TPCH", - lambda: pd.DataFrame({}), + lambda: pd.DataFrame( + { + "key": [105056, 428615, 847044, 918659, 1125219], + "x": ["10-100-106-1617"] * 5, + } + ), "shadowed_calc_term_exe", ), id="shadowed_calc_term_exe", @@ -7599,16 +7604,6 @@ def test_pipeline_e2e_simple_week( ), id="parent_calc_access_error", ), - pytest.param( - shadowed_calc_term, - None, - re.escape( - "Cannot access inherited term 'cust_name' through child collection " - "'customer' in customer.cust_name. Terms down-streamed from an " - "ancestor CALCULATE can only be referenced directly, e.g. 'cust_name'." - ), - id="shadowed_calc_term", - ), ], ) def test_pipeline_e2e_errors( From f8c55291b10ce4a2ad3a9c11d47d6fd413bfb8e3 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 11:07:20 -0600 Subject: [PATCH 06/14] fixing shadowed test[run ci][run all] --- tests/test_pipeline_tpch_custom.py | 14 ++++++++++---- ...lc_term_exe.txt => shadowed_calc_term.txt} | 2 +- .../simple_pydough_functions.py | 2 +- ...sqlite.sql => shadowed_calc_term_ansi.sql} | 2 +- ....sql => shadowed_calc_term_databricks.sql} | 2 +- ...gres.sql => shadowed_calc_term_duckdb.sql} | 2 +- .../shadowed_calc_term_exe_snowflake.sql | 19 ------------------- .../shadowed_calc_term_exe_trino.sql | 19 ------------------- ...mysql.sql => shadowed_calc_term_mysql.sql} | 4 ++-- ...acle.sql => shadowed_calc_term_oracle.sql} | 2 +- ...db.sql => shadowed_calc_term_postgres.sql} | 2 +- .../shadowed_calc_term_snowflake.sql | 19 +++++++++++++++++++ ...icks.sql => shadowed_calc_term_sqlite.sql} | 2 +- .../shadowed_calc_term_trino.sql | 19 +++++++++++++++++++ tests/testing_utilities.py | 1 + 15 files changed, 59 insertions(+), 52 deletions(-) rename tests/test_plan_refsols/{shadowed_calc_term_exe.txt => shadowed_calc_term.txt} (94%) rename tests/test_sql_refsols/{shadowed_calc_term_exe_sqlite.sql => shadowed_calc_term_ansi.sql} (98%) rename tests/test_sql_refsols/{shadowed_calc_term_exe_ansi.sql => shadowed_calc_term_databricks.sql} (98%) rename tests/test_sql_refsols/{shadowed_calc_term_exe_postgres.sql => shadowed_calc_term_duckdb.sql} (95%) delete mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql delete mode 100644 tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql rename tests/test_sql_refsols/{shadowed_calc_term_exe_mysql.sql => shadowed_calc_term_mysql.sql} (88%) rename tests/test_sql_refsols/{shadowed_calc_term_exe_oracle.sql => shadowed_calc_term_oracle.sql} (96%) rename tests/test_sql_refsols/{shadowed_calc_term_exe_duckdb.sql => shadowed_calc_term_postgres.sql} (95%) create mode 100644 tests/test_sql_refsols/shadowed_calc_term_snowflake.sql rename tests/test_sql_refsols/{shadowed_calc_term_exe_databricks.sql => shadowed_calc_term_sqlite.sql} (98%) create mode 100644 tests/test_sql_refsols/shadowed_calc_term_trino.sql diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 45bf112e9..a1b11e472 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -6825,13 +6825,19 @@ "TPCH", lambda: pd.DataFrame( { - "key": [105056, 428615, 847044, 918659, 1125219], - "x": ["10-100-106-1617"] * 5, + "key": [1, 2, 3, 4, 5], + "x": [ + "23-644-998-4944", + "20-715-308-7926", + "25-884-345-1592", + "20-500-807-1549", + "30-452-969-2072", + ], } ), - "shadowed_calc_term_exe", + "shadowed_calc_term", ), - id="shadowed_calc_term_exe", + id="shadowed_calc_term", ), ], ) diff --git a/tests/test_plan_refsols/shadowed_calc_term_exe.txt b/tests/test_plan_refsols/shadowed_calc_term.txt similarity index 94% rename from tests/test_plan_refsols/shadowed_calc_term_exe.txt rename to tests/test_plan_refsols/shadowed_calc_term.txt index 7ccea377d..45959c79c 100644 --- a/tests/test_plan_refsols/shadowed_calc_term_exe.txt +++ b/tests/test_plan_refsols/shadowed_calc_term.txt @@ -1,4 +1,4 @@ -ROOT(columns=[('key', o_orderkey), ('x', max_c_phone)], orderings=[(max_c_phone):asc_first], limit=5:numeric) +ROOT(columns=[('key', o_orderkey), ('x', max_c_phone)], orderings=[(o_orderkey):asc_first], limit=5:numeric) JOIN(condition=t0.o_custkey == t1.c_custkey, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=PLURAL_FILTER, columns={'max_c_phone': t1.max_c_phone, 'o_orderkey': t0.o_orderkey}) SCAN(table=tpch.ORDERS, columns={'o_custkey': o_custkey, 'o_orderkey': o_orderkey}) AGGREGATE(keys={'c_custkey': c_custkey}, aggregations={'max_c_phone': MAX(c_phone)}) diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 513443a06..05e7b4112 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -3997,5 +3997,5 @@ def shadowed_calc_term(): key, x=MAX(customer.CALCULATE(cust_name=phone).orders.cust_name), ) - .TOP_K(5, by=x) + .TOP_K(5, by=key.ASC()) ) diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql b/tests/test_sql_refsols/shadowed_calc_term_ansi.sql similarity index 98% rename from tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql rename to tests/test_sql_refsols/shadowed_calc_term_ansi.sql index d5ac40870..430faed0d 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_sqlite.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_ansi.sql @@ -15,5 +15,5 @@ FROM tpch.orders AS orders LEFT JOIN _s3 AS _s3 ON _s3.c_custkey = orders.o_custkey ORDER BY - 2 + 1 LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql b/tests/test_sql_refsols/shadowed_calc_term_databricks.sql similarity index 98% rename from tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql rename to tests/test_sql_refsols/shadowed_calc_term_databricks.sql index d5ac40870..430faed0d 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_ansi.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_databricks.sql @@ -15,5 +15,5 @@ FROM tpch.orders AS orders LEFT JOIN _s3 AS _s3 ON _s3.c_custkey = orders.o_custkey ORDER BY - 2 + 1 LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql b/tests/test_sql_refsols/shadowed_calc_term_duckdb.sql similarity index 95% rename from tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql rename to tests/test_sql_refsols/shadowed_calc_term_duckdb.sql index a6ddae08f..eaafb127d 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_postgres.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_duckdb.sql @@ -15,5 +15,5 @@ FROM tpch.orders AS orders LEFT JOIN _s3 AS _s3 ON _s3.c_custkey = orders.o_custkey ORDER BY - 2 NULLS FIRST + 1 NULLS FIRST LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql deleted file mode 100644 index a6ddae08f..000000000 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_snowflake.sql +++ /dev/null @@ -1,19 +0,0 @@ -WITH _s3 AS ( - SELECT - customer.c_custkey, - MAX(customer.c_phone) AS max_c_phone - FROM tpch.customer AS customer - JOIN tpch.orders AS orders - ON customer.c_custkey = orders.o_custkey - GROUP BY - 1 -) -SELECT - orders.o_orderkey AS key, - _s3.max_c_phone AS x -FROM tpch.orders AS orders -LEFT JOIN _s3 AS _s3 - ON _s3.c_custkey = orders.o_custkey -ORDER BY - 2 NULLS FIRST -LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql b/tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql deleted file mode 100644 index a6ddae08f..000000000 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_trino.sql +++ /dev/null @@ -1,19 +0,0 @@ -WITH _s3 AS ( - SELECT - customer.c_custkey, - MAX(customer.c_phone) AS max_c_phone - FROM tpch.customer AS customer - JOIN tpch.orders AS orders - ON customer.c_custkey = orders.o_custkey - GROUP BY - 1 -) -SELECT - orders.o_orderkey AS key, - _s3.max_c_phone AS x -FROM tpch.orders AS orders -LEFT JOIN _s3 AS _s3 - ON _s3.c_custkey = orders.o_custkey -ORDER BY - 2 NULLS FIRST -LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql b/tests/test_sql_refsols/shadowed_calc_term_mysql.sql similarity index 88% rename from tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql rename to tests/test_sql_refsols/shadowed_calc_term_mysql.sql index 37bf3aac8..675c6300a 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_mysql.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_mysql.sql @@ -10,10 +10,10 @@ WITH _s3 AS ( ) SELECT ORDERS.o_orderkey AS `key`, - _s3.max_c_phone COLLATE utf8mb4_bin AS x + _s3.max_c_phone AS x FROM tpch.ORDERS AS ORDERS LEFT JOIN _s3 AS _s3 ON ORDERS.o_custkey = _s3.c_custkey ORDER BY - 2 + 1 LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql b/tests/test_sql_refsols/shadowed_calc_term_oracle.sql similarity index 96% rename from tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql rename to tests/test_sql_refsols/shadowed_calc_term_oracle.sql index baba13f49..e2f0097bf 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_oracle.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_oracle.sql @@ -15,5 +15,5 @@ FROM TPCH.ORDERS ORDERS LEFT JOIN "_S3" "_S3" ON ORDERS.o_custkey = "_S3".C_CUSTKEY ORDER BY - 2 NULLS FIRST + 1 NULLS FIRST FETCH FIRST 5 ROWS ONLY diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql b/tests/test_sql_refsols/shadowed_calc_term_postgres.sql similarity index 95% rename from tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql rename to tests/test_sql_refsols/shadowed_calc_term_postgres.sql index a6ddae08f..eaafb127d 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_duckdb.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_postgres.sql @@ -15,5 +15,5 @@ FROM tpch.orders AS orders LEFT JOIN _s3 AS _s3 ON _s3.c_custkey = orders.o_custkey ORDER BY - 2 NULLS FIRST + 1 NULLS FIRST LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_snowflake.sql b/tests/test_sql_refsols/shadowed_calc_term_snowflake.sql new file mode 100644 index 000000000..eaafb127d --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_snowflake.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 1 NULLS FIRST +LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql b/tests/test_sql_refsols/shadowed_calc_term_sqlite.sql similarity index 98% rename from tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql rename to tests/test_sql_refsols/shadowed_calc_term_sqlite.sql index d5ac40870..430faed0d 100644 --- a/tests/test_sql_refsols/shadowed_calc_term_exe_databricks.sql +++ b/tests/test_sql_refsols/shadowed_calc_term_sqlite.sql @@ -15,5 +15,5 @@ FROM tpch.orders AS orders LEFT JOIN _s3 AS _s3 ON _s3.c_custkey = orders.o_custkey ORDER BY - 2 + 1 LIMIT 5 diff --git a/tests/test_sql_refsols/shadowed_calc_term_trino.sql b/tests/test_sql_refsols/shadowed_calc_term_trino.sql new file mode 100644 index 000000000..eaafb127d --- /dev/null +++ b/tests/test_sql_refsols/shadowed_calc_term_trino.sql @@ -0,0 +1,19 @@ +WITH _s3 AS ( + SELECT + customer.c_custkey, + MAX(customer.c_phone) AS max_c_phone + FROM tpch.customer AS customer + JOIN tpch.orders AS orders + ON customer.c_custkey = orders.o_custkey + GROUP BY + 1 +) +SELECT + orders.o_orderkey AS key, + _s3.max_c_phone AS x +FROM tpch.orders AS orders +LEFT JOIN _s3 AS _s3 + ON _s3.c_custkey = orders.o_custkey +ORDER BY + 1 NULLS FIRST +LIMIT 5 diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 8a6c612f5..e3aaeb5fa 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1562,6 +1562,7 @@ def run_e2e_test( # If the query does not care about column names, update the answer to use # the column names in the refsol. + # breakpoint() if self.fix_column_names: assert len(result.columns) == len(refsol.columns) result.columns = refsol.columns From fec0f66b7230e979d9d07d322187fcfe0174bc7e Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 11:07:45 -0600 Subject: [PATCH 07/14] deleting breakpoint [run ci][run all] --- tests/testing_utilities.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index e3aaeb5fa..8a6c612f5 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1562,7 +1562,6 @@ def run_e2e_test( # If the query does not care about column names, update the answer to use # the column names in the refsol. - # breakpoint() if self.fix_column_names: assert len(result.columns) == len(refsol.columns) result.columns = refsol.columns From cf356839c7e1f4f14a490d7fd56bba3c76662f93 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 11:29:42 -0600 Subject: [PATCH 08/14] updating error message test [run ci][run all] --- tests/test_pipeline_tpch_custom.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index a1b11e472..3f8ad1859 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -7662,7 +7662,10 @@ def test_pipeline_e2e_simple_week( parent_calc_access_error, None, re.escape( - "Unrecognized term of customer: 'cust_name'. Did you mean: name, phone, key?" + "Cannot access inherited term 'cust_name' through child collection " + "'customer' in customer.cust_name. Terms down-streamed from an " + "ancestor CALCULATE can only be referenced directly, " + "e.g. 'cust_name'." ), id="parent_calc_access_error", ), From bd9cec32815daced76e4b6888d84f3c72e69eb7f Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 11:31:34 -0600 Subject: [PATCH 09/14] deleting breakpoint [run ci][run all] --- pydough/conversion/relational_converter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index 4774d6661..6a42fa3b3 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -1920,7 +1920,6 @@ def convert_ast_to_relational( of `node` are included in the root in the correct order, and if it has an ordering then the relational root stores that information. """ - # breakpoint() # Pre-process the QDAG node so the final CALCULATE includes any ordering # keys. rel_translator: RelTranslation = RelTranslation(session) From 959483dcfe17dfc5923b3b425229b90bac29eec4 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 5 Oct 2026 13:20:38 -0600 Subject: [PATCH 10/14] fixing bodosql and pyiceberg version [run ci][run all] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 098aee9d8..ea45f3ce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ trino = ["trino"] mysql = ["mysql-connector-python==9.5.0"] postgres = ["psycopg2-binary"] server = ["fastapi", "httpx", "uvicorn"] -bodosql = ["bodo>=2026.2", "bodosql>=2026.2", "pyiceberg[pyiceberg-core]==0.10.0"] +bodosql = ["bodo>=2026.10", "bodosql>=2026.10", "pyiceberg[pyiceberg-core]==0.12.0"] oracle = ["oracledb==3.4.2"] databricks = ["databricks-sql-connector==4.2.6"] duckdb = ["duckdb>=1.5.4"] From 7ff3ccdd45317c2cd0af83f31c542818c52ed77a Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Tue, 6 Oct 2026 08:42:43 -0600 Subject: [PATCH 11/14] reversing bodo version related changes [run ci][run all] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ea45f3ce2..098aee9d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ trino = ["trino"] mysql = ["mysql-connector-python==9.5.0"] postgres = ["psycopg2-binary"] server = ["fastapi", "httpx", "uvicorn"] -bodosql = ["bodo>=2026.10", "bodosql>=2026.10", "pyiceberg[pyiceberg-core]==0.12.0"] +bodosql = ["bodo>=2026.2", "bodosql>=2026.2", "pyiceberg[pyiceberg-core]==0.10.0"] oracle = ["oracledb==3.4.2"] databricks = ["databricks-sql-connector==4.2.6"] duckdb = ["duckdb>=1.5.4"] From 91c534ac61596b2c540146877a14aafc068b9303 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Tue, 6 Oct 2026 16:09:46 -0600 Subject: [PATCH 12/14] adding fix for server tests setup and new test [run ci][run all] --- tests/conftest.py | 6 ++++- tests/test_pipeline_tpch_custom.py | 22 ++++++++++++++++++ .../partition_child_inherited_term.txt | 5 ++++ .../simple_pydough_functions.py | 23 +++++++++++++++++++ .../partition_child_inherited_term_ansi.sql | 11 +++++++++ ...tition_child_inherited_term_databricks.sql | 11 +++++++++ .../partition_child_inherited_term_duckdb.sql | 11 +++++++++ .../partition_child_inherited_term_mysql.sql | 11 +++++++++ .../partition_child_inherited_term_oracle.sql | 11 +++++++++ ...artition_child_inherited_term_postgres.sql | 11 +++++++++ ...rtition_child_inherited_term_snowflake.sql | 11 +++++++++ .../partition_child_inherited_term_sqlite.sql | 11 +++++++++ .../partition_child_inherited_term_trino.sql | 11 +++++++++ 13 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 tests/test_plan_refsols/partition_child_inherited_term.txt create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_ansi.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_databricks.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_duckdb.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_mysql.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_oracle.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_postgres.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_snowflake.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_sqlite.sql create mode 100644 tests/test_sql_refsols/partition_child_inherited_term_trino.sql diff --git a/tests/conftest.py b/tests/conftest.py index 6213eccce..bf042beee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3222,7 +3222,11 @@ def mock_server_setup(): time.sleep(0.5) else: proc.terminate() - output, _ = proc.communicate() + try: + output, _ = proc.communicate(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + output, _ = proc.communicate() raise RuntimeError(f"Mock server failed to start:\n{output}") yield server_url diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 3f8ad1859..32b5f3c46 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -131,6 +131,7 @@ part_reduced_size, partition_by_child_reference, partition_by_literal, + partition_child_inherited_term, partition_key_name_collision, partition_self_reference, parts_quantity_increase_95_96, @@ -6839,6 +6840,27 @@ ), id="shadowed_calc_term", ), + pytest.param( + PyDoughPandasTest( + partition_child_inherited_term, + "TPCH", + lambda: pd.DataFrame( + { + "rkey": [0, 1, 2, 3, 4], + "region_name": [ + "AFRICA", + "AMERICA", + "ASIA", + "EUROPE", + "MIDDLE EAST", + ], + "n_nations": [5, 5, 5, 5, 5], + } + ), + "partition_child_inherited_term", + ), + id="partition_child_inherited_term", + ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: diff --git a/tests/test_plan_refsols/partition_child_inherited_term.txt b/tests/test_plan_refsols/partition_child_inherited_term.txt new file mode 100644 index 000000000..98c0f4cd0 --- /dev/null +++ b/tests/test_plan_refsols/partition_child_inherited_term.txt @@ -0,0 +1,5 @@ +ROOT(columns=[('rkey', r_regionkey), ('region_name', max_r_name), ('n_nations', n_rows)], orderings=[(r_regionkey):asc_first]) + AGGREGATE(keys={'r_regionkey': r_regionkey}, aggregations={'max_r_name': MAX(r_name), 'n_rows': COUNT()}) + JOIN(condition=t0.r_regionkey == t1.n_regionkey, type=INNER, cardinality=PLURAL_FILTER, reverse_cardinality=PLURAL_FILTER, columns={'r_name': t0.r_name, 'r_regionkey': t0.r_regionkey}) + SCAN(table=tpch.REGION, columns={'r_name': r_name, 'r_regionkey': r_regionkey}) + SCAN(table=tpch.NATION, columns={'n_regionkey': n_regionkey}) diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 05e7b4112..41e228d0f 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -3999,3 +3999,26 @@ def shadowed_calc_term(): ) .TOP_K(5, by=key.ASC()) ) + + +def partition_child_inherited_term(): + """ + Pair every region with its nations via CROSS, partition the pairs by + region, and aggregate a term (`region_name`) that was down-streamed from + the regions CALCULATE *inside the partitioned data*. The PARTITION + context only exposes `rkey`, so accessing `nations.region_name` through + the partition child is valid (context_inherits is False). + """ + + return ( + regions.CALCULATE(rkey=key, region_name=name) + .CROSS(nations) + .WHERE(region_key == rkey) + .PARTITION(name="region_groups", by=rkey) + .CALCULATE( + rkey, + region_name=MAX(nations.region_name), + n_nations=COUNT(nations), + ) + .ORDER_BY(rkey.ASC()) + ) diff --git a/tests/test_sql_refsols/partition_child_inherited_term_ansi.sql b/tests/test_sql_refsols/partition_child_inherited_term_ansi.sql new file mode 100644 index 000000000..5cfe7d54d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_ansi.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 diff --git a/tests/test_sql_refsols/partition_child_inherited_term_databricks.sql b/tests/test_sql_refsols/partition_child_inherited_term_databricks.sql new file mode 100644 index 000000000..5cfe7d54d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_databricks.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 diff --git a/tests/test_sql_refsols/partition_child_inherited_term_duckdb.sql b/tests/test_sql_refsols/partition_child_inherited_term_duckdb.sql new file mode 100644 index 000000000..7029b203d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_duckdb.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/partition_child_inherited_term_mysql.sql b/tests/test_sql_refsols/partition_child_inherited_term_mysql.sql new file mode 100644 index 000000000..4f14d5032 --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_mysql.sql @@ -0,0 +1,11 @@ +SELECT + REGION.r_regionkey AS rkey, + MAX(REGION.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.REGION AS REGION +JOIN tpch.NATION AS NATION + ON NATION.n_regionkey = REGION.r_regionkey +GROUP BY + 1 +ORDER BY + 1 diff --git a/tests/test_sql_refsols/partition_child_inherited_term_oracle.sql b/tests/test_sql_refsols/partition_child_inherited_term_oracle.sql new file mode 100644 index 000000000..b7565cde8 --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_oracle.sql @@ -0,0 +1,11 @@ +SELECT + REGION.r_regionkey AS rkey, + MAX(REGION.r_name) AS region_name, + COUNT(*) AS n_nations +FROM TPCH.REGION REGION +JOIN TPCH.NATION NATION + ON NATION.n_regionkey = REGION.r_regionkey +GROUP BY + REGION.r_regionkey +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/partition_child_inherited_term_postgres.sql b/tests/test_sql_refsols/partition_child_inherited_term_postgres.sql new file mode 100644 index 000000000..7029b203d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_postgres.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/partition_child_inherited_term_snowflake.sql b/tests/test_sql_refsols/partition_child_inherited_term_snowflake.sql new file mode 100644 index 000000000..7029b203d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_snowflake.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/partition_child_inherited_term_sqlite.sql b/tests/test_sql_refsols/partition_child_inherited_term_sqlite.sql new file mode 100644 index 000000000..5cfe7d54d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_sqlite.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 diff --git a/tests/test_sql_refsols/partition_child_inherited_term_trino.sql b/tests/test_sql_refsols/partition_child_inherited_term_trino.sql new file mode 100644 index 000000000..7029b203d --- /dev/null +++ b/tests/test_sql_refsols/partition_child_inherited_term_trino.sql @@ -0,0 +1,11 @@ +SELECT + region.r_regionkey AS rkey, + MAX(region.r_name) AS region_name, + COUNT(*) AS n_nations +FROM tpch.region AS region +JOIN tpch.nation AS nation + ON nation.n_regionkey = region.r_regionkey +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST From 4f70f32dfa19b3a2ec437373d4f934c67037f4a9 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Thu, 8 Oct 2026 09:20:51 -0600 Subject: [PATCH 13/14] changing names_defined_in_child_chain implementation [run ci][run all] --- pydough/unqualified/qualification.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index b948a1337..1d2cdd8ce 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -657,27 +657,27 @@ def qualify_access( children, ref_num, name ) - def names_defined_in_child_chain(self, unqualified: UnqualifiedNode) -> set[str]: + def names_defined_in_child_chain( + self, qualified_node: PyDoughCollectionQDAG + ) -> set[str]: """ - Walks the unqualified collection chain of a child access (e.g. - `customer.WHERE(...).CALCULATE(customer_name=name).orders.BEST(...)`) - down to the root, collecting every term name defined by a CALCULATE - that is part of the chain itself. + Walks the qualified node chain down to the root, collecting every term + name defined by a CALCULATE that is part of the chain itself. Args: - `unqualified`: the unqualified collection node to walk. + `qualified_node`: the qualified node to walk. Returns: The set of term names defined by CALCULATEs within the chain. """ defined: set[str] = set() - node: UnqualifiedNode = unqualified - while not isinstance(node, UnqualifiedRoot): - if isinstance(node, UnqualifiedCalculate): - for term_name, _ in node._parcel[1]: + node: PyDoughCollectionQDAG = qualified_node + while not isinstance(node, GlobalContext): + if isinstance(node, PyDoughCollectionQDAG): + for term_name in node.calc_terms: defined.add(term_name) - predecessor = node._parcel[0] - if not isinstance(predecessor, UnqualifiedNode): + predecessor = node.ancestor_context + if not predecessor: break node = predecessor return defined @@ -727,7 +727,7 @@ def verify_not_inherited_through_child( # The child chain may redefine (shadow) the same name; then the # value comes from inside the child, so it is valid. - if name in self.names_defined_in_child_chain(unqualified_parent): + if name in self.names_defined_in_child_chain(qualified_parent): return raise PyDoughUnqualifiedException( From 4945d50f57743a405f05d3b42b5ab980c4ae336e Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Thu, 8 Oct 2026 10:27:59 -0600 Subject: [PATCH 14/14] fixing implementation with qdag [run ci][run all] --- pydough/unqualified/qualification.py | 41 ++++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index 1d2cdd8ce..5cef3fa70 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -34,6 +34,7 @@ SubCollection, WindowCall, ) +from pydough.qdag.collections.calculate import Calculate from pydough.types import PyDoughType from pydough.utilities import ExplodeSpec @@ -658,28 +659,38 @@ def qualify_access( ) def names_defined_in_child_chain( - self, qualified_node: PyDoughCollectionQDAG + self, + qualified_node: PyDoughCollectionQDAG, + context: PyDoughCollectionQDAG, ) -> set[str]: """ - Walks the qualified node chain down to the root, collecting every term - name defined by a CALCULATE that is part of the chain itself. + Walks the qualified chain of a child access up to (but not including) + the context the child is evaluated in, collecting every term name + defined by a CALCULATE that is part of the child chain itself. Args: - `qualified_node`: the qualified node to walk. + `qualified_node`: the qualified collection the term is accessed from. + `context`: the context the child chain is attached to; the walk + stops here so CALCULATEs from the enclosing context are ignored. Returns: - The set of term names defined by CALCULATEs within the chain. + The set of term names defined by CALCULATEs within the child chain. """ defined: set[str] = set() - node: PyDoughCollectionQDAG = qualified_node - while not isinstance(node, GlobalContext): - if isinstance(node, PyDoughCollectionQDAG): - for term_name in node.calc_terms: - defined.add(term_name) - predecessor = node.ancestor_context - if not predecessor: - break - node = predecessor + node: PyDoughCollectionQDAG | None = qualified_node + while ( + node is not None + and node is not context + and not isinstance(node, GlobalContext) + ): + if isinstance(node, Calculate): + defined.update(name for name in node.calc_term_values.keys()) + # Operators (CALCULATE, WHERE, SINGULAR, ...) step back through + # their preceding context; child accesses step up to their parent. + if node.preceding_context is not None: + node = node.preceding_context + else: + node = node.ancestor_context return defined def verify_not_inherited_through_child( @@ -727,7 +738,7 @@ def verify_not_inherited_through_child( # The child chain may redefine (shadow) the same name; then the # value comes from inside the child, so it is valid. - if name in self.names_defined_in_child_chain(qualified_parent): + if name in self.names_defined_in_child_chain(qualified_parent, context): return raise PyDoughUnqualifiedException(