From 1d705538aa6d1bde3429e5911f590f5181875680 Mon Sep 17 00:00:00 2001 From: SAY-5 Date: Wed, 15 Apr 2026 08:52:59 -0700 Subject: [PATCH] fix(optimizer): preserve filter execution order in PushDownFilter When PushDownFilter collapses a parent Filter into a child Filter, the combined predicate list was built as parent_predicates.chain(child_predicates). Because the unoptimized plan evaluates inner (child) filters before outer (parent) filters, this reversed the user-authored execution order of selective predicates: a query written as LogicalPlanBuilder::from(scan) .filter(col("a").eq(lit(10)))? // applied first (inner) .filter(col("b").gt(lit(11)))? // applied second (outer) was optimized into 'Filter: b > 11 AND a = 10' rather than the expected 'Filter: a = 10 AND b > 11'. In environments without filter-selectivity statistics this matters: users who put their cheap or highly-selective filter first expect it to execute first. Build the conjunction as child_predicates.chain(parents_predicates) so PushDownFilter actually preserves the execution order it claims to. Update the two_filters_on_same_depth snapshot to assert the corrected ordering and document why it matters in a short comment. Closes #21642. --- datafusion/optimizer/src/push_down_filter.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index a1a636cfef9af..ec0f801d40994 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -814,9 +814,16 @@ impl OptimizerRule for PushDownFilter { // remove duplicated filters let child_predicates = split_conjunction_owned(child_filter.predicate); - let new_predicates = parents_predicates + // The unoptimized plan evaluates the child filter first + // (inner nodes feed outer nodes), so when we collapse a + // parent filter into its child filter we must preserve that + // execution order — child predicates first, then the + // parent's predicates. Putting the parent's predicates + // first reverses the user-authored order and changes the + // observed evaluation order of selective predicates. + let new_predicates = child_predicates .into_iter() - .chain(child_predicates) + .chain(parents_predicates) // use IndexSet to remove dupes while preserving predicate order .collect::>() .into_iter() @@ -2472,9 +2479,14 @@ mod tests { ); assert_optimized_plan_equal!( plan, + // The unoptimized plan applies `a <= 1` first (it is the inner + // filter, fed into the outer `a >= 1` filter). PushDownFilter + // collapses the two filters into a single conjunction and must + // preserve that execution order: child predicate first, then the + // parent's predicate. @r" Projection: test.a - Filter: test.a >= Int64(1) AND test.a <= Int64(1) + Filter: test.a <= Int64(1) AND test.a >= Int64(1) Limit: skip=0, fetch=1 TableScan: test "