From 5e376e56e4f671d14ac941ebc1b2152d12c15b02 Mon Sep 17 00:00:00 2001 From: HairstonE Date: Thu, 11 Jun 2026 13:04:45 -0400 Subject: [PATCH 1/4] keep one column of the mark so it stays qualified + unit test --- .../optimizer/src/optimize_projections/mod.rs | 69 ++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index acdbf71d05d5c..593f5c8d5fdaa 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -764,12 +764,12 @@ fn split_join_requirements( // The mark column is synthetic (produced by the join itself), // so discard it and route only to the left child. let (left_indices, _mark) = indices.split_off(left_len); - (left_indices, RequiredIndices::new()) + (left_indices, RequiredIndices::new().append(&[0])) } JoinType::RightMark => { // Same as LeftMark, but for the right child. let (right_indices, _mark) = indices.split_off(right_len); - (RequiredIndices::new(), right_indices) + (RequiredIndices::new().append(&[0]), right_indices) } // All requirements can be re-routed to left child directly. JoinType::LeftAnti | JoinType::LeftSemi => (indices, RequiredIndices::new()), @@ -2388,6 +2388,71 @@ mod tests { ) } + // Regression test for https://github.com/apache/datafusion/issues/22477 + // `= ANY` / `<> ALL` decorrelate into several stacked LeftMark joins whose + // (pushed-down) filters reference no right column. OptimizeProjections must + // not prune each mark join's right child to zero columns: a zero-column + // child has no table reference, so `mark_field` would emit an unqualified + // `mark` and the stacked marks would collide on the duplicate bare name. + // Here each mark stays qualified (`__correlated_sq_N.mark`) and the plan + // optimizes without error. + #[test] + fn optimize_projections_stacked_mark_joins_keep_qualified_mark() -> Result<()> { + let person = test_table_scan_with_name("person")?; + + let aliased_scan = |table: &str, alias: &str| -> Result { + LogicalPlanBuilder::from(test_table_scan_with_name(table)?) + .project(vec![col(format!("{table}.a"))])? + .alias(alias)? + .build() + }; + + // Three stacked LeftMark joins with trivially-`true` filters (the + // filter-less shape left behind by push_down_filter), feeding a filter + // that references all three marks. + let plan = LogicalPlanBuilder::from(person) + .join_on( + aliased_scan("s1", "__correlated_sq_1")?, + JoinType::LeftMark, + vec![lit(true)], + )? + .join_on( + aliased_scan("s2", "__correlated_sq_2")?, + JoinType::LeftMark, + vec![lit(true)], + )? + .join_on( + aliased_scan("s3", "__correlated_sq_3")?, + JoinType::LeftMark, + vec![lit(true)], + )? + .filter( + col("__correlated_sq_1.mark") + .or(col("__correlated_sq_2.mark")) + .and(not(col("__correlated_sq_3.mark"))), + )? + .project(vec![col("person.a")])? + .build()?; + + assert_optimized_plan_equal!( + plan, + @r" + Projection: person.a + Filter: (__correlated_sq_1.mark OR __correlated_sq_2.mark) AND NOT __correlated_sq_3.mark + LeftMark Join: Filter: Boolean(true) + LeftMark Join: Filter: Boolean(true) + LeftMark Join: Filter: Boolean(true) + TableScan: person projection=[a] + SubqueryAlias: __correlated_sq_1 + TableScan: s1 projection=[a] + SubqueryAlias: __correlated_sq_2 + TableScan: s2 projection=[a] + SubqueryAlias: __correlated_sq_3 + TableScan: s3 projection=[a] + " + ) + } + fn observe(_plan: &LogicalPlan, _rule: &dyn OptimizerRule) {} fn optimize(plan: LogicalPlan) -> Result { From de20a51491f3508d77737fe0d5f6773e8e9d8937 Mon Sep 17 00:00:00 2001 From: HairstonE Date: Thu, 11 Jun 2026 13:05:01 -0400 Subject: [PATCH 2/4] .slt regression coverage --- .../sqllogictest/test_files/subquery.slt | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 908ee6bb3be75..72810b29d7cd4 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -1713,6 +1713,32 @@ logical_plan 21)----------Projection: column1 AS v 22)------------Values: (Int64(5)), (Int64(NULL)) +# Regression test for https://github.com/apache/datafusion/issues/22477 +# `= ANY` / `<> ALL` over the SAME table decorrelate into several stacked +# LeftMark joins. After push_down_filter the join filters reference no right +# column, so the mark joins are filter-less. optimize_projections must not prune +# each mark join's right child to zero columns (which would make the synthetic +# `mark` unqualified and collide across the stack). Previously failed planning +# with "Schema contains duplicate unqualified field name mark". +statement ok +create table set_cmp_self(id int, age int) as values (1, 20), (2, 30), (3, 40); + +# every row's age is in {20, 30, 40}, so `= ANY` holds for all rows +query I rowsort +select id from set_cmp_self where age = any(select age from set_cmp_self); +---- +1 +2 +3 + +# each row's age equals itself in the set, so `<> ALL` is false for every row +query I +select id from set_cmp_self where age <> all(select age from set_cmp_self); +---- + +statement count 0 +drop table set_cmp_self; + # correlated_recursive_scalar_subquery_with_level_3_exists_subquery_referencing_level1_relation query TT explain select c_custkey from customer From 2cb6aacc00156f678aeceeecb2b9e88ac78f846d Mon Sep 17 00:00:00 2001 From: HairstonE Date: Thu, 11 Jun 2026 13:15:44 -0400 Subject: [PATCH 3/4] cutting comments on unit and .slt tests --- .../optimizer/src/optimize_projections/mod.rs | 13 ++----------- datafusion/sqllogictest/test_files/subquery.slt | 11 ++--------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index 593f5c8d5fdaa..f96a209e31c65 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -2388,14 +2388,8 @@ mod tests { ) } - // Regression test for https://github.com/apache/datafusion/issues/22477 - // `= ANY` / `<> ALL` decorrelate into several stacked LeftMark joins whose - // (pushed-down) filters reference no right column. OptimizeProjections must - // not prune each mark join's right child to zero columns: a zero-column - // child has no table reference, so `mark_field` would emit an unqualified - // `mark` and the stacked marks would collide on the duplicate bare name. - // Here each mark stays qualified (`__correlated_sq_N.mark`) and the plan - // optimizes without error. + // Stacked filter-less LeftMark joins (from `= ANY` / `<> ALL`) must keep + // each `mark` qualified so they don't collide. #[test] fn optimize_projections_stacked_mark_joins_keep_qualified_mark() -> Result<()> { let person = test_table_scan_with_name("person")?; @@ -2407,9 +2401,6 @@ mod tests { .build() }; - // Three stacked LeftMark joins with trivially-`true` filters (the - // filter-less shape left behind by push_down_filter), feeding a filter - // that references all three marks. let plan = LogicalPlanBuilder::from(person) .join_on( aliased_scan("s1", "__correlated_sq_1")?, diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 72810b29d7cd4..504c8062a00a5 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -1713,17 +1713,11 @@ logical_plan 21)----------Projection: column1 AS v 22)------------Values: (Int64(5)), (Int64(NULL)) -# Regression test for https://github.com/apache/datafusion/issues/22477 -# `= ANY` / `<> ALL` over the SAME table decorrelate into several stacked -# LeftMark joins. After push_down_filter the join filters reference no right -# column, so the mark joins are filter-less. optimize_projections must not prune -# each mark join's right child to zero columns (which would make the synthetic -# `mark` unqualified and collide across the stack). Previously failed planning -# with "Schema contains duplicate unqualified field name mark". +# same-table `= ANY` / `<> ALL` must plan without +# "duplicate unqualified field name mark". statement ok create table set_cmp_self(id int, age int) as values (1, 20), (2, 30), (3, 40); -# every row's age is in {20, 30, 40}, so `= ANY` holds for all rows query I rowsort select id from set_cmp_self where age = any(select age from set_cmp_self); ---- @@ -1731,7 +1725,6 @@ select id from set_cmp_self where age = any(select age from set_cmp_self); 2 3 -# each row's age equals itself in the set, so `<> ALL` is false for every row query I select id from set_cmp_self where age <> all(select age from set_cmp_self); ---- From 9af3387fe3448caf0b4038d356f3d2ac57ae15c0 Mon Sep 17 00:00:00 2001 From: HairstonE Date: Mon, 15 Jun 2026 11:04:30 -0400 Subject: [PATCH 4/4] Ensure an empty mark join still has a column to qualify mark, no longer unconditional addition of that column --- .../optimizer/src/optimize_projections/mod.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index f96a209e31c65..80aceb8cad44c 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -386,10 +386,20 @@ fn optimize_projections( let right_len = join.right.schema().fields().len(); let (left_req_indices, right_req_indices) = split_join_requirements(left_len, right_len, indices, &join.join_type); - let left_indices = + let mut left_indices = left_req_indices.with_plan_exprs(&plan, join.left.schema())?; - let right_indices = + let mut right_indices = right_req_indices.with_plan_exprs(&plan, join.right.schema())?; + // Ensure an empty mark join still has a column to qualify mark + match join.join_type { + JoinType::LeftMark if right_indices.indices().is_empty() => { + right_indices = right_indices.append(&[0]); + } + JoinType::RightMark if left_indices.indices().is_empty() => { + left_indices = left_indices.append(&[0]); + } + _ => {} + } // Joins benefit from "small" input tables (lower memory usage). // Therefore, each child benefits from projection: vec![ @@ -764,12 +774,12 @@ fn split_join_requirements( // The mark column is synthetic (produced by the join itself), // so discard it and route only to the left child. let (left_indices, _mark) = indices.split_off(left_len); - (left_indices, RequiredIndices::new().append(&[0])) + (left_indices, RequiredIndices::new()) } JoinType::RightMark => { // Same as LeftMark, but for the right child. let (right_indices, _mark) = indices.split_off(right_len); - (RequiredIndices::new().append(&[0]), right_indices) + (RequiredIndices::new(), right_indices) } // All requirements can be re-routed to left child directly. JoinType::LeftAnti | JoinType::LeftSemi => (indices, RequiredIndices::new()),