From 697973f95c3af0410f0291a734de5da6a74d190a Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 11 Jun 2026 16:54:03 +0100 Subject: [PATCH 1/3] Show if a join is null_aware in DisplayAs Signed-off-by: Adam Gutglick --- datafusion/expr/src/logical_plan/plan.rs | 7 ++++++- datafusion/physical-plan/src/joins/hash_join/exec.rs | 9 ++++++++- .../sqllogictest/test_files/null_aware_anti_join.slt | 8 ++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 3608c81878d17..8dbf41c37f4d1 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -2073,6 +2073,7 @@ impl LogicalPlan { filter, join_constraint, join_type, + null_aware, .. }) => { let join_expr: Vec = @@ -2081,6 +2082,8 @@ impl LogicalPlan { .as_ref() .map(|expr| format!(" Filter: {expr}")) .unwrap_or_else(|| "".to_string()); + let null_aware_expr = + if *null_aware { " null_aware" } else { "" }; let join_type = if filter.is_none() && keys.is_empty() && *join_type == JoinType::Inner @@ -2100,15 +2103,17 @@ impl LogicalPlan { filter_expr )?; } + write!(f, "{null_aware_expr}")?; Ok(()) } JoinConstraint::Using => { write!( f, - "{} Join: Using {}{}", + "{} Join: Using {}{}{}", join_type, join_expr.join(", "), filter_expr, + null_aware_expr, ) } } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 7cddae276f5fa..6e73c4d2e0157 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -1140,6 +1140,8 @@ impl DisplayAs for HashJoinExec { let display_fetch = self .fetch .map_or_else(String::new, |f| format!(", fetch={f}")); + let display_null_aware = + if self.null_aware { ", null_aware" } else { "" }; let on = self .on .iter() @@ -1148,7 +1150,7 @@ impl DisplayAs for HashJoinExec { .join(", "); write!( f, - "HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}", + "HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}{}", self.mode, self.join_type, on, @@ -1156,6 +1158,7 @@ impl DisplayAs for HashJoinExec { display_projections, display_null_equality, display_fetch, + display_null_aware, ) } DisplayFormatType::TreeRender => { @@ -1178,6 +1181,10 @@ impl DisplayAs for HashJoinExec { writeln!(f, "NullsEqual: true")?; } + if self.null_aware { + writeln!(f, "null_aware")?; + } + if let Some(filter) = self.filter.as_ref() { writeln!(f, "filter={filter}")?; } diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index 5907a85a9b923..b18f3b3ae7a99 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -53,12 +53,12 @@ query TT EXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_no_null); ---- logical_plan -01)LeftAnti Join: outer_table.id = __correlated_sq_1.id +01)LeftAnti Join: outer_table.id = __correlated_sq_1.id null_aware 02)--TableScan: outer_table projection=[id, value] 03)--SubqueryAlias: __correlated_sq_1 04)----TableScan: inner_table_no_null projection=[id] physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)] +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware 02)--DataSourceExec: partitions=1, partition_sizes=[1] 03)--DataSourceExec: partitions=1, partition_sizes=[1] @@ -193,12 +193,12 @@ query TT EXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null); ---- logical_plan -01)LeftAnti Join: outer_table.id = __correlated_sq_1.id +01)LeftAnti Join: outer_table.id = __correlated_sq_1.id null_aware 02)--TableScan: outer_table projection=[id, value] 03)--SubqueryAlias: __correlated_sq_1 04)----TableScan: inner_table_with_null projection=[id] physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)] +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware 02)--DataSourceExec: partitions=1, partition_sizes=[1] 03)--DataSourceExec: partitions=1, partition_sizes=[1] From e5e2aeef1c4f7c73df3eba79b5fd6bda0cd7fd2a Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 11 Jun 2026 18:58:48 +0100 Subject: [PATCH 2/3] Fix tests --- .../test_files/dynamic_filter_pushdown_config.slt | 8 ++++---- datafusion/sqllogictest/test_files/joins.slt | 2 +- .../sqllogictest/test_files/tpch/plans/q16.slt.part | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt index e779ce2cbffb0..e436ca795208d 100644 --- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt +++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt @@ -374,14 +374,14 @@ FROM left_parquet l WHERE l.id NOT IN (SELECT r.id FROM right_parquet r); ---- logical_plan -01)LeftAnti Join: l.id = __correlated_sq_1.id +01)LeftAnti Join: l.id = __correlated_sq_1.id null_aware 02)--SubqueryAlias: l 03)----TableScan: left_parquet projection=[id, data] 04)--SubqueryAlias: __correlated_sq_1 05)----SubqueryAlias: r 06)------TableScan: right_parquet projection=[id] physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)] +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet 03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] @@ -469,7 +469,7 @@ ORDER BY l.id LIMIT 2; ---- logical_plan 01)Sort: l.id ASC NULLS LAST, fetch=2 -02)--LeftAnti Join: l.id = __correlated_sq_1.id +02)--LeftAnti Join: l.id = __correlated_sq_1.id null_aware 03)----SubqueryAlias: l 04)------TableScan: left_parquet projection=[id, data] 05)----SubqueryAlias: __correlated_sq_1 @@ -477,7 +477,7 @@ logical_plan 07)--------TableScan: right_parquet projection=[id] physical_plan 01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] -02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)] +02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] 04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt index 9be1d39d63605..082b10167274c 100644 --- a/datafusion/sqllogictest/test_files/joins.slt +++ b/datafusion/sqllogictest/test_files/joins.slt @@ -1989,7 +1989,7 @@ where join_t1.t1_id + 12 not in (select join_t2.t2_id + 1 from join_t2 where join_t1.t1_int > 0) ---- logical_plan -01)LeftAnti Join: CAST(join_t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.join_t2.t2_id + Int64(1) Filter: join_t1.t1_int > UInt32(0) +01)LeftAnti Join: CAST(join_t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.join_t2.t2_id + Int64(1) Filter: join_t1.t1_int > UInt32(0) null_aware 02)--TableScan: join_t1 projection=[t1_id, t1_name, t1_int] 03)--SubqueryAlias: __correlated_sq_1 04)----Projection: CAST(join_t2.t2_id AS Int64) + Int64(1) diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part index 8d8eb0ed11828..ab830714b1dde 100644 --- a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part +++ b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part @@ -54,7 +54,7 @@ logical_plan 02)--Projection: part.p_brand, part.p_type, part.p_size, count(alias1) AS supplier_cnt 03)----Aggregate: groupBy=[[part.p_brand, part.p_type, part.p_size]], aggr=[[count(alias1)]] 04)------Aggregate: groupBy=[[part.p_brand, part.p_type, part.p_size, partsupp.ps_suppkey AS alias1]], aggr=[[]] -05)--------LeftAnti Join: partsupp.ps_suppkey = __correlated_sq_1.s_suppkey +05)--------LeftAnti Join: partsupp.ps_suppkey = __correlated_sq_1.s_suppkey null_aware 06)----------Projection: partsupp.ps_suppkey, part.p_brand, part.p_type, part.p_size 07)------------Inner Join: partsupp.ps_partkey = part.p_partkey 08)--------------TableScan: partsupp projection=[ps_partkey, ps_suppkey] @@ -74,7 +74,7 @@ physical_plan 07)------------AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[] 08)--------------RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 4), input_partitions=4 09)----------------AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] -10)------------------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)] +10)------------------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)], null_aware 11)--------------------CoalescePartitionsExec 12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_partkey@0, p_partkey@0)], projection=[ps_suppkey@1, p_brand@3, p_type@4, p_size@5] 13)------------------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4 From e4e7bf32f309a9455b6349803a2c48eb0fc8750a Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 12 Jun 2026 18:13:59 +0100 Subject: [PATCH 3/3] tree explain test Signed-off-by: Adam Gutglick --- .../sqllogictest/test_files/explain_tree.slt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index d8e90e294f8a3..8588c0e7ba2ae 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -1100,6 +1100,31 @@ physical_plan 24)-----------------------------│ format: csv │ 25)-----------------------------└───────────────────────────┘ +# Query with null-aware anti join (NOT IN subquery). +query TT +explain select int_col from table1 where int_col not in (select int_col from table2); +---- +physical_plan +01)┌───────────────────────────┐ +02)│ HashJoinExec │ +03)│ -------------------- │ +04)│ join_type: LeftAnti │ +05)│ │ +06)│ null_aware ├──────────────┐ +07)│ │ │ +08)│ on: │ │ +09)│ (int_col = int_col) │ │ +10)└─────────────┬─────────────┘ │ +11)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +12)│ DataSourceExec ││ DataSourceExec │ +13)│ -------------------- ││ -------------------- │ +14)│ files: 1 ││ files: 1 │ +15)│ format: csv ││ format: parquet │ +16)│ ││ │ +17)│ ││ predicate: │ +18)│ ││ DynamicFilter [ empty ] │ +19)└───────────────────────────┘└───────────────────────────┘ + # Query with nested loop join. query TT explain select int_col from table1 where exists (select count(*) from table2);