Skip to content

refactor: make join projection pushdown schema-aware via ColumnIndex/… #23185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Phoenix500526 wants to merge 6 commits into
base: main
Choose a base branch
from
Open

refactor: make join projection pushdown schema-aware via ColumnIndex/… #23185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dca264a
refactor: make join projection pushdown schema-aware via ColumnIndex/…
Phoenix500526 Jun 25, 2026
03fc98a
refactor: make try_pushdown_through_join pub(crate)
Phoenix500526 Jun 25, 2026
5bfbeba
refactor: avoid public join column API
Phoenix500526 Jun 29, 2026
a62b5c6
refactor(physical-plan): preserve join pushdown API
Phoenix500526 Jul 3, 2026
e9db144
refactor(join): consolidate projection pushdown
Phoenix500526 Jul 11, 2026
1ff782e
refactor(join): deprecate positional pushdown
Phoenix500526 Jul 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 15 additions & 17 deletions datafusion/physical-plan/src/joins/hash_join/exec.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ use crate::joins::{JoinOn, JoinOnRef, PartitionMode, SharedBitmapBuilder};
use crate::metrics::{Count, MetricBuilder, MetricCategory};
use crate::projection::{
EmbeddedProjection, JoinData, ProjectionExec, try_embed_projection,
try_pushdown_through_join,
try_pushdown_through_join_with_column_indices,
};
use crate::repartition::REPARTITION_RANDOM_STATE;
use crate::statistics::StatisticsArgs;
Expand Down Expand Up @@ -1557,23 +1557,21 @@ impl ExecutionPlan for HashJoinExec {
return Ok(None);
}

// TODO: split by `col`/`JoinSide` instead so mark joins can also push down to children.
let schema = self.schema();
if !matches!(self.join_type(), JoinType::LeftMark | JoinType::RightMark)
&& let Some(JoinData {
projected_left_child,
projected_right_child,
join_filter,
join_on,
}) = try_pushdown_through_join(

@alamb alamb Jul 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we still need try_pushdown_through_join after this PR? Perhaps we should remove it (or migrate other sites over that still use it)?

projection,
self.left(),
self.right(),
self.on(),
&schema,
self.filter(),
)?
{
if let Some(JoinData {
projected_left_child,
projected_right_child,
join_filter,
join_on,
}) = try_pushdown_through_join_with_column_indices(
projection,
self.left(),
self.right(),
self.on(),
&schema,
self.filter(),
self.column_indices.as_slice(),
)? {
self.builder()
.with_new_children(vec![
Arc::new(projected_left_child),
Expand Down
32 changes: 15 additions & 17 deletions datafusion/physical-plan/src/joins/nested_loop_join.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ use crate::metrics::{
};
use crate::projection::{
EmbeddedProjection, JoinData, ProjectionExec, try_embed_projection,
try_pushdown_through_join,
try_pushdown_through_join_with_column_indices,
};
use crate::statistics::StatisticsArgs;
use crate::{
Expand Down Expand Up @@ -733,23 +733,21 @@ impl ExecutionPlan for NestedLoopJoinExec {
return Ok(None);
}

// TODO: split by `col`/`JoinSide` instead so mark joins can also push down to children.
let schema = self.schema();
if !matches!(self.join_type(), JoinType::LeftMark | JoinType::RightMark)
&& let Some(JoinData {
projected_left_child,
projected_right_child,
join_filter,
..
}) = try_pushdown_through_join(
projection,
self.left(),
self.right(),
&[],
&schema,
self.filter(),
)?
{
if let Some(JoinData {
projected_left_child,
projected_right_child,
join_filter,
..
}) = try_pushdown_through_join_with_column_indices(
projection,
self.left(),
self.right(),
&[],
&schema,
self.filter(),
self.column_indices.as_slice(),
)? {
Ok(Some(Arc::new(NestedLoopJoinExec::try_new(
Arc::new(projected_left_child),
Arc::new(projected_right_child),
Expand Down
92 changes: 29 additions & 63 deletions datafusion/physical-plan/src/joins/symmetric_hash_join.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,7 @@ use crate::joins::utils::{
matchable_join_keys, symmetric_join_output_partitioning, update_hash,
};
use crate::projection::{
ProjectionExec, join_allows_pushdown, join_table_borders, new_join_children,
physical_to_column_exprs, update_join_filter, update_join_on,
JoinData, ProjectionExec, try_pushdown_through_join_with_column_indices,
};
use crate::stream::EmptyRecordBatchStream;
use crate::{
Expand Down Expand Up @@ -589,69 +588,36 @@ impl ExecutionPlan for SymmetricHashJoinExec {
&self,
projection: &ProjectionExec,
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
// Convert projected PhysicalExpr's to columns. If not possible, we cannot proceed.
let Some(projection_as_columns) = physical_to_column_exprs(projection.expr())
else {
return Ok(None);
};

let (far_right_left_col_ind, far_left_right_col_ind) = join_table_borders(
self.left().schema().fields().len(),
&projection_as_columns,
);

if !join_allows_pushdown(
&projection_as_columns,
&self.schema(),
far_right_left_col_ind,
far_left_right_col_ind,
) {
return Ok(None);
}

let Some(new_on) = update_join_on(
&projection_as_columns[0..=far_right_left_col_ind as _],
&projection_as_columns[far_left_right_col_ind as _..],
self.on(),
self.left().schema().fields().len(),
) else {
return Ok(None);
};

let new_filter = if let Some(filter) = self.filter() {
match update_join_filter(
&projection_as_columns[0..=far_right_left_col_ind as _],
&projection_as_columns[far_left_right_col_ind as _..],
filter,
self.left().schema().fields().len(),
) {
Some(updated_filter) => Some(updated_filter),
None => return Ok(None),
}
} else {
None
};

let (new_left, new_right) = new_join_children(
&projection_as_columns,
far_right_left_col_ind,
far_left_right_col_ind,
let schema = self.schema();
if let Some(JoinData {
projected_left_child,
projected_right_child,
join_filter,
join_on,
}) = try_pushdown_through_join_with_column_indices(
projection,
self.left(),
self.right(),
)?;

SymmetricHashJoinExec::try_new(
Arc::new(new_left),
Arc::new(new_right),
new_on,
new_filter,
self.join_type(),
self.null_equality(),
self.right().output_ordering().cloned(),
self.left().output_ordering().cloned(),
self.partition_mode(),
)
.map(|e| Some(Arc::new(e) as _))
self.on(),
&schema,
self.filter(),
self.column_indices.as_slice(),
)? {
SymmetricHashJoinExec::try_new(
Arc::new(projected_left_child),
Arc::new(projected_right_child),
join_on,
join_filter,
self.join_type(),
self.null_equality(),
self.right().output_ordering().cloned(),
self.left().output_ordering().cloned(),
self.partition_mode(),
)
.map(|e| Some(Arc::new(e) as _))
} else {
Ok(None)
}
}
}

Expand Down
Loading
Loading