Skip to content

Commit 76897f2

Browse files
committed
Avoid possibly expensive string formatting if no error is encountered
1 parent fb8fe7a commit 76897f2

1 file changed

Lines changed: 44 additions & 1 deletion

File tree

  • datafusion/datasource/src/file_scan_config

datafusion/datasource/src/file_scan_config/mod.rs

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ use datafusion_execution::{
3939
use datafusion_expr::Operator;
4040

4141
use crate::source::OpenArgs;
42+
use datafusion_common::stats::Precision;
4243
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
4344
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
4445
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -1225,7 +1226,9 @@ impl FileScanConfig {
12251226
/// we can't guarantee the statistics are exact because we don't know how many
12261227
/// rows will be filtered out.
12271228
pub fn statistics(&self) -> Statistics {
1228-
if self.file_source.filter().is_some() {
1229+
let filter_may_change_row_count = self.file_source.filter().is_some()
1230+
&& self.statistics.num_rows != Precision::Exact(0);
1231+
if filter_may_change_row_count {
12291232
self.statistics.clone().to_inexact()
12301233
} else {
12311234
self.statistics.clone()
@@ -2480,6 +2483,46 @@ mod tests {
24802483
assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));
24812484
}
24822485

2486+
#[test]
2487+
fn test_partition_statistics_filter() {
2488+
assert_num_rows_with_filter(Precision::Absent, Precision::Absent);
2489+
assert_num_rows_with_filter(Precision::Exact(100), Precision::Inexact(100));
2490+
assert_num_rows_with_filter(Precision::Inexact(100), Precision::Inexact(100));
2491+
assert_num_rows_with_filter(Precision::Exact(0), Precision::Exact(0));
2492+
2493+
/// Creates a file scan with filter and checks the output num_rows stats, given the input
2494+
/// num_rows stats.
2495+
fn assert_num_rows_with_filter(
2496+
input_num_rows: Precision<usize>,
2497+
expected_num_rows: Precision<usize>,
2498+
) {
2499+
// Create a schema with 4 columns
2500+
let schema = Arc::new(Schema::new(vec![Field::new(
2501+
"col0",
2502+
DataType::Int32,
2503+
false,
2504+
)]));
2505+
2506+
let stats =
2507+
Statistics::new_unknown(schema.as_ref()).with_num_rows(input_num_rows);
2508+
let file_group =
2509+
FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]);
2510+
2511+
let table_schema = TableSchema::from(&schema);
2512+
let config = FileScanConfigBuilder::new(
2513+
ObjectStoreUrl::parse("test:///").unwrap(),
2514+
Arc::new(MockSource::new(table_schema.clone()).with_filter(Arc::new(
2515+
Literal::new(ScalarValue::Boolean(Some(true))),
2516+
))),
2517+
)
2518+
.with_file_groups(vec![file_group])
2519+
.with_statistics(stats)
2520+
.build();
2521+
2522+
assert_eq!(config.statistics().num_rows, expected_num_rows,);
2523+
}
2524+
}
2525+
24832526
/// Regression test for reusing a `DataSourceExec` after its execution-local
24842527
/// shared work queue has been drained.
24852528
///

0 commit comments

Comments
 (0)