From e1923aec9042748c3a1c64e6cc0ec5c2305c2b90 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 16 Jul 2026 09:53:39 -0600 Subject: [PATCH] fix: preserve null_equality so distributed INTERSECT/EXCEPT match NULLs `INTERSECT` and `EXCEPT` lower to semi/anti joins carrying `NullEquality::NullEqualsNull` so that NULL matches NULL. Ballista sends the client's logical plan to the scheduler as protobuf, and datafusion-proto's logical `JoinNode` decode rebuilt the join through `LogicalPlanBuilder`, whose `join_with_expr_keys` / `join_using` hardcode `NullEqualsNothing`. The encoded `null_equality` was therefore dropped on decode and the scheduler planned a different query than the client asked for: NULL stopped matching itself, so INTERSECT dropped NULL rows and EXCEPT retained them. This reproduces with a single partition, so it is not a shuffle or join-strategy artifact, and it affects both hash and sort-merge joins: select count(*) from (select distinct c_last_name from customer intersect select distinct c_last_name from customer); -- 4972, expected 4973 Fixed upstream by apache/datafusion#22104 and backported to `branch-54` as apache/datafusion#22785, which is not in the released 54.0.0 tag. Pin DataFusion to the `branch-54` tip to pick it up; the pin can be dropped once a 54.x patch release containing it is published. TPC-DS q38 (INTERSECT) and q87 (EXCEPT) now match single-process DataFusion and join the correctness gate, taking it from 86 to 88 queries. q4 and q78 remain skipped: neither uses a set operation and both are unaffected by this fix, so they have a separate root cause. --- Cargo.lock | 233 +++++++++------------------------ Cargo.toml | 45 ++++++- ballista/core/src/serde/mod.rs | 56 ++++++++ benchmarks/src/bin/tpcds.rs | 9 +- 4 files changed, 162 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc3b3181f5..0e0ffc70d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -123,7 +123,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2169,8 +2169,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-schema", @@ -2223,8 +2222,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2248,8 +2246,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2271,8 +2268,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a9533845ab888508718c2ad540cff205d1111525a9952def8b0e584b6453784" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2299,8 +2295,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-ipc", @@ -2326,8 +2321,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "futures", "log", @@ -2337,8 +2331,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-compression", @@ -2373,8 +2366,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-ipc", @@ -2397,8 +2389,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-avro", @@ -2416,8 +2407,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2439,8 +2429,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2462,8 +2451,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2493,14 +2481,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" [[package]] name = "datafusion-execution" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-buffer", @@ -2522,8 +2508,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-schema", @@ -2545,8 +2530,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2557,8 +2541,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-buffer", @@ -2589,8 +2572,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2610,8 +2592,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2622,8 +2603,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-ord", @@ -2647,8 +2627,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "async-trait", @@ -2663,8 +2642,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2680,8 +2658,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2690,8 +2667,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "datafusion-doc", "quote", @@ -2701,8 +2677,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "chrono", @@ -2721,8 +2696,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2743,8 +2717,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2758,8 +2731,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "chrono", @@ -2775,8 +2747,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2794,8 +2765,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "arrow-data", @@ -2827,8 +2797,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "chrono", @@ -2854,8 +2823,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2865,8 +2833,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "datafusion-common", @@ -2881,8 +2848,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "async-trait", "datafusion-common", @@ -2895,8 +2861,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "bigdecimal", @@ -2925,8 +2890,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "arrow", "bigdecimal", @@ -2944,8 +2908,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2" +source = "git+https://github.com/apache/datafusion.git?rev=2142d5b2a737fa01836906d2526265035964ac99#2142d5b2a737fa01836906d2526265035964ac99" dependencies = [ "async-recursion", "async-trait", @@ -3175,7 +3138,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5396,7 +5359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.12.1", "log", "multimap", "petgraph", @@ -5417,7 +5380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.117", @@ -5545,7 +5508,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6076,7 +6039,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6135,7 +6098,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6604,7 +6567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6645,7 +6608,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6856,10 +6819,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6872,7 +6835,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7970,7 +7933,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -8097,7 +8060,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -8106,16 +8069,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -8133,31 +8087,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -8175,96 +8112,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 5719b9b16c..e31b1b55b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,4 +115,47 @@ codegen-units = 256 lto = false debug = false debug-assertions = false -incremental = false \ No newline at end of file +incremental = false + +# Temporarily pin DataFusion to the tip of `branch-54` to pick up the +# logical JoinNode proto round-trip fix (apache/datafusion#22785, backport of +# #22104), which is on branch-54 but not in the released 54.0.0 tag. +# Remove once a 54.x patch release containing it is published. +[patch.crates-io] +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-catalog = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-catalog-listing = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-common-runtime = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-datasource-arrow = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-datasource-avro = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-datasource-csv = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-datasource-json = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-datasource-parquet = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-doc = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions-table = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions-window = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-functions-window-common = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-macros = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-optimizer = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-physical-optimizer = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-pruning = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-session = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } +datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "2142d5b2a737fa01836906d2526265035964ac99" } diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index 9c19585151..bf88f77794 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -1234,4 +1234,60 @@ mod test { assert_eq!(decoded_exec.upstream_partition_count, 4); assert_eq!(decoded_exec.partition.len(), 1); } + + /// `INTERSECT` / `EXCEPT` lower to semi/anti joins with + /// `NullEquality::NullEqualsNull` so that NULL matches NULL. Ballista ships + /// the client's *logical* plan to the scheduler as protobuf, so that flag + /// has to survive the roundtrip: if it decodes back as `NullEqualsNothing` + /// the scheduler silently plans a different query than the client asked for + /// (NULL stops matching itself, so INTERSECT drops rows and EXCEPT keeps + /// them). See https://github.com/apache/datafusion-ballista/issues/2046 + /// + /// `null_equality` is not part of the rendered plan, so this inspects the + /// decoded `Join` directly rather than asserting on a plan string. + #[tokio::test] + async fn logical_join_roundtrip_preserves_null_equality() { + use datafusion::common::NullEquality; + use datafusion::logical_expr::LogicalPlan; + + fn join_null_equality(plan: &LogicalPlan) -> Option { + if let LogicalPlan::Join(join) = plan { + return Some(join.null_equality); + } + plan.inputs().into_iter().find_map(join_null_equality) + } + + let ctx = SessionContext::new(); + ctx.register_parquet( + "t", + "../../examples/testdata/alltypes_plain.parquet", + datafusion::prelude::ParquetReadOptions::default(), + ) + .await + .unwrap(); + + let plan = ctx + .state() + .create_logical_plan( + "select string_col from t intersect select string_col from t", + ) + .await + .unwrap(); + let plan = ctx.state().optimize(&plan).unwrap(); + assert_eq!( + join_null_equality(&plan), + Some(NullEquality::NullEqualsNull), + "precondition: INTERSECT should plan a NullEqualsNull join" + ); + + let codec = BallistaLogicalExtensionCodec::default(); + let node = LogicalPlanNode::try_from_logical_plan(&plan, &codec).unwrap(); + let roundtripped = node.try_into_logical_plan(&ctx.task_ctx(), &codec).unwrap(); + + assert_eq!( + join_null_equality(&roundtripped), + Some(NullEquality::NullEqualsNull), + "null_equality must survive the logical plan protobuf roundtrip" + ); + } } diff --git a/benchmarks/src/bin/tpcds.rs b/benchmarks/src/bin/tpcds.rs index ca95b61483..9a13538bed 100644 --- a/benchmarks/src/bin/tpcds.rs +++ b/benchmarks/src/bin/tpcds.rs @@ -64,23 +64,16 @@ const TABLES: &[&str] = &[ const SKIP: &[(usize, &str)] = &[ // Distributed execution diverges from single-process DataFusion on the same // data (confirmed reproducible; DataFusion is correct under both join modes). + // Neither query uses a set operation; the root cause is unexplained. // See https://github.com/apache/datafusion-ballista/issues/2046 ( 4, "distributed result diverges from DataFusion (issue #2046)", ), - ( - 38, - "distributed INTERSECT diverges from DataFusion (issue #2046)", - ), ( 78, "distributed LIMIT drops rows vs DataFusion (issue #2046)", ), - ( - 87, - "distributed EXCEPT diverges from DataFusion (issue #2046)", - ), // Non-deterministic: LIMIT/ORDER BY ties without a total order make the // result vary run-to-run in both engines, so a row-by-row diff is unstable. (2, "non-deterministic (ORDER BY ties; varies run-to-run)"),