From 063db3db3754c355025108f079c525db3f22e5fe Mon Sep 17 00:00:00 2001 From: XL Liang Date: Sun, 26 Jul 2026 19:02:48 +0800 Subject: [PATCH 1/2] chore: bump datafusion 54.1.0 --- Cargo.lock | 192 +++++++++--------- Cargo.toml | 26 +-- .../proto/sail/plan/physical.proto | 1 + crates/sail-execution/src/proto/codec.rs | 45 ++++ .../src/scalar/array/spark_array_compact.rs | 3 +- .../spark/__snapshots__/test_tpcds.plan.yaml | 55 +++-- .../function/features/array_compact.feature | 9 + .../function/features/width_bucket.feature | 15 ++ 8 files changed, 213 insertions(+), 133 deletions(-) create mode 100644 python/pysail/tests/spark/function/features/width_bucket.feature diff --git a/Cargo.lock b/Cargo.lock index f8f68f5ce2..260af0feef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -155,7 +155,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1884,9 +1884,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -2044,9 +2044,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", @@ -2099,9 +2099,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -2124,9 +2124,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -2147,9 +2147,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ "arrow", "arrow-ipc", @@ -2173,9 +2173,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -2184,9 +2184,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-compression", @@ -2220,9 +2220,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -2244,9 +2244,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" dependencies = [ "arrow", "arrow-avro", @@ -2263,9 +2263,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -2286,9 +2286,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2309,9 +2309,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ "arrow", "async-trait", @@ -2340,15 +2340,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", @@ -2368,9 +2368,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", "arrow-schema", @@ -2391,9 +2391,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", @@ -2403,9 +2403,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -2435,9 +2435,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ "arrow", "datafusion-common", @@ -2456,9 +2456,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ "arrow", "datafusion-common", @@ -2468,9 +2468,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2493,9 +2493,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", @@ -2509,9 +2509,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2526,9 +2526,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2536,9 +2536,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", @@ -2547,9 +2547,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2567,9 +2567,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ "arrow", "datafusion-common", @@ -2589,9 +2589,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2604,9 +2604,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ "arrow", "chrono", @@ -2621,9 +2621,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2640,9 +2640,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ "arrow", "arrow-data", @@ -2673,9 +2673,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" dependencies = [ "arrow", "chrono", @@ -2700,9 +2700,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" dependencies = [ "arrow", "datafusion-common", @@ -2711,9 +2711,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2727,9 +2727,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2741,9 +2741,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" dependencies = [ "arrow", "bigdecimal", @@ -2771,9 +2771,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2892,7 +2892,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3057,7 +3057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3473,11 +3473,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -3487,9 +3485,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -4605,7 +4605,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5381,7 +5381,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6642,14 +6642,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash 2.1.2", "rustls", @@ -6809,6 +6810,15 @@ dependencies = [ "rand 0.10.2", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -7254,7 +7264,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8346,7 +8356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8654,7 +8664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8734,7 +8744,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8921,10 +8931,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -9987,7 +9997,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1d96fb7e7b..afc0a871a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -150,7 +150,7 @@ prost-types = "0.14" # The `axum` version must match the one used in `tonic` (replace `RELEASE` with the release we are using): # https://github.com/hyperium/tonic/blob/vRELEASE/tonic/Cargo.toml axum = "0.8.9" -datafusion = { version = "54.0.0", default-features = false, features = [ +datafusion = { version = "54.1.0", default-features = false, features = [ "nested_expressions", "crypto_expressions", "datetime_expressions", @@ -164,18 +164,18 @@ datafusion = { version = "54.0.0", default-features = false, features = [ "avro", "recursive_protection", ] } -datafusion-common = { version = "54.0.0", features = ["object_store"] } -datafusion-datasource = { version = "54.0.0" } -datafusion-datasource-avro = { version = "54.0.0" } -datafusion-datasource-json = { version = "54.0.0" } -datafusion-expr = { version = "54.0.0", default-features = false } -datafusion-expr-common = { version = "54.0.0" } -datafusion-proto = { version = "54.0.0" } -datafusion-functions = { version = "54.0.0" } -datafusion-functions-nested = { version = "54.0.0", default-features = false } -datafusion-physical-expr = { version = "54.0.0" } -datafusion-session = { version = "54.0.0" } -datafusion-spark = { version = "54.0.0", features = ["core"] } +datafusion-common = { version = "54.1.0", features = ["object_store"] } +datafusion-datasource = { version = "54.1.0" } +datafusion-datasource-avro = { version = "54.1.0" } +datafusion-datasource-json = { version = "54.1.0" } +datafusion-expr = { version = "54.1.0", default-features = false } +datafusion-expr-common = { version = "54.1.0" } +datafusion-proto = { version = "54.1.0" } +datafusion-functions = { version = "54.1.0" } +datafusion-functions-nested = { version = "54.1.0", default-features = false } +datafusion-physical-expr = { version = "54.1.0" } +datafusion-session = { version = "54.1.0" } +datafusion-spark = { version = "54.1.0", features = ["core"] } pyo3 = { version = "0.29.0", features = ["serde"] } jiter = { version = "0.15.0", default-features = false } arrow = { version = "58.3.0", features = ["chrono-tz"] } diff --git a/crates/sail-execution/proto/sail/plan/physical.proto b/crates/sail-execution/proto/sail/plan/physical.proto index 3dadbe8ba3..8d9b3c9553 100644 --- a/crates/sail-execution/proto/sail/plan/physical.proto +++ b/crates/sail-execution/proto/sail/plan/physical.proto @@ -649,6 +649,7 @@ message RecursiveQueryExecNode { bytes static_term = 2; bytes recursive_term = 3; bool is_distinct = 4; + bytes output_schema = 5; } message SortMergeJoinExecNode { diff --git a/crates/sail-execution/src/proto/codec.rs b/crates/sail-execution/src/proto/codec.rs index c572a7c846..30825ea228 100644 --- a/crates/sail-execution/src/proto/codec.rs +++ b/crates/sail-execution/src/proto/codec.rs @@ -614,11 +614,14 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { static_term, recursive_term, is_distinct, + output_schema, }) => { let static_term = try_decode_physical_plan(ctx, self, &static_term)?; let recursive_term = try_decode_physical_plan(ctx, self, &recursive_term)?; + let output_schema = Arc::new(try_decode_schema(&output_schema)?); Ok(Arc::new(RecursiveQueryExec::try_new( name, + output_schema, static_term, recursive_term, is_distinct, @@ -1518,11 +1521,13 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let recursive_term = try_encode_physical_plan(self, recursive_query.recursive_term().clone())?; let is_distinct = recursive_query.is_distinct(); + let output_schema = try_encode_schema(recursive_query.schema().as_ref())?; NodeKind::RecursiveQuery(r#gen::RecursiveQueryExecNode { name, static_term, recursive_term, is_distinct, + output_schema, }) } else if let Some(sort_merge_join) = node.downcast_ref::() { let left = try_encode_physical_plan(self, sort_merge_join.left().clone())?; @@ -4373,6 +4378,46 @@ mod tests { codec.try_decode_udwf(&name, &buf) } + #[test] + fn test_round_trip_recursive_query_preserves_output_schema() -> Result<()> { + use datafusion::arrow::datatypes::{DataType, Field}; + use datafusion::physical_plan::empty::EmptyExec; + + let static_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let output_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + true, + )])); + let static_term: Arc = + Arc::new(EmptyExec::new(Arc::clone(&static_schema))); + let recursive_term: Arc = + Arc::new(EmptyExec::new(Arc::clone(&output_schema))); + let plan = Arc::new(RecursiveQueryExec::try_new( + "numbers".to_string(), + Arc::clone(&output_schema), + static_term, + recursive_term, + false, + )?); + + let codec = RemoteExecutionCodec; + let bytes = try_encode_physical_plan(&codec, plan)?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let recursive_query = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a RecursiveQueryExec"))?; + + assert_eq!(recursive_query.schema(), output_schema); + assert_eq!(recursive_query.static_term().schema(), output_schema); + assert_eq!(recursive_query.recursive_term().schema(), output_schema); + Ok(()) + } + #[test] fn test_round_trip_spark_variant_explode_helper_udf() -> Result<()> { let decoded = round_trip_udf(ScalarUDF::from(SparkVariantExplodeUdf::new()))?; diff --git a/crates/sail-function/src/scalar/array/spark_array_compact.rs b/crates/sail-function/src/scalar/array/spark_array_compact.rs index 435539d189..54a490ef62 100644 --- a/crates/sail-function/src/scalar/array/spark_array_compact.rs +++ b/crates/sail-function/src/scalar/array/spark_array_compact.rs @@ -84,6 +84,7 @@ fn array_compact_inner(args: &[ArrayRef]) -> Result { fn array_compact_generic(list: &GenericListArray) -> Result { let values = list.values(); + let values_nulls = values.logical_nulls(); let offsets = list.offsets(); // Build a boolean filter mask for the flattened values array. @@ -106,7 +107,7 @@ fn array_compact_generic(list: &GenericListArray) -> Resu } else { // Non-null row: keep only non-null values. for j in start..end { - let keep_value = values.is_valid(j); + let keep_value = values_nulls.as_ref().is_none_or(|nulls| nulls.is_valid(j)); keep.push(keep_value); if keep_value { total_kept += 1; diff --git a/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml b/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml index 5a414a05c6..57e4cc0cca 100644 --- a/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml +++ b/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml @@ -342,36 +342,35 @@ data: AggregateExec: mode=FinalPartitioned, gby=[#8@0 as #8], aggr=[count(Int64(1))] RepartitionExec: partitioning=Hash([#8@0], 4), input_partitions=4 AggregateExec: mode=Partial, gby=[#8@0 as #8], aggr=[count(Int64(1))] - FilterExec: CAST(#87@1 AS Decimal128(11, 7)) > Some(12),2,1 * #155@2, projection=[#8@0] - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#94@2, #145@1)], projection=[#8@0, #87@1, #155@3] - CoalescePartitionsExec - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#82@0, #33@1)], projection=[#8@3, #87@1, #94@2] - ProjectionExec: expr=[_0@0 as #82, _5@1 as #87, _12@2 as #94] - DataSourceExec: partitions=1, partition_sizes=[1] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#31@1, CAST(d.#54 AS Float64)@1)], projection=[#8@0, #33@2] - CoalescePartitionsExec - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.#13 AS Float64)@2, #34@2)], projection=[#8@0, #31@3, #33@4] - CoalescePartitionsExec - ProjectionExec: expr=[#8@0 as #8, #13@1 as #13, CAST(#13@1 AS Float64) as CAST(c.#13 AS Float64)] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#17@1, #0@0)], projection=[#8@3, #13@0] - ProjectionExec: expr=[_0@0 as #13, _4@1 as #17] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[_0@0 as #0, _8@1 as #8] - DataSourceExec: partitions=1, partition_sizes=[1] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - ProjectionExec: expr=[_0@0 as #31, _2@1 as #33, _3@2 as #34] - DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[_0@0 as #54, CAST(_0@0 AS Float64) as CAST(d.#54 AS Float64)] - FilterExec: _3@1 = scalar_subquery(), projection=[_0@0] - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#94@2, #145@1)], filter=CAST(#87@0 AS Decimal128(11, 7)) > Some(12),2,1 * #155@1, projection=[#8@0] + CoalescePartitionsExec + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#82@0, #33@1)], projection=[#8@3, #87@1, #94@2] + ProjectionExec: expr=[_0@0 as #82, _5@1 as #87, _12@2 as #94] + DataSourceExec: partitions=1, partition_sizes=[1] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#31@1, CAST(d.#54 AS Float64)@1)], projection=[#8@0, #33@2] + CoalescePartitionsExec + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.#13 AS Float64)@2, #34@2)], projection=[#8@0, #31@3, #33@4] + CoalescePartitionsExec + ProjectionExec: expr=[#8@0 as #8, #13@1 as #13, CAST(#13@1 AS Float64) as CAST(c.#13 AS Float64)] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#17@1, #0@0)], projection=[#8@3, #13@0] + ProjectionExec: expr=[_0@0 as #13, _4@1 as #17] + DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[_0@0 as #0, _8@1 as #8] + DataSourceExec: partitions=1, partition_sizes=[1] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + ProjectionExec: expr=[_0@0 as #31, _2@1 as #33, _3@2 as #34] DataSourceExec: partitions=1, partition_sizes=[1] - ProjectionExec: expr=[avg(j.#138)@1 as #155, #145@0 as #145] - AggregateExec: mode=FinalPartitioned, gby=[#145@0 as #145], aggr=[avg(j.#138)] - RepartitionExec: partitioning=Hash([#145@0], 4), input_partitions=1 - AggregateExec: mode=Partial, gby=[#145@1 as #145], aggr=[avg(j.#138)] - ProjectionExec: expr=[_5@0 as #138, _12@1 as #145] + ProjectionExec: expr=[_0@0 as #54, CAST(_0@0 AS Float64) as CAST(d.#54 AS Float64)] + FilterExec: _3@1 = scalar_subquery(), projection=[_0@0] + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 DataSourceExec: partitions=1, partition_sizes=[1] + ProjectionExec: expr=[avg(j.#138)@1 as #155, #145@0 as #145] + AggregateExec: mode=FinalPartitioned, gby=[#145@0 as #145], aggr=[avg(j.#138)] + RepartitionExec: partitioning=Hash([#145@0], 4), input_partitions=1 + AggregateExec: mode=Partial, gby=[#145@1 as #145], aggr=[avg(j.#138)] + ProjectionExec: expr=[_5@0 as #138, _12@1 as #145] + DataSourceExec: partitions=1, partition_sizes=[1] AggregateExec: mode=FinalPartitioned, gby=[#132@0 as #132], aggr=[] RepartitionExec: partitioning=Hash([#132@0], 4), input_partitions=4 AggregateExec: mode=Partial, gby=[#132@0 as #132], aggr=[] diff --git a/python/pysail/tests/spark/function/features/array_compact.feature b/python/pysail/tests/spark/function/features/array_compact.feature index c9db3399e5..ac35a0e701 100644 --- a/python/pysail/tests/spark/function/features/array_compact.feature +++ b/python/pysail/tests/spark/function/features/array_compact.feature @@ -38,6 +38,15 @@ Feature: array_compact() removes null values from an array | result | | [] | + Scenario: array_compact with untyped null values returns empty array + When query + """ + SELECT array_compact(array(NULL, NULL)) AS result + """ + Then query result + | result | + | [] | + Scenario: array_compact with null at beginning When query """ diff --git a/python/pysail/tests/spark/function/features/width_bucket.feature b/python/pysail/tests/spark/function/features/width_bucket.feature new file mode 100644 index 0000000000..4a66c1fc97 --- /dev/null +++ b/python/pysail/tests/spark/function/features/width_bucket.feature @@ -0,0 +1,15 @@ +Feature: width_bucket() assigns values to buckets + + Scenario: width_bucket returns a Spark BIGINT result + When query + """ + SELECT width_bucket(5.0, 0.0, 10.0, 5) AS result + """ + Then query result + | result | + | 3 | + And query schema + """ + root + |-- result: long (nullable = true) + """ From 13463cc3025889bf189081aa7fe3bf4b4002a140 Mon Sep 17 00:00:00 2001 From: XL Liang Date: Tue, 28 Jul 2026 19:38:19 +0800 Subject: [PATCH 2/2] update width_bucket feature tests --- .../function/features/math/width_bucket.feature | 11 +++++------ .../spark/function/features/width_bucket.feature | 15 --------------- 2 files changed, 5 insertions(+), 21 deletions(-) delete mode 100644 python/pysail/tests/spark/function/features/width_bucket.feature diff --git a/python/pysail/tests/spark/function/features/math/width_bucket.feature b/python/pysail/tests/spark/function/features/math/width_bucket.feature index 76ce7961f5..a9e0e93787 100644 --- a/python/pysail/tests/spark/function/features/math/width_bucket.feature +++ b/python/pysail/tests/spark/function/features/math/width_bucket.feature @@ -4,7 +4,6 @@ Feature: width_bucket output schema @spark_null Rule: Output schema - @sail-bug Scenario: a non-null literal input to width_bucket yields the schema Spark declares When query """ @@ -16,7 +15,6 @@ Feature: width_bucket output schema |-- result: long (nullable = true) """ - @sail-bug Scenario: a nullable column input to width_bucket stays nullable When query """ @@ -53,7 +51,8 @@ Feature: width_bucket output schema | | Examples: - | case | args | result | - | width_bucket doctest #2 (result) | 0.0, 10.0, 0.0, 5 | 6 | - | width_bucket doctest #3 (result) | 10.0, 0.0, 10.0, 5 | 6 | - | width_bucket doctest #4 (result) | 10.0, 0.0, 0.0, 5 | NULL | + | case | args | result | + | width_bucket returns middle bucket | 5.0, 0.0, 10.0, 5 | 3 | + | width_bucket doctest #2 (result) | 0.0, 10.0, 0.0, 5 | 6 | + | width_bucket doctest #3 (result) | 10.0, 0.0, 10.0, 5 | 6 | + | width_bucket doctest #4 (result) | 10.0, 0.0, 0.0, 5 | NULL | diff --git a/python/pysail/tests/spark/function/features/width_bucket.feature b/python/pysail/tests/spark/function/features/width_bucket.feature deleted file mode 100644 index 4a66c1fc97..0000000000 --- a/python/pysail/tests/spark/function/features/width_bucket.feature +++ /dev/null @@ -1,15 +0,0 @@ -Feature: width_bucket() assigns values to buckets - - Scenario: width_bucket returns a Spark BIGINT result - When query - """ - SELECT width_bucket(5.0, 0.0, 10.0, 5) AS result - """ - Then query result - | result | - | 3 | - And query schema - """ - root - |-- result: long (nullable = true) - """