From 52c78ddacb819ab84a40a5a04c4c236fd3146ac3 Mon Sep 17 00:00:00 2001 From: Jeadie Date: Tue, 23 Jun 2026 14:03:37 +1000 Subject: [PATCH 001/107] feat: allow an external semaphore for executor task concurrency Adds an optional available_task_slots: Option> parameter to the executor poll_loop. When provided, the caller supplies the semaphore that bounds concurrent task execution; this enables sharing it across poll loops connected to different schedulers and observing executor busy state via available_permits(). When None, the loop creates one internally sized to the executor's task slots (unchanged behavior). Ported from spiceai/datafusion-ballista#14 for upstreaming. Note: layers on the poll_loop signature; if landed after #12 (poll_now_notify) it needs a trivial rebase to add both parameters. --- ballista/executor/src/execution_loop.rs | 11 +++++++++-- ballista/executor/src/executor_process.rs | 1 + ballista/executor/src/standalone.rs | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index f64fae3014..bc67540555 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -57,10 +57,16 @@ use tonic::codegen::{Body, Bytes, StdError}; /// /// The loop respects the executor's concurrent task limit via a semaphore, /// ensuring no more than the configured number of tasks run simultaneously. +/// +/// `available_task_slots`, when provided, lets the caller supply the semaphore +/// controlling task concurrency (e.g. to share it across poll loops or to +/// observe executor busy state via `available_permits()`). When `None`, a +/// semaphore sized to the executor's task slots is created internally. pub async fn poll_loop( mut scheduler: SchedulerGrpcClient, executor: Arc, codec: BallistaCodec, + available_task_slots: Option>, ) -> Result<(), BallistaError> where C: tonic::client::GrpcService, @@ -75,8 +81,9 @@ where .unwrap() .clone() .into(); - let available_task_slots = - Arc::new(Semaphore::new(executor_specification.task_slots as usize)); + let available_task_slots = available_task_slots.unwrap_or_else(|| { + Arc::new(Semaphore::new(executor_specification.task_slots as usize)) + }); let (task_status_sender, mut task_status_receiver) = std::sync::mpsc::channel::(); diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 6050649ab9..36b2dcbb42 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -505,6 +505,7 @@ pub async fn start_executor_process( scheduler.clone(), executor.clone(), default_codec, + None, // available_task_slots: use internal semaphore ))); } }; diff --git a/ballista/executor/src/standalone.rs b/ballista/executor/src/standalone.rs index bd72dee4b4..f21e5c765f 100644 --- a/ballista/executor/src/standalone.rs +++ b/ballista/executor/src/standalone.rs @@ -142,7 +142,9 @@ pub async fn new_standalone_executor_from_builder( )), ); - tokio::spawn(execution_loop::poll_loop(scheduler, executor, codec)); + tokio::spawn(execution_loop::poll_loop( + scheduler, executor, codec, None, + )); Ok(()) } From 65bcfc3d976518fc2937fc3c2837d3cdfe34d33e Mon Sep 17 00:00:00 2001 From: jeadie Date: Tue, 23 Jun 2026 15:22:56 +1000 Subject: [PATCH 002/107] fix(examples): pass optional semaphore arg in mtls-cluster and reformat standalone --- ballista/executor/src/standalone.rs | 4 +--- examples/examples/mtls-cluster.rs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ballista/executor/src/standalone.rs b/ballista/executor/src/standalone.rs index f21e5c765f..a77ba1b868 100644 --- a/ballista/executor/src/standalone.rs +++ b/ballista/executor/src/standalone.rs @@ -142,9 +142,7 @@ pub async fn new_standalone_executor_from_builder( )), ); - tokio::spawn(execution_loop::poll_loop( - scheduler, executor, codec, None, - )); + tokio::spawn(execution_loop::poll_loop(scheduler, executor, codec, None)); Ok(()) } diff --git a/examples/examples/mtls-cluster.rs b/examples/examples/mtls-cluster.rs index 593248183e..128e2b64a7 100644 --- a/examples/examples/mtls-cluster.rs +++ b/examples/examples/mtls-cluster.rs @@ -430,7 +430,7 @@ async fn run_executor() -> Result<(), Box> { // This registers the executor and starts polling for tasks info!("Starting execution poll loop..."); let poll_handle = tokio::spawn(async move { - execution_loop::poll_loop(scheduler, executor, codec).await + execution_loop::poll_loop(scheduler, executor, codec, None).await }); tokio::select! { From 7c673cf524f33bb877ad7a05e25423e2fa248df5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:21:28 +0300 Subject: [PATCH 003/107] chore(deps): bump rustls from 0.23.40 to 0.23.41 (#1890) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd2c433c0f..4768c38179 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6186,9 +6186,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", From 2ad6574e963a9e61a92c39827d34ba85719b2b31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 08:13:39 +0300 Subject: [PATCH 004/107] chore(deps): bump taiki-e/install-action from 2.82.2 to 2.82.3 (#1894) --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2a77a53d5c..de51956598 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2.82.2 + uses: taiki-e/install-action@ace6ebe54a6a0c86dfb5f7764b17f793b6925bc3 # v2.82.3 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index fac580fa05..08c31b21a0 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2.82.2 + uses: taiki-e/install-action@ace6ebe54a6a0c86dfb5f7764b17f793b6925bc3 # v2.82.3 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 151e360f22..6d47caef85 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,7 +64,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2.82.2 + uses: taiki-e/install-action@ace6ebe54a6a0c86dfb5f7764b17f793b6925bc3 # v2.82.3 with: tool: trunk@0.21.14 From 6d6881ec520fc0879498745acd2286accb86ea3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 09:34:29 +0300 Subject: [PATCH 005/107] chore(deps): bump uuid from 1.23.3 to 1.23.4 (#1895) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.23.3 to 1.23.4. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.23.3...v1.23.4) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.23.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4768c38179..afc5ca20ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6977,7 +6977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -7793,9 +7793,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.3" +version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "atomic", "getrandom 0.4.2", From adfcfc923622361d39903fe505d8e67f3904de36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 08:00:40 +0300 Subject: [PATCH 006/107] chore(deps): bump env_logger from 0.11.10 to 0.11.11 (#1898) --- Cargo.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afc5ca20ec..bca6161665 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -123,7 +123,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3121,7 +3121,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3222,9 +3222,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_filter" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -3232,9 +3232,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -3256,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4740,7 +4740,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6181,7 +6181,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6240,7 +6240,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6708,7 +6708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6749,7 +6749,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6980,7 +6980,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6993,7 +6993,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8101,7 +8101,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 28a526747bd99c3e43e02710c30a95311eec48ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 08:01:10 +0300 Subject: [PATCH 007/107] chore(deps): bump taiki-e/install-action from 2.82.3 to 2.82.4 (#1897) --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index de51956598..1d8f81c3e0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@ace6ebe54a6a0c86dfb5f7764b17f793b6925bc3 # v2.82.3 + uses: taiki-e/install-action@682e7d9e49c5e653d371fc6adbda67653461378a # v2.82.4 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 08c31b21a0..da29044b1d 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@ace6ebe54a6a0c86dfb5f7764b17f793b6925bc3 # v2.82.3 + uses: taiki-e/install-action@682e7d9e49c5e653d371fc6adbda67653461378a # v2.82.4 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 6d47caef85..9ae87bf055 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,7 +64,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@ace6ebe54a6a0c86dfb5f7764b17f793b6925bc3 # v2.82.3 + uses: taiki-e/install-action@682e7d9e49c5e653d371fc6adbda67653461378a # v2.82.4 with: tool: trunk@0.21.14 From 55c2408734ff6d3566711d12bd22d712cf619829 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Fri, 26 Jun 2026 11:47:35 +0300 Subject: [PATCH 008/107] chore(ci): Deploy WebTUI to nightlies.a.o (#1888) * chore(ci): Deploy WebTUI to nightlies.a.o * Use v8.0.5 of rsync-deployments that is allowed by ASF Infra https://github.com/apache/infrastructure-actions/blob/8d7b86287d0beafc4d68a06fa23e751a04e3976d/actions.yml#L159C3-L160 * burnett01/rsync-deployments works only on Linux * Test whether rsync is pre-installed on macos runners * Use rsync command instead of Github Actions * Load the RSYNC_KEY (a PEM content) into SSH agent * Add RSYNC_HOST to SSH's known hosts Trying to solve error: "Host key verification failed." * Make use of secrets.NIGHTLIES_RSYNC_PATH * Try to figure out the destination PATH * Create the folders for datafusion/ballista/tui/ * Debug the copied files * Custom build for nightlies * Deploy to nightlies.a.o only on merge * Merge two CI jobs into one * Extract local env vars for reuse * Rename SSH_OPTIONS to SSH_OPTS * Rename SSH_OPTS to OPTIONS_SSH * Inline the SSH options * Experiment with './' as a public url * Build the WASM32 app once with public_dir="./" This way it could be used at any public url * Deploy at nightlies.a.o only on push to main * Use long name for the rsync options. It is easier to understand the command by humans Also remove some options which are not relevant for us - recursive, links, specials+devices --- .github/workflows/web-tui.yml | 25 +++++++++++++++++++++++++ ballista-cli/Trunk.toml | 1 + 2 files changed, 26 insertions(+) diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 9ae87bf055..1414e512ab 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -68,12 +68,37 @@ jobs: with: tool: trunk@0.21.14 + - name: Install cargo-get + uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 # v2.81.10 + with: + tool: cargo-get@1.4.0 + + - name: Get package.version + id: cargo_metadata + run: | + BALLISTA_VERSION=$(cargo get --entry ballista/core/ package.version) + echo "ballista_version=${BALLISTA_VERSION}" >> "$GITHUB_OUTPUT" + - name: Build working-directory: ballista-cli run: | trunk build --cargo-profile release-nonlto --minify --locked \ --no-default-features --features web + - name: Deploy at nightlies.a.o + if: ${{ github.event == 'push' && github.ref == 'refs/heads/main' }} + run: | + TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}" + REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/" + + echo "${{ secrets.NIGHTLIES_RSYNC_KEY }}" | ssh-add - + + ssh-keyscan -H "${{ secrets.NIGHTLIES_RSYNC_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null + + ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }} ${{ secrets.NIGHTLIES_RSYNC_HOST }} "mkdir -p ${REMOTE_TARGET_DIR}" + + rsync --times --compress --delete --verbose -e "ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }}" ./target/web-tui/* ${{ secrets.NIGHTLIES_RSYNC_HOST }}:${REMOTE_TARGET_DIR} + - name: Upload WASM32 application uses: actions/upload-artifact@v7 with: diff --git a/ballista-cli/Trunk.toml b/ballista-cli/Trunk.toml index 5f59741483..61ace21658 100644 --- a/ballista-cli/Trunk.toml +++ b/ballista-cli/Trunk.toml @@ -28,6 +28,7 @@ [build] target = "ballista-web-tui.html" dist = "../target/web-tui" +public_url = "./" [watch] ignore = [] From c89189c5a146ee8f5dc81fdf5dfaeea10d2d52c0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 27 Jun 2026 10:32:14 -0600 Subject: [PATCH 009/107] fix: preserve user session config overrides in upgrade_for_ballista (#1902) upgrade_for_ballista re-applied Ballista's opinionated DataFusion defaults (prefer_hash_join, hash_join_single_partition_threshold, view-type flags) unconditionally. In remote_with_state this runs after the user has set their own values, silently reverting datafusion.* overrides; ballista.* settings were unaffected because the defaults do not touch them. Apply the defaults only when the config has not already been through Ballista setup (no BallistaConfig extension present), so user overrides survive. --- ballista/core/src/extension.rs | 68 ++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs index a9da57c786..d53307339c 100644 --- a/ballista/core/src/extension.rs +++ b/ballista/core/src/extension.rs @@ -355,15 +355,22 @@ impl SessionConfigExt for SessionConfig { } fn upgrade_for_ballista(self) -> SessionConfig { - // if ballista config is not provided - // one is created and session state is updated + // Ballista's opinionated DataFusion defaults are applied once, when a + // plain config is first upgraded. The `BallistaConfig` extension marks a + // config that has already been through this path; re-applying the + // defaults on such a config would overwrite any values the user set + // afterwards (e.g. via `-c`/`SET`), so they are left untouched. + let already_upgraded = + self.options().extensions.get::().is_some(); + let ballista_config = self.ballista_config(); + let config = self.with_option_extension(ballista_config); - // session config has ballista config extension and - // default datafusion configuration is altered - // to fit ballista execution - self.with_option_extension(ballista_config) - .ballista_restricted_configuration() + if already_upgraded { + config + } else { + config.ballista_restricted_configuration() + } } fn ballista_config(&self) -> BallistaConfig { @@ -1018,6 +1025,53 @@ mod test { assert!(!state.config().round_robin_repartition()); } + + // User overrides of Ballista's soft defaults must survive `upgrade_for_ballista`; + // re-applying the defaults would discard them. See #1901. + #[test] + fn should_preserve_user_overrides_on_upgrade() { + // Ballista defaults these to prefer_hash_join=false and threshold=0. + let mut config = SessionConfig::new_with_ballista(); + config + .options_mut() + .set("datafusion.optimizer.prefer_hash_join", "true") + .unwrap(); + config + .options_mut() + .set( + "datafusion.optimizer.hash_join_single_partition_threshold", + "10485760", + ) + .unwrap(); + + let upgraded = config.upgrade_for_ballista(); + + assert!(upgraded.options().optimizer.prefer_hash_join); + assert_eq!( + upgraded + .options() + .optimizer + .hash_join_single_partition_threshold, + 10485760 + ); + } + + // A plain (non-Ballista) config still receives Ballista's opinionated + // defaults when upgraded. + #[test] + fn should_apply_defaults_when_upgrading_plain_config() { + let config = SessionConfig::new().upgrade_for_ballista(); + + assert!(!config.options().optimizer.prefer_hash_join); + assert_eq!( + config + .options() + .optimizer + .hash_join_single_partition_threshold, + 0 + ); + } + #[test] fn should_convert_to_key_value_pairs() { // key value pairs should contain datafusion and ballista values From 07bb0e1cec2039aaf5f61fb5a5fe9ee565c83d59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:28:44 +0300 Subject: [PATCH 010/107] chore(deps): bump config from 0.15.24 to 0.15.25 (#1915) Bumps [config](https://github.com/rust-cli/config-rs) from 0.15.24 to 0.15.25. - [Changelog](https://github.com/rust-cli/config-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/config-rs/compare/v0.15.24...v0.15.25) --- updated-dependencies: - dependency-name: config dependency-version: 0.15.25 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bca6161665..342ad458aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -123,7 +123,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1805,9 +1805,9 @@ checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "config" -version = "0.15.24" +version = "0.15.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b34d0237145f33580b89724f75d16950efd3e2c91b2d823917ecb69ec7f84f0" +checksum = "b85f248a4de22d204ceabc6299d89d2c70fbd7f09fea53c06c852369652d8139" dependencies = [ "pathdiff", "serde_core", @@ -3121,7 +3121,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3256,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4740,7 +4740,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6181,7 +6181,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6240,7 +6240,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6708,7 +6708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6749,7 +6749,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6977,10 +6977,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6993,7 +6993,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8101,7 +8101,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From a666e5a2a0848f3afb0eef22464d2e596fc7c88f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:29:47 +0300 Subject: [PATCH 011/107] chore(deps): bump taiki-e/install-action from 2.81.10 to 2.82.6 (#1914) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.81.10 to 2.82.6. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/v2.81.10...9bcaee1dcae34154180f412e2fa69355a7cda9f6) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.82.6 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1d8f81c3e0..da851b7402 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@682e7d9e49c5e653d371fc6adbda67653461378a # v2.82.4 + uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index da29044b1d..7dda5c1827 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@682e7d9e49c5e653d371fc6adbda67653461378a # v2.82.4 + uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 1414e512ab..a3ea4838d9 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,12 +64,12 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@682e7d9e49c5e653d371fc6adbda67653461378a # v2.82.4 + uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 with: tool: trunk@0.21.14 - name: Install cargo-get - uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 # v2.81.10 + uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 with: tool: cargo-get@1.4.0 From 278c1ad09c128511d0c786156188020ad5d9ffd0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 29 Jun 2026 07:38:25 -0600 Subject: [PATCH 012/107] Improve changelog generator and split CHANGELOG.md into per-version files (#1768) --- .github_changelog_generator | 28 - CHANGELOG.md | 1629 +---------------------------- dev/release/README.md | 53 +- dev/release/generate-changelog.py | 105 +- docs/source/changelog/0.10.0.md | 95 ++ docs/source/changelog/0.11.0.md | 99 ++ docs/source/changelog/0.12.0.md | 119 +++ docs/source/changelog/0.5.0.md | 174 +++ docs/source/changelog/0.6.0.md | 105 ++ docs/source/changelog/0.7.0.md | 165 +++ docs/source/changelog/0.8.0.md | 64 ++ docs/source/changelog/0.9.0.md | 184 ++++ docs/source/changelog/43.0.0.md | 146 +++ docs/source/changelog/44.0.0.md | 56 + docs/source/changelog/45.0.0.md | 45 + docs/source/changelog/46.0.0.md | 58 + docs/source/changelog/47.0.0.md | 48 + docs/source/changelog/48.0.0.md | 44 + docs/source/changelog/49.0.0.md | 58 + docs/source/changelog/50.0.0.md | 46 + docs/source/changelog/51.0.0.md | 45 + docs/source/changelog/52.0.0.md | 92 ++ docs/source/changelog/53.0.0.md | 233 +++++ docs/source/changelog/index.md | 46 + docs/source/index.rst | 6 + 25 files changed, 2045 insertions(+), 1698 deletions(-) delete mode 100644 .github_changelog_generator create mode 100644 docs/source/changelog/0.10.0.md create mode 100644 docs/source/changelog/0.11.0.md create mode 100644 docs/source/changelog/0.12.0.md create mode 100644 docs/source/changelog/0.5.0.md create mode 100644 docs/source/changelog/0.6.0.md create mode 100644 docs/source/changelog/0.7.0.md create mode 100644 docs/source/changelog/0.8.0.md create mode 100644 docs/source/changelog/0.9.0.md create mode 100644 docs/source/changelog/43.0.0.md create mode 100644 docs/source/changelog/44.0.0.md create mode 100644 docs/source/changelog/45.0.0.md create mode 100644 docs/source/changelog/46.0.0.md create mode 100644 docs/source/changelog/47.0.0.md create mode 100644 docs/source/changelog/48.0.0.md create mode 100644 docs/source/changelog/49.0.0.md create mode 100644 docs/source/changelog/50.0.0.md create mode 100644 docs/source/changelog/51.0.0.md create mode 100644 docs/source/changelog/52.0.0.md create mode 100644 docs/source/changelog/53.0.0.md create mode 100644 docs/source/changelog/index.md diff --git a/.github_changelog_generator b/.github_changelog_generator deleted file mode 100644 index 45eef2f518..0000000000 --- a/.github_changelog_generator +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# some issues are just documentation -add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]},"performance":{"prefix":"**Performance improvements:**","labels":["performance"]}} -# uncomment to not show PRs. TBD if we shown them or not. -#pull-requests=false -# so that the component is shown associated with the issue -issue-line-labels=sql -exclude-labels=development-process,invalid -breaking-labels=api change diff --git a/CHANGELOG.md b/CHANGELOG.md index dfab6fb9f6..c8d0c31fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,1630 +17,7 @@ under the License. --> -# Changelog +# Apache DataFusion Ballista Changelog -## [53.0.0](https://github.com/apache/datafusion-ballista/tree/53.0.0) (2026-05-19) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/52.0.0...53.0.0) - -**Implemented enhancements:** - -- [REST]: Cancelling a completed/failed job should return "false" [#1494](https://github.com/apache/datafusion-ballista/pull/1494) (martin-g) -- feat: Add plain "status" field to the JobResponse [#1497](https://github.com/apache/datafusion-ballista/pull/1497) (martin-g) -- feat: Expose Logical and Physical plan details in the REST API [#1498](https://github.com/apache/datafusion-ballista/pull/1498) (milenkovicm) -- feat: (remote) shuffle reader cleanup [#1503](https://github.com/apache/datafusion-ballista/pull/1503) (milenkovicm) -- feat: enable scheduler rest api by default [#1506](https://github.com/apache/datafusion-ballista/pull/1506) (milenkovicm) -- feat: [REST] Return the job's start and end times [#1511](https://github.com/apache/datafusion-ballista/pull/1511) (martin-g) -- feat: remove hash policy task distribution as not used [#1526](https://github.com/apache/datafusion-ballista/pull/1526) (eachsaj) -- feat: remove full path from partition locations [#1527](https://github.com/apache/datafusion-ballista/pull/1527) (milenkovicm) -- feat: add config to ExecutorEngine::create_query_stage_exec [#1542](https://github.com/apache/datafusion-ballista/pull/1542) (milenkovicm) -- feat: Improve REST API adding task related info to job stages [#1543](https://github.com/apache/datafusion-ballista/pull/1543) (milenkovicm) -- feat: Ballista Text User Interface [#1436](https://github.com/apache/datafusion-ballista/pull/1436) (martin-g) -- feat: jupyter notebook support [#1513](https://github.com/apache/datafusion-ballista/pull/1513) (sandugood) -- feat: ExecutionEngine::create_query_stage_exec accepts partition_id [#1556](https://github.com/apache/datafusion-ballista/pull/1556) (milenkovicm) -- feat: make ballista client retry policy configurable [#1577](https://github.com/apache/datafusion-ballista/pull/1577) (eachsaj) -- feat: Executor system & process metrics reporting [#1547](https://github.com/apache/datafusion-ballista/pull/1547) (sandugood) -- feat: Scheduler config update [#1597](https://github.com/apache/datafusion-ballista/pull/1597) (sandugood) -- feat: Support `EXPLAIN ANALYZE` in Ballista [#1567](https://github.com/apache/datafusion-ballista/pull/1567) (danielhumanmod) -- feat: Add standalone shuffle writer benchmark that shuffles real Parquet input [#1600](https://github.com/apache/datafusion-ballista/pull/1600) (andygrove) -- feat: defer sort-shuffle materialization with interleave_record_batch [#1598](https://github.com/apache/datafusion-ballista/pull/1598) (andygrove) -- feat(executor): bound executor memory via --memory-pool-size [#1624](https://github.com/apache/datafusion-ballista/pull/1624) (andygrove) -- feat(bench): show TPC-H query timings in seconds and add total time [#1641](https://github.com/apache/datafusion-ballista/pull/1641) (andygrove) -- feat(aqe): Support sort-based shuffle writer in AQE [#1640](https://github.com/apache/datafusion-ballista/pull/1640) (danielhumanmod) -- feat: rest api supports plan tree rendering [#1650](https://github.com/apache/datafusion-ballista/pull/1650) (sandugood) -- feat: Cache ballista clients on executor [#1578](https://github.com/apache/datafusion-ballista/pull/1578) (milenkovicm) -- feat(aqe): Lazy stage evaluation in AQE [#1649](https://github.com/apache/datafusion-ballista/pull/1649) (milenkovicm) -- feat: move shuffle writer disk I/O off tokio worker threads [#1537](https://github.com/apache/datafusion-ballista/pull/1537) (hcrosse) -- feat(scheduler): broadcast-style hash join for small-side joins [#1647](https://github.com/apache/datafusion-ballista/pull/1647) (andygrove) -- feat: default to sort-merge join [#1651](https://github.com/apache/datafusion-ballista/pull/1651) (andygrove) -- feat: TUI shows running job information [#1717](https://github.com/apache/datafusion-ballista/pull/1717) (milenkovicm) -- feat(aqe): CoalescePartitionsRule — shuffle-partition coalescing on resolved stats [#1684](https://github.com/apache/datafusion-ballista/pull/1684) (metegenez) -- feat: TUI make task popup scrollable [#1725](https://github.com/apache/datafusion-ballista/pull/1725) (milenkovicm) -- feat(tui): Use separate areas for the table and its associated scrollbar [#1729](https://github.com/apache/datafusion-ballista/pull/1729) (martin-g) - -**Fixed bugs:** - -- fix: remove unwrap from executor and improve task error handling [#1540](https://github.com/apache/datafusion-ballista/pull/1540) (eachsaj) -- fix: handle None task slot in update_task_info after executor lost [#1523](https://github.com/apache/datafusion-ballista/pull/1523) (milenkovicm) -- fix: [Python] serialize optimized logical plan to fix subquery support [#1586](https://github.com/apache/datafusion-ballista/pull/1586) (andygrove) -- fix: allow S3 access without explicit credentials [#1584](https://github.com/apache/datafusion-ballista/pull/1584) (andygrove) -- fix: route collect/show/to_pandas through Ballista cluster [#1585](https://github.com/apache/datafusion-ballista/pull/1585) (andygrove) -- fix: propagate session config from Python client to Ballista cluster [#1592](https://github.com/apache/datafusion-ballista/pull/1592) (andygrove) -- fix(python): ignore ballista-namespaced cluster_config keys locally [#1613](https://github.com/apache/datafusion-ballista/pull/1613) (andygrove) -- fix: `df.write_` fix as it was broken after update [#1625](https://github.com/apache/datafusion-ballista/pull/1625) (milenkovicm) -- fix(rest): remove unwrap and return 404 if executor does not exist [#1628](https://github.com/apache/datafusion-ballista/pull/1628) (milenkovicm) -- fix(metrics): avoid stage metrics inflation by tracking partition snapshots [#1652](https://github.com/apache/datafusion-ballista/pull/1652) (danielhumanmod) -- fix: compilation issue after merge [#1658](https://github.com/apache/datafusion-ballista/pull/1658) (milenkovicm) -- fix: rest api calculates stage running time correctly [#1675](https://github.com/apache/datafusion-ballista/pull/1675) (milenkovicm) -- fix: REST API does not show running jobs [#1703](https://github.com/apache/datafusion-ballista/pull/1703) (gittihub-jpg) -- fix: no executor warning, correct prometheus feature name in TUI [#1698](https://github.com/apache/datafusion-ballista/pull/1698) (killzoner) - -**Documentation updates:** - -- docs: document hash-based and sort-based shuffle implementations [#1595](https://github.com/apache/datafusion-ballista/pull/1595) (andygrove) -- docs: improve Python documentation structure [#1579](https://github.com/apache/datafusion-ballista/pull/1579) (andygrove) -- docs: add Python client build-from-source section to contributor guide [#1620](https://github.com/apache/datafusion-ballista/pull/1620) (andygrove) -- chore: remove NYC Taxi benchmark [#1644](https://github.com/apache/datafusion-ballista/pull/1644) (andygrove) -- docs: document experimental Adaptive Query Execution in tuning guide [#1645](https://github.com/apache/datafusion-ballista/pull/1645) (andygrove) -- build(bench): rework docker-compose TPC-H stack [#1646](https://github.com/apache/datafusion-ballista/pull/1646) (andygrove) -- docs: add Ballista TUI documentation [#1593](https://github.com/apache/datafusion-ballista/pull/1593) (goingforstudying-ctrl) -- [TUI] Show executor's details in a popup [#1670](https://github.com/apache/datafusion-ballista/pull/1670) (martin-g) -- [TUI] Add a config setting for rendering job stage's plan as a tree [#1704](https://github.com/apache/datafusion-ballista/pull/1704) (martin-g) -- [TUI] Add support for horizontal scrolling to the job/stage plan popups [#1711](https://github.com/apache/datafusion-ballista/pull/1711) (martin-g) -- [TUI] Add screenshots of the TUI application in README/cli.md [#1714](https://github.com/apache/datafusion-ballista/pull/1714) (martin-g) - -**Merged pull requests:** - -- CI: Add CodeQL workflow for GitHub Actions security scanning [#1484](https://github.com/apache/datafusion-ballista/pull/1484) (kevinjqliu) -- ci: Harden labeler workflow, remove unnecessary checkout from pull_request_target job [#1487](https://github.com/apache/datafusion-ballista/pull/1487) (kevinjqliu) -- ci: add take and stale [#1488](https://github.com/apache/datafusion-ballista/pull/1488) (kevinjqliu) -- [REST]: Cancelling a completed/failed job should return "false" [#1494](https://github.com/apache/datafusion-ballista/pull/1494) (martin-g) -- feat: Add plain "status" field to the JobResponse [#1497](https://github.com/apache/datafusion-ballista/pull/1497) (martin-g) -- feat: Expose Logical and Physical plan details in the REST API [#1498](https://github.com/apache/datafusion-ballista/pull/1498) (milenkovicm) -- chore(deps): cargo update deps [#1502](https://github.com/apache/datafusion-ballista/pull/1502) (milenkovicm) -- feat: (remote) shuffle reader cleanup [#1503](https://github.com/apache/datafusion-ballista/pull/1503) (milenkovicm) -- minor: task scheduling policy config cleanup [#1507](https://github.com/apache/datafusion-ballista/pull/1507) (milenkovicm) -- minor: address task policy config comments [#1508](https://github.com/apache/datafusion-ballista/pull/1508) (milenkovicm) -- feat: enable scheduler rest api by default [#1506](https://github.com/apache/datafusion-ballista/pull/1506) (milenkovicm) -- feat: [REST] Return the job's start and end times [#1511](https://github.com/apache/datafusion-ballista/pull/1511) (martin-g) -- minor: [REST] add datafusion version info [#1512](https://github.com/apache/datafusion-ballista/pull/1512) (milenkovicm) -- ci: pin third-party actions to Apache-approved SHAs [#1516](https://github.com/apache/datafusion-ballista/pull/1516) (kevinjqliu) -- chore(deps): update to datafusion v.53 [#1486](https://github.com/apache/datafusion-ballista/pull/1486) (milenkovicm) -- feat: remove hash policy task distribution as not used [#1526](https://github.com/apache/datafusion-ballista/pull/1526) (eachsaj) -- chore: update datafusion proto [#1528](https://github.com/apache/datafusion-ballista/pull/1528) (milenkovicm) -- minor: cleanup scheduler clap configuration [#1529](https://github.com/apache/datafusion-ballista/pull/1529) (milenkovicm) -- fix: remove unwrap from executor and improve task error handling [#1540](https://github.com/apache/datafusion-ballista/pull/1540) (eachsaj) -- feat: remove full path from partition locations [#1527](https://github.com/apache/datafusion-ballista/pull/1527) (milenkovicm) -- fix: handle None task slot in update_task_info after executor lost [#1523](https://github.com/apache/datafusion-ballista/pull/1523) (milenkovicm) -- feat: add config to ExecutorEngine::create_query_stage_exec [#1542](https://github.com/apache/datafusion-ballista/pull/1542) (milenkovicm) -- feat: Improve REST API adding task related info to job stages [#1543](https://github.com/apache/datafusion-ballista/pull/1543) (milenkovicm) -- feat: Ballista Text User Interface [#1436](https://github.com/apache/datafusion-ballista/pull/1436) (martin-g) -- feat: jupyter notebook support [#1513](https://github.com/apache/datafusion-ballista/pull/1513) (sandugood) -- feat: ExecutionEngine::create_query_stage_exec accepts partition_id [#1556](https://github.com/apache/datafusion-ballista/pull/1556) (milenkovicm) -- chore: Fix Clippy issues with Rust 1.95.0 [#1558](https://github.com/apache/datafusion-ballista/pull/1558) (martin-g) -- chore: update deps [#1561](https://github.com/apache/datafusion-ballista/pull/1561) (milenkovicm) -- [TUI] Change the key binding for job's plans [#1573](https://github.com/apache/datafusion-ballista/pull/1573) (martin-g) -- feat: make ballista client retry policy configurable [#1577](https://github.com/apache/datafusion-ballista/pull/1577) (eachsaj) -- feat: Executor system & process metrics reporting [#1547](https://github.com/apache/datafusion-ballista/pull/1547) (sandugood) -- fix: [Python] serialize optimized logical plan to fix subquery support [#1586](https://github.com/apache/datafusion-ballista/pull/1586) (andygrove) -- fix: allow S3 access without explicit credentials [#1584](https://github.com/apache/datafusion-ballista/pull/1584) (andygrove) -- fix: route collect/show/to_pandas through Ballista cluster [#1585](https://github.com/apache/datafusion-ballista/pull/1585) (andygrove) -- fix: propagate session config from Python client to Ballista cluster [#1592](https://github.com/apache/datafusion-ballista/pull/1592) (andygrove) -- minor: make config naming convention consistent [#1580](https://github.com/apache/datafusion-ballista/pull/1580) (milenkovicm) -- chore: Remove unnecessary optimizer rules due to datafusion upgrade to v53 [#1594](https://github.com/apache/datafusion-ballista/pull/1594) (sandugood) -- docs: document hash-based and sort-based shuffle implementations [#1595](https://github.com/apache/datafusion-ballista/pull/1595) (andygrove) -- feat: Scheduler config update [#1597](https://github.com/apache/datafusion-ballista/pull/1597) (sandugood) -- feat: Support `EXPLAIN ANALYZE` in Ballista [#1567](https://github.com/apache/datafusion-ballista/pull/1567) (danielhumanmod) -- docs: improve Python documentation structure [#1579](https://github.com/apache/datafusion-ballista/pull/1579) (andygrove) -- feat: Add standalone shuffle writer benchmark that shuffles real Parquet input [#1600](https://github.com/apache/datafusion-ballista/pull/1600) (andygrove) -- feat: defer sort-shuffle materialization with interleave_record_batch [#1598](https://github.com/apache/datafusion-ballista/pull/1598) (andygrove) -- ci: drop Intel macOS Python wheel build [#1612](https://github.com/apache/datafusion-ballista/pull/1612) (andygrove) -- fix(python): ignore ballista-namespaced cluster_config keys locally [#1613](https://github.com/apache/datafusion-ballista/pull/1613) (andygrove) -- chore: update python deps to ballista and datafusion 52 [#1590](https://github.com/apache/datafusion-ballista/pull/1590) (andygrove) -- feat(sort-shuffle): byte-copy spill files and enable block-IO transport [#1615](https://github.com/apache/datafusion-ballista/pull/1615) (andygrove) -- Use BallistaSessionContext instead of BallistaBuilder in tpch.py [#1621](https://github.com/apache/datafusion-ballista/pull/1621) (martin-g) -- docs: add Python client build-from-source section to contributor guide [#1620](https://github.com/apache/datafusion-ballista/pull/1620) (andygrove) -- feat(executor): bound executor memory via --memory-pool-size [#1624](https://github.com/apache/datafusion-ballista/pull/1624) (andygrove) -- fix: `df.write_` fix as it was broken after update [#1625](https://github.com/apache/datafusion-ballista/pull/1625) (milenkovicm) -- feat(sort-shuffle): enable sort-based shuffle by default [#1623](https://github.com/apache/datafusion-ballista/pull/1623) (andygrove) -- fix(rest): remove unwrap and return 404 if executor does not exist [#1628](https://github.com/apache/datafusion-ballista/pull/1628) (milenkovicm) -- perf(sort-shuffle): fix performance regression caused by datafusion upgrade [#1626](https://github.com/apache/datafusion-ballista/pull/1626) (andygrove) -- fix(sort-shuffle): bound writer memory with per-task spill threshold [#1636](https://github.com/apache/datafusion-ballista/pull/1636) (andygrove) -- chore: remove NYC Taxi benchmark [#1644](https://github.com/apache/datafusion-ballista/pull/1644) (andygrove) -- docs: document experimental Adaptive Query Execution in tuning guide [#1645](https://github.com/apache/datafusion-ballista/pull/1645) (andygrove) -- feat(bench): show TPC-H query timings in seconds and add total time [#1641](https://github.com/apache/datafusion-ballista/pull/1641) (andygrove) -- build(bench): rework docker-compose TPC-H stack [#1646](https://github.com/apache/datafusion-ballista/pull/1646) (andygrove) -- feat(aqe): Support sort-based shuffle writer in AQE [#1640](https://github.com/apache/datafusion-ballista/pull/1640) (danielhumanmod) -- fix(metrics): avoid stage metrics inflation by tracking partition snapshots [#1652](https://github.com/apache/datafusion-ballista/pull/1652) (danielhumanmod) -- feat: rest api supports plan tree rendering [#1650](https://github.com/apache/datafusion-ballista/pull/1650) (sandugood) -- feat: Cache ballista clients on executor [#1578](https://github.com/apache/datafusion-ballista/pull/1578) (milenkovicm) -- fix: compilation issue after merge [#1658](https://github.com/apache/datafusion-ballista/pull/1658) (milenkovicm) -- [TUI] Show job's stages and their tasks [#1574](https://github.com/apache/datafusion-ballista/pull/1574) (martin-g) -- [TUI] Configurable tick interval [#1669](https://github.com/apache/datafusion-ballista/pull/1669) (martin-g) -- feat(aqe): Lazy stage evaluation in AQE [#1649](https://github.com/apache/datafusion-ballista/pull/1649) (milenkovicm) -- docs: add Ballista TUI documentation [#1593](https://github.com/apache/datafusion-ballista/pull/1593) (goingforstudying-ctrl) -- feat: move shuffle writer disk I/O off tokio worker threads [#1537](https://github.com/apache/datafusion-ballista/pull/1537) (hcrosse) -- feat(scheduler): broadcast-style hash join for small-side joins [#1647](https://github.com/apache/datafusion-ballista/pull/1647) (andygrove) -- feat: default to sort-merge join [#1651](https://github.com/apache/datafusion-ballista/pull/1651) (andygrove) -- minor: add pending stage indicator for `AdaptiveDatafusionExec` [#1672](https://github.com/apache/datafusion-ballista/pull/1672) (milenkovicm) -- fix: rest api calculates stage running time correctly [#1675](https://github.com/apache/datafusion-ballista/pull/1675) (milenkovicm) -- minor: change parameter ordering in `AdaptivePlanner::try_new_with_optimizers` [#1687](https://github.com/apache/datafusion-ballista/pull/1687) (milenkovicm) -- [TUI] Show executor's details in a popup [#1670](https://github.com/apache/datafusion-ballista/pull/1670) (martin-g) -- Fix REST API panic on job list/detail when end_time < start_time [#1693](https://github.com/apache/datafusion-ballista/pull/1693) (abhinavgautam01) -- Right align all numeric columns in the TUI tables [#1695](https://github.com/apache/datafusion-ballista/pull/1695) (martin-g) -- fix(join-selection): guard CollectLeft swap when right has multiple partitions [#1691](https://github.com/apache/datafusion-ballista/pull/1691) (andygrove) -- Minor improvements for #1675 [#1686](https://github.com/apache/datafusion-ballista/pull/1686) (martin-g) -- [CLI/TUI] Use only tracing crate for logging in CLI and TUI [#1697](https://github.com/apache/datafusion-ballista/pull/1697) (martin-g) -- minor: change log level for few statements [#1706](https://github.com/apache/datafusion-ballista/pull/1706) (milenkovicm) -- Saturate scheduler job elapsed time [#1708](https://github.com/apache/datafusion-ballista/pull/1708) (MukundaKatta) -- [TUI] Add a config setting for rendering job stage's plan as a tree [#1704](https://github.com/apache/datafusion-ballista/pull/1704) (martin-g) -- [TUI] Add support for horizontal scrolling to the job/stage plan popups [#1711](https://github.com/apache/datafusion-ballista/pull/1711) (martin-g) -- [TUI] Executor's id is not a numeric column. It should be center aligned [#1713](https://github.com/apache/datafusion-ballista/pull/1713) (martin-g) -- [TUI] Add screenshots of the TUI application in README/cli.md [#1714](https://github.com/apache/datafusion-ballista/pull/1714) (martin-g) -- Make use of Swatinem/rust-cache to make the CI workflows faster [#1705](https://github.com/apache/datafusion-ballista/pull/1705) (martin-g) -- Merge Executor's brief and extended properties [#1716](https://github.com/apache/datafusion-ballista/pull/1716) (martin-g) -- fix: REST API does not show running jobs [#1703](https://github.com/apache/datafusion-ballista/pull/1703) (gittihub-jpg) -- [INFRA] Set up default rulesets for default and release branches [#1715](https://github.com/apache/datafusion-ballista/pull/1715) (asf-gitbox-commits) -- feat: TUI shows running job information [#1717](https://github.com/apache/datafusion-ballista/pull/1717) (milenkovicm) -- feat(aqe): CoalescePartitionsRule — shuffle-partition coalescing on resolved stats [#1684](https://github.com/apache/datafusion-ballista/pull/1684) (metegenez) -- feat: TUI make task popup scrollable [#1725](https://github.com/apache/datafusion-ballista/pull/1725) (milenkovicm) -- ci: add TPC-H SF10 workflow [#1688](https://github.com/apache/datafusion-ballista/pull/1688) (andygrove) -- feat(tui): Use separate areas for the table and its associated scrollbar [#1729](https://github.com/apache/datafusion-ballista/pull/1729) (martin-g) -- minor: [TUI] Extract a helper method for the table/scrollbar area splitter [#1730](https://github.com/apache/datafusion-ballista/pull/1730) (martin-g) -- minor: [TUI] Sort the metrics before rendering them [#1731](https://github.com/apache/datafusion-ballista/pull/1731) (martin-g) -- chore: Do not run -cli tests twice [#1732](https://github.com/apache/datafusion-ballista/pull/1732) (martin-g) -- fix: no executor warning, correct prometheus feature name in TUI [#1698](https://github.com/apache/datafusion-ballista/pull/1698) (killzoner) - -## [52.0.0](https://github.com/apache/datafusion-ballista/tree/52.0.0) (2026-03-02) - -**Performance related:** - -- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove) - -**Implemented enhancements:** - -- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner) -- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove) -- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove) -- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm) -- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod) -- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg) -- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento) -- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc) -- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove) -- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm) -- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento) -- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner) -- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm) -- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm) -- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm) - -**Fixed bugs:** - -- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm) -- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner) -- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm) -- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g) - -**Documentation updates:** - -- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove) -- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove) -- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove) -- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento) - -**Merged pull requests:** - -- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner) -- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove) -- restrict python CI to python directory [#1383](https://github.com/apache/datafusion-ballista/pull/1383) (Huy1Ng) -- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove) -- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove) -- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove) -- doc: Add Ballista extensions example to the docs. [#1382](https://github.com/apache/datafusion-ballista/pull/1382) (LouisBurke) -- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove) -- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm) -- doc: add more details for protobuf extension [#1393](https://github.com/apache/datafusion-ballista/pull/1393) (LouisBurke) -- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod) -- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove) -- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg) -- chore: update datafusion to 52 [#1394](https://github.com/apache/datafusion-ballista/pull/1394) (killzoner) -- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento) -- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm) -- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc) -- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento) -- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner) -- Improve sort-based shuffle: single spill file per partition and batch coalescing [#1431](https://github.com/apache/datafusion-ballista/pull/1431) (andygrove) -- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove) -- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm) -- chore: add confirmation before tarball is released [#1445](https://github.com/apache/datafusion-ballista/pull/1445) (milenkovicm) -- minor: add test to cover IPC arrow file read [#1450](https://github.com/apache/datafusion-ballista/pull/1450) (milenkovicm) -- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento) -- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner) -- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm) -- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm) -- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm) -- minor: improve log statements [#1482](https://github.com/apache/datafusion-ballista/pull/1482) (milenkovicm) -- chore: update datafusion to 52.2 and other deps to latest [#1483](https://github.com/apache/datafusion-ballista/pull/1483) (milenkovicm) -- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g) -- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm) - -## [51.0.0](https://github.com/apache/datafusion-ballista/tree/51.0.0) (2026-01-11) - -**Implemented enhancements:** - -- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod) -- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner) -- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang) -- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento) -- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento) - -**Merged pull requests:** - -- minor: minor changes to release script/docs [#1342](https://github.com/apache/datafusion-ballista/pull/1342) (andygrove) -- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod) -- chore: update datafusion to 51.0 [#1345](https://github.com/apache/datafusion-ballista/pull/1345) (danielhumanmod) -- doc: Add a note that datafusion may need to be downgraded after installing it [#1348](https://github.com/apache/datafusion-ballista/pull/1348) (martin-g) -- minor: Make `DisplayAs` consistent and more readable for ShuffleExec [#1347](https://github.com/apache/datafusion-ballista/pull/1347) (milenkovicm) -- minor: remove unnecessary clone functions [#1352](https://github.com/apache/datafusion-ballista/pull/1352) (mmooyyii) -- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner) -- chore(doc): Clean up deployment docs. [#1354](https://github.com/apache/datafusion-ballista/pull/1354) (LouisBurke) -- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang) -- doc: Fix plan translation example to use correct aggregation and column [#1362](https://github.com/apache/datafusion-ballista/pull/1362) (mattcuento) -- chore: update ballista version to 51.0.0 (from 50.0.0) [#1363](https://github.com/apache/datafusion-ballista/pull/1363) (milenkovicm) -- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento) -- chore: Add missing public API documentation/comments [#1364](https://github.com/apache/datafusion-ballista/pull/1364) (killzoner) -- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento) - -## [50.0.0](https://github.com/apache/datafusion-ballista/tree/50.0.0) (2025-11-04) - -**Implemented enhancements:** - -- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan) - -**Fixed bugs:** - -- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear) -- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel) -- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm) - -**Merged pull requests:** - -- infra: macos-13 is deprecated [#1324](https://github.com/apache/datafusion-ballista/pull/1324) (kevinjqliu) -- chore: update to datafusion v50 [#1320](https://github.com/apache/datafusion-ballista/pull/1320) (milenkovicm) -- chore: Update datafusion to 50.2 [#1326](https://github.com/apache/datafusion-ballista/pull/1326) (milenkovicm) -- document private items and update docs CI [#1327](https://github.com/apache/datafusion-ballista/pull/1327) (killzoner) -- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear) -- add msrvcheck [#1328](https://github.com/apache/datafusion-ballista/pull/1328) (killzoner) -- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel) -- chore: Pinning versions of external actions. [#1334](https://github.com/apache/datafusion-ballista/pull/1334) (samueleresca) -- chore: update python deps to 49 [#1335](https://github.com/apache/datafusion-ballista/pull/1335) (milenkovicm) -- chore: update datafusion to 50.3 [#1336](https://github.com/apache/datafusion-ballista/pull/1336) (milenkovicm) -- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm) -- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan) -- minor: change log level [#1340](https://github.com/apache/datafusion-ballista/pull/1340) (milenkovicm) - -## [49.0.0](https://github.com/apache/datafusion-ballista/tree/49.0.0) (2025-09-12) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/48.0.0...49.0.0) - -**Implemented enhancements:** - -- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm) -- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay) - -**Fixed bugs:** - -- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm) -- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez) -- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm) -- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm) -- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm) -- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm) - -**Merged pull requests:** - -- chore: update datafusion to 49 [#1285](https://github.com/apache/datafusion-ballista/pull/1285) (milenkovicm) -- chore: Improve GitHub actions/python workflows [#1289](https://github.com/apache/datafusion-ballista/pull/1289) (Huy1Ng) -- chore: update datafusion to 49.0.2 [#1298](https://github.com/apache/datafusion-ballista/pull/1298) (milenkovicm) -- minor: make shuffle exec display consistent [#1299](https://github.com/apache/datafusion-ballista/pull/1299) (milenkovicm) -- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm) -- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez) -- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm) -- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm) -- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm) -- chore: notice and cargo deps cleanup [#1295](https://github.com/apache/datafusion-ballista/pull/1295) (milenkovicm) -- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 [#1303](https://github.com/apache/datafusion-ballista/pull/1303) (dependabot[bot]) -- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 in /python [#1308](https://github.com/apache/datafusion-ballista/pull/1308) (dependabot[bot]) -- refactor: expand ClusterEventSender visibility to public [#1312](https://github.com/apache/datafusion-ballista/pull/1312) (Th824) -- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm) -- minor: enable json write test [#1311](https://github.com/apache/datafusion-ballista/pull/1311) (milenkovicm) -- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm) -- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay) -- feat: Improve Remote Shuffle Read Speed and Resource Utilisation [#1318](https://github.com/apache/datafusion-ballista/pull/1318) (milenkovicm) -- fix: Issue with `JoinSelection` and `CrossJoinExec` when stages have been resoled [#1322](https://github.com/apache/datafusion-ballista/pull/1322) (ZihuanLing) - -## [48.0.0](https://github.com/apache/datafusion-ballista/tree/48.0.0) (2025-07-30) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/47.0.0...48.0.0) - -**Fixed bugs:** - -- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG) -- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm) -- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm) - -**Documentation updates:** - -- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm) -- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone) - -**Merged pull requests:** - -- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm) -- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone) -- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG) -- improve rust workflows without cache [#1275](https://github.com/apache/datafusion-ballista/pull/1275) (Huy1Ng) -- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm) -- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm) -- chore: update python module to latest ballista release (v.47) [#1279](https://github.com/apache/datafusion-ballista/pull/1279) (milenkovicm) -- chore: update datafusion to 48.0.1 [#1284](https://github.com/apache/datafusion-ballista/pull/1284) (milenkovicm) - -## [47.0.0](https://github.com/apache/datafusion-ballista/tree/47.0.0) (2025-06-15) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/46.0.0...47.0.0) - -**Implemented enhancements:** - -- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm) -- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm) -- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm) -- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm) -- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm) -- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm) - -**Fixed bugs:** - -- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm) -- fix: fix tests failing on windows [#1273](https://github.com/apache/datafusion-ballista/pull/1273) (Huy1Ng) - -**Merged pull requests:** - -- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm) -- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm) -- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm) -- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm) -- minor: release docker on when release has been tagged [#1264](https://github.com/apache/datafusion-ballista/pull/1264) (milenkovicm) -- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm) -- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm) -- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm) -- chore(deps): update to datafusion 47.0.0 [#1250](https://github.com/apache/datafusion-ballista/pull/1250) (milenkovicm) - -## [46.0.0](https://github.com/apache/datafusion-ballista/tree/46.0.0) (2025-05-05) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/45.0.0...46.0.0) - -**Implemented enhancements:** - -- feat: make task distribution policies pluggable [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm) -- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm) -- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm) -- feat: make execution_graph.stages() public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm) - -**Fixed bugs:** - -- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii) - -**Merged pull requests:** - -- chore: Remove some arrow references [#1232](https://github.com/apache/datafusion-ballista/pull/1232) (andygrove) -- chore: remove unused executor configuration option [#1229](https://github.com/apache/datafusion-ballista/pull/1229) (milenkovicm) -- chore: return `404` for api requests if path does not exist [#1224](https://github.com/apache/datafusion-ballista/pull/1224) (milenkovicm) -- chore(ci): replace `actions-rs` which are deprecated [#1222](https://github.com/apache/datafusion-ballista/pull/1222) (milenkovicm) -- minor: Decouple `ExecutionGraph` and `DistributedPlanner` [#1221](https://github.com/apache/datafusion-ballista/pull/1221) (milenkovicm) -- chore: update datafusion to 46 [#1201](https://github.com/apache/datafusion-ballista/pull/1201) (milenkovicm) -- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 [#1233](https://github.com/apache/datafusion-ballista/pull/1233) (dependabot[bot]) -- chore(deps): bump tokio from 1.44.1 to 1.44.2 [#1234](https://github.com/apache/datafusion-ballista/pull/1234) (dependabot[bot]) -- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii) -- chore: update python deps to 45 [#1240](https://github.com/apache/datafusion-ballista/pull/1240) (milenkovicm) -- Add S3 object store support to executor and scheduler [#1230](https://github.com/apache/datafusion-ballista/pull/1230) (milenkovicm) -- feat: make task distribution policies pluggable [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm) -- chore: reduce log levels for few log statements [#1237](https://github.com/apache/datafusion-ballista/pull/1237) (milenkovicm) -- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 in /python [#1244](https://github.com/apache/datafusion-ballista/pull/1244) (dependabot[bot]) -- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm) -- minor: `executor_shutdown_while_running` test has race condition [#1248](https://github.com/apache/datafusion-ballista/pull/1248) (milenkovicm) -- bug: build fails with `--no-default-features` [#1255](https://github.com/apache/datafusion-ballista/pull/1255) (milenkovicm) -- doc: update architectural diagram [#1253](https://github.com/apache/datafusion-ballista/pull/1253) (milenkovicm) -- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm) -- chore: add read/write roundtrip tests [#1249](https://github.com/apache/datafusion-ballista/pull/1249) (milenkovicm) -- minor: change log level for object store creation [#1247](https://github.com/apache/datafusion-ballista/pull/1247) (milenkovicm) -- feat: make execution_graph.stages() public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm) - -## [45.0.0](https://github.com/apache/datafusion-ballista/tree/45.0.0) (2025-03-30) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/44.0.0...45.0.0) - -**Implemented enhancements:** - -- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm) -- feat: publish docker containers for executor and scheduler [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm) - -**Merged pull requests:** - -- chore: minor release script fix [#1192](https://github.com/apache/datafusion-ballista/pull/1192) (andygrove) -- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm) -- chore: update datafusion to v45 [#1176](https://github.com/apache/datafusion-ballista/pull/1176) (milenkovicm) -- chore: Update changelog for 44.0.0 [#1191](https://github.com/apache/datafusion-ballista/pull/1191) (andygrove) -- minor: fix repo and homepage url in `cargo.toml` [#1196](https://github.com/apache/datafusion-ballista/pull/1196) (milenkovicm) -- chore(deps): bump ring from 0.17.11 to 0.17.13 [#1199](https://github.com/apache/datafusion-ballista/pull/1199) (dependabot[bot]) -- feat: publish docker containers for executor and scheduler [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm) -- minor: make `graphviz-rust` dependency optional [#1203](https://github.com/apache/datafusion-ballista/pull/1203) (milenkovicm) -- doc: update docker related documentation [#1204](https://github.com/apache/datafusion-ballista/pull/1204) (milenkovicm) -- chore: update python dependencies [#1197](https://github.com/apache/datafusion-ballista/pull/1197) (milenkovicm) -- chore(deps): bump ring from 0.17.8 to 0.17.14 in /python [#1206](https://github.com/apache/datafusion-ballista/pull/1206) (dependabot[bot]) -- doc: remove arrow from doc title [#1207](https://github.com/apache/datafusion-ballista/pull/1207) (milenkovicm) -- Fix unit tests in tpch.rs [#1195](https://github.com/apache/datafusion-ballista/pull/1195) (vmingchen) -- documentation :: quick-start.md sample source code correction [#1213](https://github.com/apache/datafusion-ballista/pull/1213) (nj7) -- doc: fix quick-start executor command [#1217](https://github.com/apache/datafusion-ballista/pull/1217) (westhide) - -## [44.0.0](https://github.com/apache/datafusion-ballista/tree/44.0.0) (2025-03-01) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/43.0.0...44.0.0) - -**Implemented enhancements:** - -- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm) -- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm) -- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm) -- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm) - -**Fixed bugs:** - -- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm) -- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm) - -**Documentation updates:** - -- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm) - -**Merged pull requests:** - -- chore: planner cleanup and refactor [#1160](https://github.com/apache/datafusion-ballista/pull/1160) (milenkovicm) -- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm) -- chore: fix executor build issue on release [#1167](https://github.com/apache/datafusion-ballista/pull/1167) (milenkovicm) -- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm) -- chore: update to DF.44 [#1153](https://github.com/apache/datafusion-ballista/pull/1153) (milenkovicm) -- chore: publicly expose datafusion in ballista client [#1169](https://github.com/apache/datafusion-ballista/pull/1169) (milenkovicm) -- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm) -- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm) -- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm) -- chore: generate change log for 44.0.0 [#1173](https://github.com/apache/datafusion-ballista/pull/1173) (milenkovicm) -- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm) -- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm) -- chore: fix tpch data generator [#1186](https://github.com/apache/datafusion-ballista/pull/1186) (milenkovicm) -- chore: fix clippy after rust 1.85 update [#1188](https://github.com/apache/datafusion-ballista/pull/1188) (milenkovicm) -- chore: commit `Cargo.lock` file to make builds more predictable [#1190](https://github.com/apache/datafusion-ballista/pull/1190) (milenkovicm) - -## [43.0.0]( (2025-01-07) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.12.0...43.0.0-rc2) - -**Implemented enhancements:** - -- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov) -- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove) -- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove) -- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm) -- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm) -- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm) - -**Documentation updates:** - -- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd) -- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove) -- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins) -- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov) -- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins) -- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski) -- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove) -- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove) -- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove) -- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove) -- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm) -- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm) -- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm) -- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4) -- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm) -- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove) - -**Merged pull requests:** - -- PyBallista - Python SQL client for Ballista [#970](https://github.com/apache/datafusion-ballista/pull/970) (andygrove) -- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd) -- [Python] Add `read_csv` and `read_parquet` methods [#976](https://github.com/apache/datafusion-ballista/pull/976) (andygrove) -- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove) -- [Python] Add more methods to SessionContext [#978](https://github.com/apache/datafusion-ballista/pull/978) (andygrove) -- [Python] Add `execute_logical_plan` to context [#972](https://github.com/apache/datafusion-ballista/pull/972) (andygrove) -- Use correct product name in docs [#975](https://github.com/apache/datafusion-ballista/pull/975) (andygrove) -- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins) -- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov) -- Upgrade Rust version to 1.72 to keep the same as DataFusion v35 [#982](https://github.com/apache/datafusion-ballista/pull/982) (haoxins) -- build: Fix the ballista-cli Dockerfile [#981](https://github.com/apache/datafusion-ballista/pull/981) (haoxins) -- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov) -- Remove some hard-coded gRPC max message sizes [#984](https://github.com/apache/datafusion-ballista/pull/984) (andygrove) -- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins) -- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski) -- Fix ExecutorLost event debug info [#988](https://github.com/apache/datafusion-ballista/pull/988) (lewiszlw) -- Fix shuffle writer test [#998](https://github.com/apache/datafusion-ballista/pull/998) (Jefffrey) -- Bump graphviz-rust from 0.6.1 to 0.8.0 [#999](https://github.com/apache/datafusion-ballista/pull/999) (Jefffrey) -- Add rust-toolchain.toml for clarity [#1014](https://github.com/apache/datafusion-ballista/pull/1014) (scnerd) -- Fix executor metadata decode bug [#1004](https://github.com/apache/datafusion-ballista/pull/1004) (lewiszlw) -- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove) -- Fix Ballista rust.yml github workflow [#1026](https://github.com/apache/datafusion-ballista/pull/1026) (RaphaelMarinier) -- Bump datafusion to 36.0.0 and make ballista compatible with it. [#1027](https://github.com/apache/datafusion-ballista/pull/1027) (RaphaelMarinier) -- Make Ballista compatible with Datafusion 37.0.0 (from 36.0.0) [#1031](https://github.com/apache/datafusion-ballista/pull/1031) (RaphaelMarinier) -- Fixes Setting Job Name Not Reflected in Ballista UI [#1039](https://github.com/apache/datafusion-ballista/pull/1039) (athultr1997) -- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove) -- [Docs] fix good_first_issue link in the contribution md doc [#1022](https://github.com/apache/datafusion-ballista/pull/1022) (Almaz-KG) -- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove) -- Fix job hangs when partition count of plan is zero [#1024](https://github.com/apache/datafusion-ballista/pull/1024) (lewiszlw) -- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove) -- Fix cargo build [#1045](https://github.com/apache/datafusion-ballista/pull/1045) (andygrove) -- fix docker build in CI [#1046](https://github.com/apache/datafusion-ballista/pull/1046) (andygrove) -- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove) -- Bump actions/setup-node from 3 to 4 [#909](https://github.com/apache/datafusion-ballista/pull/909) (dependabot[bot]) -- Bump actions/cache from 3 to 4 [#958](https://github.com/apache/datafusion-ballista/pull/958) (dependabot[bot]) -- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove) -- Update datafusion protobuf definitions [#1057](https://github.com/apache/datafusion-ballista/pull/1057) (palaska) -- Fix regression with TPC-H benchmark [#1060](https://github.com/apache/datafusion-ballista/pull/1060) (andygrove) -- Upgrade to Datafusion 41 [#1062](https://github.com/apache/datafusion-ballista/pull/1062) (palaska) -- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm) -- Remove plugin subsystem [#1070](https://github.com/apache/datafusion-ballista/pull/1070) (milenkovicm) -- Remove CI folder [#1074](https://github.com/apache/datafusion-ballista/pull/1074) (milenkovicm) -- Code cleanup, move examples, remove unused files [#1075](https://github.com/apache/datafusion-ballista/pull/1075) (milenkovicm) -- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm) -- Remove key-value stores for scheduler persistence [#1077](https://github.com/apache/datafusion-ballista/pull/1077) (milenkovicm) -- Remove cache functionality [#1076](https://github.com/apache/datafusion-ballista/pull/1076) (milenkovicm) -- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm) -- Reorganise and remove dependencies [#1078](https://github.com/apache/datafusion-ballista/pull/1078) (milenkovicm) -- Promote keda and flight-sql to optional features [#1079](https://github.com/apache/datafusion-ballista/pull/1079) (milenkovicm) -- Update to datafusion 42 ... [#1080](https://github.com/apache/datafusion-ballista/pull/1080) (milenkovicm) -- #1086 solve examples errors [#1087](https://github.com/apache/datafusion-ballista/pull/1087) (tbar4) -- fix issue with not building python package ... [#1085](https://github.com/apache/datafusion-ballista/pull/1085) (milenkovicm) -- another round of code cleanup ... [#1089](https://github.com/apache/datafusion-ballista/pull/1089) (milenkovicm) -- Make rest-api optional feature ... [#1084](https://github.com/apache/datafusion-ballista/pull/1084) (milenkovicm) -- fix clippy issues after updating to rust 1.82 [#1090](https://github.com/apache/datafusion-ballista/pull/1090) (milenkovicm) -- Replace BallistaContext with SessionContext [#1088](https://github.com/apache/datafusion-ballista/pull/1088) (milenkovicm) -- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4) -- Ergonomic way to setup/configure `SessionContextExt` [#1096](https://github.com/apache/datafusion-ballista/pull/1096) (milenkovicm) -- Executor configuration extended .. [#1099](https://github.com/apache/datafusion-ballista/pull/1099) (milenkovicm) -- fix issue with executor registration ... [#1101](https://github.com/apache/datafusion-ballista/pull/1101) (milenkovicm) -- Deprecate `BallistaContext` [#1103](https://github.com/apache/datafusion-ballista/pull/1103) (milenkovicm) -- fix imports after un-rebased PR [#1106](https://github.com/apache/datafusion-ballista/pull/1106) (milenkovicm) -- Ballista proto cleanup [#1110](https://github.com/apache/datafusion-ballista/pull/1110) (milenkovicm) -- Update and move deps to workspace [#1109](https://github.com/apache/datafusion-ballista/pull/1109) (milenkovicm) -- Trim down `BallistaConfig` [#1108](https://github.com/apache/datafusion-ballista/pull/1108) (milenkovicm) -- Remove build-in object store registry [#1114](https://github.com/apache/datafusion-ballista/pull/1114) (milenkovicm) -- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm) -- support window functions [#1112](https://github.com/apache/datafusion-ballista/pull/1112) (onursatici) -- added a BallistaContext to ballista to allow for Remote or standalone [#1100](https://github.com/apache/datafusion-ballista/pull/1100) (tbar4) -- Decommission `BallistaContext` [#1119](https://github.com/apache/datafusion-ballista/pull/1119) (milenkovicm) -- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove) -- Make easier to create custom schedulers and executors [#1118](https://github.com/apache/datafusion-ballista/pull/1118) (milenkovicm) -- refactor: Move BallistaRegistry to better location [#1126](https://github.com/apache/datafusion-ballista/pull/1126) (milenkovicm) -- refactor: BallistaLogicalExtensionCodec refactoring and improvements [#1127](https://github.com/apache/datafusion-ballista/pull/1127) (milenkovicm) -- refactor: consolidate ballista tests [#1129](https://github.com/apache/datafusion-ballista/pull/1129) (milenkovicm) -- refactor: SessionStateExt and SessionConfigExt [#1130](https://github.com/apache/datafusion-ballista/pull/1130) (milenkovicm) -- chore: dependancy updates [#1131](https://github.com/apache/datafusion-ballista/pull/1131) (milenkovicm) -- chore: fix warning mimaloc warning when building [#1137](https://github.com/apache/datafusion-ballista/pull/1137) (milenkovicm) -- refactor: SessionBuilder to return Result<\_> [#1138](https://github.com/apache/datafusion-ballista/pull/1138) (milenkovicm) -- chore: remove unused cache\_ options from executor [#1140](https://github.com/apache/datafusion-ballista/pull/1140) (milenkovicm) -- updated maturin version and ccargo build to build yml [#1136](https://github.com/apache/datafusion-ballista/pull/1136) (tbar4) -- chore: Fix clippy issues after rust update (1.83.0) [#1143](https://github.com/apache/datafusion-ballista/pull/1143) (milenkovicm) -- Fix documentation example which still uses BallistaContext [#1145](https://github.com/apache/datafusion-ballista/pull/1145) (milenkovicm) -- Ballista proto cleanup [#1146](https://github.com/apache/datafusion-ballista/pull/1146) (milenkovicm) -- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm) -- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm) -- chore: dependency cleanup [#1150](https://github.com/apache/datafusion-ballista/pull/1150) (milenkovicm) -- Update DataFusion to 43 [#1125](https://github.com/apache/datafusion-ballista/pull/1125) (Dandandan) -- Reinstantiate join order optimization [#1122](https://github.com/apache/datafusion-ballista/pull/1122) (Dandandan) -- add partitioning scheme for unresolved shuffle and shuffle reader exec [#1144](https://github.com/apache/datafusion-ballista/pull/1144) (onursatici) -- chore: update py-df to 43.1 [#1152](https://github.com/apache/datafusion-ballista/pull/1152) (milenkovicm) -- chore: no need to run python test in rust [#1154](https://github.com/apache/datafusion-ballista/pull/1154) (milenkovicm) -- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm) - -## [0.12.0](https://github.com/apache/datafusion-ballista/tree/0.12.0) (2024-01-14) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.11.0...0.12.0) - -**Documentation updates:** - -- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins) - -**Merged pull requests:** - -- [minor] remove outdate todo [#683](https://github.com/apache/datafusion-ballista/pull/683) (Ted-Jiang) -- Add executor terminating status for graceful shutdown [#667](https://github.com/apache/datafusion-ballista/pull/667) (thinkharderdev) -- Allow `BallistaContext::read_*` methods to read multiple paths. [#679](https://github.com/apache/datafusion-ballista/pull/679) (luckylsk34) -- Update scheduler.md [#657](https://github.com/apache/datafusion-ballista/pull/657) (psvri) -- Mark `SchedulerState` as pub [#688](https://github.com/apache/datafusion-ballista/pull/688) (Dandandan) -- Update graphviz-rust requirement from 0.5.0 to 0.6.1 [#651](https://github.com/apache/datafusion-ballista/pull/651) (dependabot[bot]) -- Upgrade DataFusion to 19.0.0 [#691](https://github.com/apache/datafusion-ballista/pull/691) (r4ntix) -- Update release docs [#692](https://github.com/apache/datafusion-ballista/pull/692) (andygrove) -- Mark `SchedulerServer::with_task_launcher` as pub [#695](https://github.com/apache/datafusion-ballista/pull/695) (Dandandan) -- Make task_manager pub [#698](https://github.com/apache/datafusion-ballista/pull/698) (Dandandan) -- Add ExecutionEngine abstraction [#687](https://github.com/apache/datafusion-ballista/pull/687) (andygrove) -- Allow accessing s3 locations in client mode [#700](https://github.com/apache/datafusion-ballista/pull/700) (luckylsk34) -- git clone branch incorrect [#699](https://github.com/apache/datafusion-ballista/pull/699) (BubbaJoe) -- Fix for error message during testing [#707](https://github.com/apache/datafusion-ballista/pull/707) (yahoNanJing) -- Upgrade datafusion to 20.0.0 & sqlparser to to 0.32.0 [#711](https://github.com/apache/datafusion-ballista/pull/711) (r4ntix) -- Update README.md [#729](https://github.com/apache/datafusion-ballista/pull/729) (jiangzhx) -- Update link to scheduler proto file in dev docs [#713](https://github.com/apache/datafusion-ballista/pull/713) (JAicewizard) -- Fix `show tables` fails [#715](https://github.com/apache/datafusion-ballista/pull/715) (r4ntix) -- Remove redundant fields in ExecutorManager [#728](https://github.com/apache/datafusion-ballista/pull/728) (yahoNanJing) -- Fix parameter '--config-backend' to '--cluster-backend' [#720](https://github.com/apache/datafusion-ballista/pull/720) (paolorechia) -- Upgrade DataFusion to 21.0.0 [#727](https://github.com/apache/datafusion-ballista/pull/727) (r4ntix) -- [minor] remove useless brackets [#739](https://github.com/apache/datafusion-ballista/pull/739) (Ted-Jiang) -- Only decode plan in `LaunchMultiTaskParams` once [#743](https://github.com/apache/datafusion-ballista/pull/743) (Dandandan) -- Upgrade DataFusion to 22.0.0 [#740](https://github.com/apache/datafusion-ballista/pull/740) (r4ntix) -- [feature] support shuffle read with retry when facing IO error. [#738](https://github.com/apache/datafusion-ballista/pull/738) (Ted-Jiang) -- [log] Print long running task status. [#750](https://github.com/apache/datafusion-ballista/pull/750) (Ted-Jiang) -- Upgrade DataFusion to 23.0.0 [#755](https://github.com/apache/datafusion-ballista/pull/755) (yahoNanJing) -- Fix plan metrics length and stage metrics length not match [#764](https://github.com/apache/datafusion-ballista/pull/764) (yahoNanJing) -- added match arms to create ClusterStorageConfig [#766](https://github.com/apache/datafusion-ballista/pull/766) (BokarevNik) -- [Improve] refactor the offer_reservation avoid wait result [#760](https://github.com/apache/datafusion-ballista/pull/760) (Ted-Jiang) -- [fea] Avoid multithreaded write lock conflicts in event queue. [#754](https://github.com/apache/datafusion-ballista/pull/754) (Ted-Jiang) -- Upgrade DataFusion to 24.0.0, Object_Store to 0.5.6 [#769](https://github.com/apache/datafusion-ballista/pull/769) (r4ntix) -- Refine create_datafusion_context() [#778](https://github.com/apache/datafusion-ballista/pull/778) (yahoNanJing) -- Remove output_partitioning for task definition [#776](https://github.com/apache/datafusion-ballista/pull/776) (yahoNanJing) -- Upgrade DataFusion to 25.0.0 [#779](https://github.com/apache/datafusion-ballista/pull/779) (r4ntix) -- Disable the ansi feature of tracing-subscriber [#784](https://github.com/apache/datafusion-ballista/pull/784) (yahoNanJing) -- Add config grpc_server_max_decoding_message_size to make the maximum size of a decoded message at the grpc server side configurable [#782](https://github.com/apache/datafusion-ballista/pull/782) (yahoNanJing) -- Fix nodejs issues in Docker build [#731](https://github.com/apache/datafusion-ballista/pull/731) (jnaous) -- Upgrade node version to fix build in `main` [#794](https://github.com/apache/datafusion-ballista/pull/794) (avantgardnerio) -- Remove redundant mod session_registry [#792](https://github.com/apache/datafusion-ballista/pull/792) (yahoNanJing) -- Make last_seen_ts_threshold for getting alive executor at the scheduler side larger than the heartbeat time interval [#786](https://github.com/apache/datafusion-ballista/pull/786) (yahoNanJing) -- Remove the prometheus-metrics from the default feature [#788](https://github.com/apache/datafusion-ballista/pull/788) (yahoNanJing) -- Refine the ExecuteQuery grpc interface [#790](https://github.com/apache/datafusion-ballista/pull/790) (yahoNanJing) -- Add config to collect statistics, enable in TPC-H benchmark [#796](https://github.com/apache/datafusion-ballista/pull/796) (Dandandan) -- Add support for GCS data sources [#805](https://github.com/apache/datafusion-ballista/pull/805) (haoxins) -- Update DataFusion to 26 [#798](https://github.com/apache/datafusion-ballista/pull/798) (Dandandan) -- Issue 162 build docker image in ci [#716](https://github.com/apache/datafusion-ballista/pull/716) (paolorechia) -- Fix index out of bounds panic [#819](https://github.com/apache/datafusion-ballista/pull/819) (yahoNanJing) -- Refactor the TaskDefinition by changing encoding execution plan to the decoded one [#817](https://github.com/apache/datafusion-ballista/pull/817) (yahoNanJing) -- Fix ballista-cli docs [#800](https://github.com/apache/datafusion-ballista/pull/800) (jonahgao) -- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins) -- Implement the with_new_children for ShuffleReaderExec [#821](https://github.com/apache/datafusion-ballista/pull/821) (yahoNanJing) -- Update to point to the correct documentation [#838](https://github.com/apache/datafusion-ballista/pull/838) (dadepo) -- Remove ExecutorReservation and change the task assignment philosophy from executor first to task first [#823](https://github.com/apache/datafusion-ballista/pull/823) (yahoNanJing) -- Upgrade DataFusion to 27.0.0 [#834](https://github.com/apache/datafusion-ballista/pull/834) (r4ntix) -- Reduce the number of calls to `create_logical_plan` [#842](https://github.com/apache/datafusion-ballista/pull/842) (jonahgao) -- Bump semver from 5.7.1 to 5.7.2 in /ballista/scheduler/ui [#843](https://github.com/apache/datafusion-ballista/pull/843) (dependabot[bot]) -- Bump actions/labeler from 4.1.0 to 4.3.0 [#841](https://github.com/apache/datafusion-ballista/pull/841) (dependabot[bot]) -- Bump tough-cookie from 4.1.2 to 4.1.3 in /ballista/scheduler/ui [#840](https://github.com/apache/datafusion-ballista/pull/840) (dependabot[bot]) -- Update flatbuffers requirement from 22.9.29 to 23.5.26 [#801](https://github.com/apache/datafusion-ballista/pull/801) (dependabot[bot]) -- Update dirs requirement from 4.0.0 to 5.0.1 [#767](https://github.com/apache/datafusion-ballista/pull/767) (dependabot[bot]) -- Update libloading requirement from 0.7.3 to 0.8.0 [#761](https://github.com/apache/datafusion-ballista/pull/761) (dependabot[bot]) -- Introduce a cache crate supporting concurrent cache value loading [#825](https://github.com/apache/datafusion-ballista/pull/825) (yahoNanJing) -- Fix cargo clippy for latest rust version [#848](https://github.com/apache/datafusion-ballista/pull/848) (yahoNanJing) -- Introduce CachedBasedObjectStoreRegistry to use data source cache transparently [#827](https://github.com/apache/datafusion-ballista/pull/827) (yahoNanJing) -- Add ConsistentHash for node topology management [#830](https://github.com/apache/datafusion-ballista/pull/830) (yahoNanJing) -- Implement 3-phase consistent hash based task assignment policy [#833](https://github.com/apache/datafusion-ballista/pull/833) (yahoNanJing) -- Update tonic requirement from 0.8 to 0.9 [#733](https://github.com/apache/datafusion-ballista/pull/733) (dependabot[bot]) -- Update itertools requirement from 0.10 to 0.11 [#844](https://github.com/apache/datafusion-ballista/pull/844) (dependabot[bot]) -- Update etcd-client requirement from 0.10 to 0.11 [#845](https://github.com/apache/datafusion-ballista/pull/845) (dependabot[bot]) -- Update hashbrown requirement from 0.13 to 0.14 [#846](https://github.com/apache/datafusion-ballista/pull/846) (dependabot[bot]) -- Bump word-wrap from 1.2.3 to 1.2.4 in /ballista/scheduler/ui [#849](https://github.com/apache/datafusion-ballista/pull/849) (dependabot[bot]) -- Update hdfs requirement from 0.1.1 to 0.1.4 [#856](https://github.com/apache/datafusion-ballista/pull/856) (yahoNanJing) -- Update to DataFusion 28 [#858](https://github.com/apache/datafusion-ballista/pull/858) (Dandandan) -- Upgrade datafusion to 30.0.0 [#866](https://github.com/apache/datafusion-ballista/pull/866) (r4ntix) -- refactor: port get_scan_files to Ballista [#877](https://github.com/apache/datafusion-ballista/pull/877) (alamb) -- Upgrade datafusion to 31.0.0 [#878](https://github.com/apache/datafusion-ballista/pull/878) (r4ntix) -- Upgrade datafusion to 32.0.0 [#899](https://github.com/apache/datafusion-ballista/pull/899) (r4ntix) -- Update to DataFusion 33 [#900](https://github.com/apache/datafusion-ballista/pull/900) (Dandandan) -- Refactor lru mod, remove linked_hash_map [#918](https://github.com/apache/datafusion-ballista/pull/918) (PsiACE) -- Dynamically optimize aggregate (count) based on shuffle stats [#919](https://github.com/apache/datafusion-ballista/pull/919) (Dandandan) -- Use lz4 compression for shuffle files & flight stream, refactoring / improvements [#920](https://github.com/apache/datafusion-ballista/pull/920) (Dandandan) -- Make max encoding message size configurable [#928](https://github.com/apache/datafusion-ballista/pull/928) (andygrove) -- Set max message size to 16MB in gRPC clients [#931](https://github.com/apache/datafusion-ballista/pull/931) (andygrove) -- Upgrade to DataFusion 34.0.0-rc1 [#927](https://github.com/apache/datafusion-ballista/pull/927) (andygrove) -- Use official DF 34 release [#939](https://github.com/apache/datafusion-ballista/pull/939) (andygrove) -- Use StreamWriter instead of FileWriter [#943](https://github.com/apache/datafusion-ballista/pull/943) (avantgardnerio) -- Remove some TODO comments related to context fetching schemas from scheduler [#946](https://github.com/apache/datafusion-ballista/pull/946) (andygrove) -- Fix Docker build [#947](https://github.com/apache/datafusion-ballista/pull/947) (andygrove) -- Fix regression in DataFrame.write_xxx [#945](https://github.com/apache/datafusion-ballista/pull/945) (andygrove) - -## [0.11.0](https://github.com/apache/datafusion-ballista/tree/0.11.0) (2023-02-19) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.10.0...0.11.0) - -**Implemented enhancements:** - -- Remove `python` since it has been moved to its own repo, `datafusion-ballista-python` [\#653](https://github.com/apache/datafusion-ballista/issues/653) -- Add executor self-registration mechanism in the heartbeat service [\#648](https://github.com/apache/datafusion-ballista/issues/648) -- Upgrade to DataFusion 17 [\#638](https://github.com/apache/datafusion-ballista/issues/638) -- Move Python bindings to separate repo? [\#635](https://github.com/apache/datafusion-ballista/issues/635) -- Implement new release process [\#622](https://github.com/apache/datafusion-ballista/issues/622) -- Change default branch name from master to main [\#618](https://github.com/apache/datafusion-ballista/issues/618) -- Update latest datafusion dependency [\#610](https://github.com/apache/datafusion-ballista/issues/610) -- Implement optimizer rule to remove redundant repartitioning [\#608](https://github.com/apache/datafusion-ballista/issues/608) -- ballista-cli as \(docker\) images [\#600](https://github.com/apache/datafusion-ballista/issues/600) -- Update contributor guide [\#598](https://github.com/apache/datafusion-ballista/issues/598) -- Fix cargo clippy [\#570](https://github.com/apache/datafusion-ballista/issues/570) -- Support Alibaba Cloud OSS with ObjectStore [\#566](https://github.com/apache/datafusion-ballista/issues/566) -- Refactor `StateBackendClient` to be a higher-level interface [\#554](https://github.com/apache/datafusion-ballista/issues/554) -- Make it concurrently to launch tasks to executors [\#544](https://github.com/apache/datafusion-ballista/issues/544) -- Simplify docs [\#531](https://github.com/apache/datafusion-ballista/issues/531) -- Provide an in-memory StateBackend [\#505](https://github.com/apache/datafusion-ballista/issues/505) -- Add support for Azure blob storage [\#294](https://github.com/apache/datafusion-ballista/issues/294) -- Add a workflow to build the image and publish it to the package [\#71](https://github.com/apache/datafusion-ballista/issues/71) - -**Fixed bugs:** - -- Rust / Check Cargo.toml formatting \(amd64, stable\) \(pull_request\) Failing [\#662](https://github.com/apache/datafusion-ballista/issues/662) -- Protobuf parsing error [\#646](https://github.com/apache/datafusion-ballista/issues/646) -- jobs from python client not showing up in Scheduler UI [\#625](https://github.com/apache/datafusion-ballista/issues/625) -- ballista ui fails to build [\#594](https://github.com/apache/datafusion-ballista/issues/594) -- cargo build --release fails for ballista-scheduler [\#590](https://github.com/apache/datafusion-ballista/issues/590) -- docker build fails [\#589](https://github.com/apache/datafusion-ballista/issues/589) -- Multi-scheduler Job Starvation [\#585](https://github.com/apache/datafusion-ballista/issues/585) -- Cannot query file from S3 [\#559](https://github.com/apache/datafusion-ballista/issues/559) -- Benchmark q16 fails [\#373](https://github.com/apache/datafusion-ballista/issues/373) - -**Documentation updates:** - -- Check in benchmark image [\#647](https://github.com/apache/datafusion-ballista/pull/647) ([andygrove](https://github.com/andygrove)) -- MINOR: Fix benchmark image link [\#596](https://github.com/apache/datafusion-ballista/pull/596) ([andygrove](https://github.com/andygrove)) - -**Merged pull requests:** - -- Upgrade to DataFusion 18 [\#668](https://github.com/apache/datafusion-ballista/pull/668) ([andygrove](https://github.com/andygrove)) -- Enable physical plan round-trip tests [\#666](https://github.com/apache/datafusion-ballista/pull/666) ([andygrove](https://github.com/andygrove)) -- Upgrade to DataFusion 18.0.0-rc1 [\#664](https://github.com/apache/datafusion-ballista/pull/664) ([andygrove](https://github.com/andygrove)) -- add test_util to make examples work well [\#661](https://github.com/apache/datafusion-ballista/pull/661) ([jiangzhx](https://github.com/jiangzhx)) -- Minor refactor to reduce duplicate code [\#659](https://github.com/apache/datafusion-ballista/pull/659) ([andygrove](https://github.com/andygrove)) -- Cluster state refactor Part 2 [\#658](https://github.com/apache/datafusion-ballista/pull/658) ([thinkharderdev](https://github.com/thinkharderdev)) -- Remove `python` dir & python-related workflows [\#654](https://github.com/apache/datafusion-ballista/pull/654) ([iajoiner](https://github.com/iajoiner)) -- Add executor self-registration mechanism in the heartbeat service [\#649](https://github.com/apache/datafusion-ballista/pull/649) ([yahoNanJing](https://github.com/yahoNanJing)) -- Upgrade to DataFusion 17 [\#639](https://github.com/apache/datafusion-ballista/pull/639) ([avantgardnerio](https://github.com/avantgardnerio)) -- Upgrade to DataFusion 16 \(again\) [\#636](https://github.com/apache/datafusion-ballista/pull/636) ([avantgardnerio](https://github.com/avantgardnerio)) -- Update release process documentation [\#632](https://github.com/apache/datafusion-ballista/pull/632) ([andygrove](https://github.com/andygrove)) -- Implement new release process [\#623](https://github.com/apache/datafusion-ballista/pull/623) ([andygrove](https://github.com/andygrove)) -- Update contributor guide [\#617](https://github.com/apache/datafusion-ballista/pull/617) ([andygrove](https://github.com/andygrove)) -- Fix Cargo.toml format issue [\#616](https://github.com/apache/datafusion-ballista/pull/616) ([andygrove](https://github.com/andygrove)) -- Refactor scheduler main [\#615](https://github.com/apache/datafusion-ballista/pull/615) ([andygrove](https://github.com/andygrove)) -- Refactor executor main [\#614](https://github.com/apache/datafusion-ballista/pull/614) ([andygrove](https://github.com/andygrove)) -- Update datafusion dependency to the latest version [\#612](https://github.com/apache/datafusion-ballista/pull/612) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add support for Azure Blob Storage [\#599](https://github.com/apache/datafusion-ballista/pull/599) ([aidankovacic-8451](https://github.com/aidankovacic-8451)) -- Python: add method to get explain output as a string [\#593](https://github.com/apache/datafusion-ballista/pull/593) ([andygrove](https://github.com/andygrove)) -- Handle job resubmission [\#586](https://github.com/apache/datafusion-ballista/pull/586) ([thinkharderdev](https://github.com/thinkharderdev)) -- updated readme to contain correct versions of dependencies. [\#580](https://github.com/apache/datafusion-ballista/pull/580) ([saikrishna1-bidgely](https://github.com/saikrishna1-bidgely)) -- Update graphviz-rust requirement from 0.4.0 to 0.5.0 [\#574](https://github.com/apache/datafusion-ballista/pull/574) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Super minor spelling error [\#573](https://github.com/apache/datafusion-ballista/pull/573) ([jdye64](https://github.com/jdye64)) -- Fix cargo clippy [\#571](https://github.com/apache/datafusion-ballista/pull/571) ([yahoNanJing](https://github.com/yahoNanJing)) -- Support Alibaba Cloud OSS with ObjectStore [\#567](https://github.com/apache/datafusion-ballista/pull/567) ([r4ntix](https://github.com/r4ntix)) -- fix\(ui\): fix last seen [\#562](https://github.com/apache/datafusion-ballista/pull/562) ([duyet](https://github.com/duyet)) -- Cluster state refactor part 1 [\#560](https://github.com/apache/datafusion-ballista/pull/560) ([thinkharderdev](https://github.com/thinkharderdev)) -- Make it concurrently to launch tasks to executors [\#557](https://github.com/apache/datafusion-ballista/pull/557) ([yahoNanJing](https://github.com/yahoNanJing)) -- Update datafusion requirement from 14.0.0 to 15.0.0 [\#552](https://github.com/apache/datafusion-ballista/pull/552) ([yahoNanJing](https://github.com/yahoNanJing)) -- docs: fix style in the Helm readme [\#551](https://github.com/apache/datafusion-ballista/pull/551) ([haoxins](https://github.com/haoxins)) -- Fix Helm chart's image format [\#550](https://github.com/apache/datafusion-ballista/pull/550) ([haoxins](https://github.com/haoxins)) -- Update env_logger requirement from 0.9 to 0.10 [\#539](https://github.com/apache/datafusion-ballista/pull/539) ([dependabot[bot]](https://github.com/apps/dependabot)) -- only build docker images on rc tags [\#535](https://github.com/apache/datafusion-ballista/pull/535) ([andygrove](https://github.com/andygrove)) -- Remove `--locked` when building Python wheels [\#533](https://github.com/apache/datafusion-ballista/pull/533) ([andygrove](https://github.com/andygrove)) -- Bump actions/labeler from 4.0.2 to 4.1.0 [\#525](https://github.com/apache/datafusion-ballista/pull/525) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Provide a memory StateBackendClient [\#523](https://github.com/apache/datafusion-ballista/pull/523) ([yahoNanJing](https://github.com/yahoNanJing)) - -## [0.10.0](https://github.com/apache/datafusion-ballista/tree/0.10.0) (2022-11-18) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.9.0...0.10.0) - -**Implemented enhancements:** - -- Add user guide section on prometheus metrics [\#507](https://github.com/apache/datafusion-ballista/issues/507) -- Don't throw error when job path not exist in remove_job_data [\#502](https://github.com/apache/datafusion-ballista/issues/502) -- Fix clippy warning [\#494](https://github.com/apache/datafusion-ballista/issues/494) -- Use job_data_clean_up_interval_seconds == 0 to indicate executor_cleanup_enable [\#488](https://github.com/apache/datafusion-ballista/issues/488) -- Add a config for tracing log rolling policy for both scheduler and executor [\#486](https://github.com/apache/datafusion-ballista/issues/486) -- Set up repo where we can push benchmark results [\#473](https://github.com/apache/datafusion-ballista/issues/473) -- Make the delayed time interval for cleanup job data in both scheduler and executor configurable [\#469](https://github.com/apache/datafusion-ballista/issues/469) -- Add some validation for the remove_job_data grpc service [\#467](https://github.com/apache/datafusion-ballista/issues/467) -- Add ability to build docker images using `release-lto` profile [\#463](https://github.com/apache/datafusion-ballista/issues/463) -- Suggest users download \(rather than build\) the FlightSQL JDBC Driver [\#460](https://github.com/apache/datafusion-ballista/issues/460) -- Clean up legacy job shuffle data [\#459](https://github.com/apache/datafusion-ballista/issues/459) -- Add grpc service for the scheduler to make it able to be triggered by client explicitly [\#458](https://github.com/apache/datafusion-ballista/issues/458) -- Replace Mutex\ by using DashMap [\#448](https://github.com/apache/datafusion-ballista/issues/448) -- Refine log level [\#446](https://github.com/apache/datafusion-ballista/issues/446) -- Upgrade to DataFusion 14.0.0 [\#445](https://github.com/apache/datafusion-ballista/issues/445) -- Add a feature for hdfs3 [\#419](https://github.com/apache/datafusion-ballista/issues/419) -- Add optional flag which advertises host for Arrow Flight SQL [\#418](https://github.com/apache/datafusion-ballista/issues/418) -- Partitioning reasoning in DataFusion and Ballista [\#284](https://github.com/apache/datafusion-ballista/issues/284) -- Stop wasting time in CI on MIRI runs [\#283](https://github.com/apache/datafusion-ballista/issues/283) -- Publish Docker images as part of each release [\#236](https://github.com/apache/datafusion-ballista/issues/236) -- Cleanup job/stage status from TaskManager and clean up shuffle data after a period after JobFinished [\#185](https://github.com/apache/datafusion-ballista/issues/185) - -**Fixed bugs:** - -- build broken: configure_me_codegen retroactively reserved `bind_host` [\#519](https://github.com/apache/datafusion-ballista/issues/519) -- Return empty results for SQLs with order by [\#451](https://github.com/apache/datafusion-ballista/issues/451) -- ballista scheduler is not taken inline parameters into account [\#443](https://github.com/apache/datafusion-ballista/issues/443) -- \[FlightSQL\] Cannot connect with Tableau Desktop [\#428](https://github.com/apache/datafusion-ballista/issues/428) -- Benchmark q15 fails [\#372](https://github.com/apache/datafusion-ballista/issues/372) -- Incorrect documentation for building Ballista on Linux when using docker-compose [\#362](https://github.com/apache/datafusion-ballista/issues/362) -- Scheduler silently replaces `ParquetExec` with `EmptyExec` if data path is not correctly mounted in container [\#353](https://github.com/apache/datafusion-ballista/issues/353) -- SQL with order by limit returns nothing [\#334](https://github.com/apache/datafusion-ballista/issues/334) - -**Documentation updates:** - -- README updates [\#433](https://github.com/apache/datafusion-ballista/pull/433) ([andygrove](https://github.com/andygrove)) - -**Merged pull requests:** - -- configure_me_codegen retroactively reserved on our `bind_host` parame… [\#520](https://github.com/apache/datafusion-ballista/pull/520) ([avantgardnerio](https://github.com/avantgardnerio)) -- Bump actions/cache from 2 to 3 [\#517](https://github.com/apache/datafusion-ballista/pull/517) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Update graphviz-rust requirement from 0.3.0 to 0.4.0 [\#515](https://github.com/apache/datafusion-ballista/pull/515) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Add Prometheus metrics endpoint [\#511](https://github.com/apache/datafusion-ballista/pull/511) ([thinkharderdev](https://github.com/thinkharderdev)) -- Enable tests that work since upgrading to DataFusion 14 [\#510](https://github.com/apache/datafusion-ballista/pull/510) ([andygrove](https://github.com/andygrove)) -- Update hashbrown requirement from 0.12 to 0.13 [\#506](https://github.com/apache/datafusion-ballista/pull/506) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Don't throw error when job shuffle data path not exist in executor [\#503](https://github.com/apache/datafusion-ballista/pull/503) ([yahoNanJing](https://github.com/yahoNanJing)) -- Upgrade to DataFusion 14.0.0 and Arrow 26.0.0 [\#499](https://github.com/apache/datafusion-ballista/pull/499) ([andygrove](https://github.com/andygrove)) -- Fix clippy warning [\#495](https://github.com/apache/datafusion-ballista/pull/495) ([yahoNanJing](https://github.com/yahoNanJing)) -- Stop wasting time in CI on MIRI runs [\#491](https://github.com/apache/datafusion-ballista/pull/491) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Remove executor config executor_cleanup_enable and make the configuation name for executor cleanup more intuitive [\#489](https://github.com/apache/datafusion-ballista/pull/489) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add a config for tracing log rolling policy for both scheduler and executor [\#487](https://github.com/apache/datafusion-ballista/pull/487) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add grpc service of cleaning up job shuffle data for the scheduler to make it able to be triggered by client explicitly [\#485](https://github.com/apache/datafusion-ballista/pull/485) ([yahoNanJing](https://github.com/yahoNanJing)) -- \[Minor\] Bump DataFusion [\#480](https://github.com/apache/datafusion-ballista/pull/480) ([Dandandan](https://github.com/Dandandan)) -- Remove benchmark results from README [\#478](https://github.com/apache/datafusion-ballista/pull/478) ([andygrove](https://github.com/andygrove)) -- Update `flightsql.md` to provide correct instruction [\#476](https://github.com/apache/datafusion-ballista/pull/476) ([iajoiner](https://github.com/iajoiner)) -- Add support for Tableau [\#475](https://github.com/apache/datafusion-ballista/pull/475) ([avantgardnerio](https://github.com/avantgardnerio)) -- Add SchedulerConfig for the scheduler configurations, like event_loop_buffer_size, finished_job_data_clean_up_interval_seconds, finished_job_state_clean_up_interval_seconds [\#472](https://github.com/apache/datafusion-ballista/pull/472) ([yahoNanJing](https://github.com/yahoNanJing)) -- Bump DataFusion [\#471](https://github.com/apache/datafusion-ballista/pull/471) ([Dandandan](https://github.com/Dandandan)) -- Add some validation for remove_job_data in the executor server [\#468](https://github.com/apache/datafusion-ballista/pull/468) ([yahoNanJing](https://github.com/yahoNanJing)) -- Update documentation to reflect the release of the FlightSQL JDBC Driver [\#461](https://github.com/apache/datafusion-ballista/pull/461) ([avantgardnerio](https://github.com/avantgardnerio)) -- Bump DataFusion version [\#453](https://github.com/apache/datafusion-ballista/pull/453) ([andygrove](https://github.com/andygrove)) -- Add shuffle for SortPreservingMergeExec physical operator [\#452](https://github.com/apache/datafusion-ballista/pull/452) ([yahoNanJing](https://github.com/yahoNanJing)) -- Replace Mutex\ by using DashMap [\#449](https://github.com/apache/datafusion-ballista/pull/449) ([yahoNanJing](https://github.com/yahoNanJing)) -- Refine log level for trial info and periodically invoked places [\#447](https://github.com/apache/datafusion-ballista/pull/447) ([yahoNanJing](https://github.com/yahoNanJing)) -- MINOR: Add `set -e` to scripts, fix a typo [\#444](https://github.com/apache/datafusion-ballista/pull/444) ([andygrove](https://github.com/andygrove)) -- Add optional flag which advertises host for Arrow Flight SQL \#418 [\#442](https://github.com/apache/datafusion-ballista/pull/442) ([DaltonModlin](https://github.com/DaltonModlin)) -- Reorder joins after resolving stage inputs [\#441](https://github.com/apache/datafusion-ballista/pull/441) ([Dandandan](https://github.com/Dandandan)) -- Add a feature for hdfs3 [\#439](https://github.com/apache/datafusion-ballista/pull/439) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add Spark benchmarks [\#438](https://github.com/apache/datafusion-ballista/pull/438) ([andygrove](https://github.com/andygrove)) -- scheduler now verifies that `file://` ListingTable URLs are accessible [\#414](https://github.com/apache/datafusion-ballista/pull/414) ([andygrove](https://github.com/andygrove)) - -## [0.9.0](https://github.com/apache/datafusion-ballista/tree/0.9.0) (2022-10-22) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.8.0...0.9.0) - -**Implemented enhancements:** - -- Support count distinct aggregation function [\#411](https://github.com/apache/datafusion-ballista/issues/411) -- Use multi-task definition in pull-based execution loop [\#400](https://github.com/apache/datafusion-ballista/issues/400) -- Make the scheduler event loop buffer size configurable [\#397](https://github.com/apache/datafusion-ballista/issues/397) -- Remove active execution graph when the related job is successful or failed. [\#391](https://github.com/apache/datafusion-ballista/issues/391) -- Improve launch task efficiency by calling LaunchMultiTask [\#389](https://github.com/apache/datafusion-ballista/issues/389) -- Use `tokio::sync::Semaphore` to wait for available task slots [\#388](https://github.com/apache/datafusion-ballista/issues/388) -- stdout and file log level settings are inconsistent [\#385](https://github.com/apache/datafusion-ballista/issues/385) -- Use dedicated executor in pull based loop [\#383](https://github.com/apache/datafusion-ballista/issues/383) -- Avoid calling scheduler when the executor cannot accept new tasks [\#377](https://github.com/apache/datafusion-ballista/issues/377) -- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#371](https://github.com/apache/datafusion-ballista/issues/371) -- Switch to mimalloc and enable by default [\#369](https://github.com/apache/datafusion-ballista/issues/369) -- Integration test script should use docker-compose [\#364](https://github.com/apache/datafusion-ballista/issues/364) -- Use local shuffle reader in containerized environments [\#356](https://github.com/apache/datafusion-ballista/issues/356) -- Add `--ext` option to benchmark [\#352](https://github.com/apache/datafusion-ballista/issues/352) -- Add job cancel in the UI [\#350](https://github.com/apache/datafusion-ballista/issues/350) -- Using local shuffle reader avoid flight rpc call. [\#346](https://github.com/apache/datafusion-ballista/issues/346) -- Add a Helm Chart [\#321](https://github.com/apache/datafusion-ballista/issues/321) -- \[UI\] Show list of query stages with metrics [\#306](https://github.com/apache/datafusion-ballista/issues/306) -- \[UI\] Add ability to specify job name and have it show in the job listing page in the UI [\#277](https://github.com/apache/datafusion-ballista/issues/277) -- \[UI\] Add ability to download query plans in dot format [\#276](https://github.com/apache/datafusion-ballista/issues/276) -- \[UI\] Add ability to render query plans [\#275](https://github.com/apache/datafusion-ballista/issues/275) -- Add REST API documentation to User Guide [\#272](https://github.com/apache/datafusion-ballista/issues/272) -- Graceful shutdown: Handle `SIGTERM` [\#266](https://github.com/apache/datafusion-ballista/issues/266) -- \[EPIC\] Scheduler UI [\#265](https://github.com/apache/datafusion-ballista/issues/265) -- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#259](https://github.com/apache/datafusion-ballista/issues/259) -- Add a feature based object store provider [\#257](https://github.com/apache/datafusion-ballista/issues/257) -- Add docker build files [\#248](https://github.com/apache/datafusion-ballista/issues/248) -- Allow IDEs to recognize generated code [\#246](https://github.com/apache/datafusion-ballista/issues/246) -- Add user guide section on Flight SQL support [\#230](https://github.com/apache/datafusion-ballista/issues/230) -- `dev/release/README.md` is outdated [\#228](https://github.com/apache/datafusion-ballista/issues/228) -- Make ShuffleReaderExec output less verbose [\#211](https://github.com/apache/datafusion-ballista/issues/211) -- Add LaunchMultiTask rpc interface for executor [\#209](https://github.com/apache/datafusion-ballista/issues/209) -- Make executor fetch shuffle partition data in parallel [\#208](https://github.com/apache/datafusion-ballista/issues/208) -- Concurrency control and rate limit during shuffle reader [\#195](https://github.com/apache/datafusion-ballista/issues/195) -- Update User Guide [\#160](https://github.com/apache/datafusion-ballista/issues/160) -- Ballista 0.8.0 Release [\#159](https://github.com/apache/datafusion-ballista/issues/159) -- Save encoded execution plan in the ExecutionStage to reduce cost of task serialization and deserialization [\#142](https://github.com/apache/datafusion-ballista/issues/142) -- Failed task retry [\#140](https://github.com/apache/datafusion-ballista/issues/140) -- Redefine the executor task slots [\#132](https://github.com/apache/datafusion-ballista/issues/132) -- Use ArrowFlight bearer token auth to create a session key for FlightSql clients [\#112](https://github.com/apache/datafusion-ballista/issues/112) -- Leverage Atomic for the in-memory states in Scheduler [\#101](https://github.com/apache/datafusion-ballista/issues/101) -- Introduce the object stores in datafusion-contrib as optional features [\#87](https://github.com/apache/datafusion-ballista/issues/87) -- Support multiple paths for ListingTableScanNode [\#75](https://github.com/apache/datafusion-ballista/issues/75) -- Need clean up intermediate data in Ballista [\#9](https://github.com/apache/datafusion-ballista/issues/9) -- Ballista does not support external file systems [\#10](https://github.com/apache/datafusion-ballista/issues/10) - -**Fixed bugs:** - -- Build errors in ./dev/build-ballista-rust.sh [\#407](https://github.com/apache/datafusion-ballista/issues/407) -- The Ballista Scheduler Dockerfile copies a file that no longer exists [\#402](https://github.com/apache/datafusion-ballista/issues/402) -- Benchmark q20 fails [\#374](https://github.com/apache/datafusion-ballista/issues/374) -- Integration tests fail [\#360](https://github.com/apache/datafusion-ballista/issues/360) -- Helm deploy fails [\#344](https://github.com/apache/datafusion-ballista/issues/344) -- Executor get stopped unexpected [\#333](https://github.com/apache/datafusion-ballista/issues/333) -- Executor poll work loop failure [\#311](https://github.com/apache/datafusion-ballista/issues/311) -- Queries with `LIMIT` are failing with "PhysicalExtensionCodec is not provided" [\#300](https://github.com/apache/datafusion-ballista/issues/300) -- Schema inference does not work in Ballista-cli with a remote context [\#287](https://github.com/apache/datafusion-ballista/issues/287) -- There are bugs in the yarn build github misses but break our internal build [\#270](https://github.com/apache/datafusion-ballista/issues/270) -- Race condition running docker-compose [\#267](https://github.com/apache/datafusion-ballista/issues/267) -- Scheduler UI not working in Docker image [\#250](https://github.com/apache/datafusion-ballista/issues/250) -- Use bind host rather than the external host for starting a local executor service [\#244](https://github.com/apache/datafusion-ballista/issues/244) -- Initial query stages read parquet files and repartition them needlessly [\#243](https://github.com/apache/datafusion-ballista/issues/243) -- Cannot build Docker images on macOS 12.5.1 with M1 chip [\#234](https://github.com/apache/datafusion-ballista/issues/234) -- CLI uses DataFusion context if no host or port are provided [\#219](https://github.com/apache/datafusion-ballista/issues/219) -- Unsupported binary operator `StringConcat` [\#201](https://github.com/apache/datafusion-ballista/issues/201) -- Ballista assumes all aggregate expressions are not DISTINCT [\#5](https://github.com/apache/datafusion-ballista/issues/5) -- Start ballista ui with docker, but it can not found ballista scheduler [\#11](https://github.com/apache/datafusion-ballista/issues/11) -- Cannot build Ballista docker images on Apple silicon [\#17](https://github.com/apache/datafusion-ballista/issues/17) - -**Documentation updates:** - -- Fixup links in README.md [\#366](https://github.com/apache/datafusion-ballista/pull/366) ([romanz](https://github.com/romanz)) -- Update README in preparation for 0.9.0 release [\#318](https://github.com/apache/datafusion-ballista/pull/318) ([andygrove](https://github.com/andygrove)) -- User Guide improvements [\#274](https://github.com/apache/datafusion-ballista/pull/274) ([andygrove](https://github.com/andygrove)) - -**Closed issues:** - -- Automatic version updates for github actions with dependabot [\#127](https://github.com/apache/datafusion-ballista/issues/127) - -**Merged pull requests:** - -- Return multiple tasks in poll_work based on free slots [\#429](https://github.com/apache/datafusion-ballista/pull/429) ([Dandandan](https://github.com/Dandandan)) -- Run integration tests as part of release verification script [\#426](https://github.com/apache/datafusion-ballista/pull/426) ([andygrove](https://github.com/andygrove)) -- Bump actions/setup-node from 2 to 3 [\#424](https://github.com/apache/datafusion-ballista/pull/424) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump actions/setup-python from 2 to 4 [\#423](https://github.com/apache/datafusion-ballista/pull/423) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump actions/checkout from 2 to 3 [\#422](https://github.com/apache/datafusion-ballista/pull/422) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump actions/download-artifact from 2 to 3 [\#421](https://github.com/apache/datafusion-ballista/pull/421) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump actions/upload-artifact from 2 to 3 [\#420](https://github.com/apache/datafusion-ballista/pull/420) ([dependabot[bot]](https://github.com/apps/dependabot)) -- MINOR: Fix yarn warnings [\#415](https://github.com/apache/datafusion-ballista/pull/415) ([andygrove](https://github.com/andygrove)) -- Fix q20 sql typo in benchmarks [\#409](https://github.com/apache/datafusion-ballista/pull/409) ([r4ntix](https://github.com/r4ntix)) -- MINOR: Add notes on Apache Reporter [\#401](https://github.com/apache/datafusion-ballista/pull/401) ([andygrove](https://github.com/andygrove)) -- Use local shuffle reader in containerized environments and some impro… [\#399](https://github.com/apache/datafusion-ballista/pull/399) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Make the scheduler event loop buffer size configurable [\#398](https://github.com/apache/datafusion-ballista/pull/398) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add RoundRobinLocal slots policy for caching executor data to avoid seld persistency [\#396](https://github.com/apache/datafusion-ballista/pull/396) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#395](https://github.com/apache/datafusion-ballista/pull/395) ([yahoNanJing](https://github.com/yahoNanJing)) -- Improve launch task efficiency by calling LaunchMultiTask [\#394](https://github.com/apache/datafusion-ballista/pull/394) ([yahoNanJing](https://github.com/yahoNanJing)) -- Cache encoded stage plan [\#393](https://github.com/apache/datafusion-ballista/pull/393) ([yahoNanJing](https://github.com/yahoNanJing)) -- Remove active execution graph when the related job is successful or failed [\#392](https://github.com/apache/datafusion-ballista/pull/392) ([yahoNanJing](https://github.com/yahoNanJing)) -- Update flatbuffers requirement from 2.1.2 to 22.9.29 [\#390](https://github.com/apache/datafusion-ballista/pull/390) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Unified the log level configuration behavior [\#386](https://github.com/apache/datafusion-ballista/pull/386) ([r4ntix](https://github.com/r4ntix)) -- Add DistinctCount support [\#384](https://github.com/apache/datafusion-ballista/pull/384) ([r4ntix](https://github.com/r4ntix)) -- Pull-based execution loop improvements [\#380](https://github.com/apache/datafusion-ballista/pull/380) ([Dandandan](https://github.com/Dandandan)) -- Fix latest commit [\#379](https://github.com/apache/datafusion-ballista/pull/379) ([Dandandan](https://github.com/Dandandan)) -- Avoid calling scheduler when the executor cannot accept new tasks [\#378](https://github.com/apache/datafusion-ballista/pull/378) ([Dandandan](https://github.com/Dandandan)) -- Switch to mimalloc and enable by default in executor [\#370](https://github.com/apache/datafusion-ballista/pull/370) ([Dandandan](https://github.com/Dandandan)) -- Benchmark looks for path with and without extension [\#354](https://github.com/apache/datafusion-ballista/pull/354) ([andygrove](https://github.com/andygrove)) -- Implement job cancellation in UI [\#349](https://github.com/apache/datafusion-ballista/pull/349) ([Dandandan](https://github.com/Dandandan)) -- Using local shuffle reader avoid flight rpc call. [\#347](https://github.com/apache/datafusion-ballista/pull/347) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Make helm deployable [\#345](https://github.com/apache/datafusion-ballista/pull/345) ([avantgardnerio](https://github.com/avantgardnerio)) -- Benchmark & UI improvements [\#343](https://github.com/apache/datafusion-ballista/pull/343) ([andygrove](https://github.com/andygrove)) -- Add `cancel_job` REST API [\#340](https://github.com/apache/datafusion-ballista/pull/340) ([tfeda](https://github.com/tfeda)) -- Fix labeler [\#337](https://github.com/apache/datafusion-ballista/pull/337) ([andygrove](https://github.com/andygrove)) -- Upgrade to DataFusion 13.0.0 [\#336](https://github.com/apache/datafusion-ballista/pull/336) ([andygrove](https://github.com/andygrove)) -- Check executor id consistency when receive stop executor request [\#335](https://github.com/apache/datafusion-ballista/pull/335) ([yahoNanJing](https://github.com/yahoNanJing)) -- Enable more benchmark serde tests [\#331](https://github.com/apache/datafusion-ballista/pull/331) ([andygrove](https://github.com/andygrove)) -- Downgrade `docker-compose.yaml` to version 3.3 so that we can support Ubuntu 20.04.4 LTS [\#329](https://github.com/apache/datafusion-ballista/pull/329) ([andygrove](https://github.com/andygrove)) -- update labeler [\#326](https://github.com/apache/datafusion-ballista/pull/326) ([andygrove](https://github.com/andygrove)) -- Upgrade to DataFusion 13.0.0-rc1 [\#325](https://github.com/apache/datafusion-ballista/pull/325) ([andygrove](https://github.com/andygrove)) -- Dependabot stop suggesting arrow and datafusion updates [\#324](https://github.com/apache/datafusion-ballista/pull/324) ([andygrove](https://github.com/andygrove)) -- Show job stages metrics [\#323](https://github.com/apache/datafusion-ballista/pull/323) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) -- Add helm chart [\#322](https://github.com/apache/datafusion-ballista/pull/322) ([avantgardnerio](https://github.com/avantgardnerio)) -- Atomic support for enhancement [\#319](https://github.com/apache/datafusion-ballista/pull/319) ([metesynnada](https://github.com/metesynnada)) -- Allow automatic schema inference when registering csv [\#313](https://github.com/apache/datafusion-ballista/pull/313) ([r4ntix](https://github.com/r4ntix)) -- Add ability to specify job name and have it show in the job listing page in the UI [\#312](https://github.com/apache/datafusion-ballista/pull/312) ([andygrove](https://github.com/andygrove)) -- Add REST API to generate DOT graph for individual query stage [\#310](https://github.com/apache/datafusion-ballista/pull/310) ([andygrove](https://github.com/andygrove)) -- \[UI\] Use tabbed pane with Queries and Executors tabs [\#309](https://github.com/apache/datafusion-ballista/pull/309) ([andygrove](https://github.com/andygrove)) -- REST API to get query stages [\#305](https://github.com/apache/datafusion-ballista/pull/305) ([andygrove](https://github.com/andygrove)) -- Add support for SortPreservingMergeExec; fix LIMIT bug [\#304](https://github.com/apache/datafusion-ballista/pull/304) ([andygrove](https://github.com/andygrove)) -- Add Python script to run benchmarks [\#302](https://github.com/apache/datafusion-ballista/pull/302) ([andygrove](https://github.com/andygrove)) -- \[UI\] Add ability to view query plans directly in the UI [\#301](https://github.com/apache/datafusion-ballista/pull/301) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) -- Update datafusion.proto [\#299](https://github.com/apache/datafusion-ballista/pull/299) ([andygrove](https://github.com/andygrove)) -- Replace function `from_proto_binary_op` from upstream [\#298](https://github.com/apache/datafusion-ballista/pull/298) ([askoa](https://github.com/askoa)) -- Fix dead link in contribution guideline readme file [\#297](https://github.com/apache/datafusion-ballista/pull/297) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) -- UI code cleanup [\#291](https://github.com/apache/datafusion-ballista/pull/291) ([KenSuenobu](https://github.com/KenSuenobu)) -- Add support for S3 data sources [\#290](https://github.com/apache/datafusion-ballista/pull/290) ([andygrove](https://github.com/andygrove)) -- Use latest datafusion [\#289](https://github.com/apache/datafusion-ballista/pull/289) ([andygrove](https://github.com/andygrove)) -- Fix documentation example [\#288](https://github.com/apache/datafusion-ballista/pull/288) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) -- Improve formatting of job status in UI [\#286](https://github.com/apache/datafusion-ballista/pull/286) ([andygrove](https://github.com/andygrove)) -- Enabled download of dot files from Download icon [\#279](https://github.com/apache/datafusion-ballista/pull/279) ([KenSuenobu](https://github.com/KenSuenobu)) -- Executor graceful shutdown: Handle SIGTERM [\#278](https://github.com/apache/datafusion-ballista/pull/278) ([mingmwang](https://github.com/mingmwang)) -- Also run yarn build to catch JavaScript errors in CI [\#271](https://github.com/apache/datafusion-ballista/pull/271) ([avantgardnerio](https://github.com/avantgardnerio)) -- Store sessions so users can register tables and query them through flight [\#269](https://github.com/apache/datafusion-ballista/pull/269) ([avantgardnerio](https://github.com/avantgardnerio)) -- Fix compose for Ian [\#268](https://github.com/apache/datafusion-ballista/pull/268) ([avantgardnerio](https://github.com/avantgardnerio)) -- Task level retry and Stage level retry [\#261](https://github.com/apache/datafusion-ballista/pull/261) ([mingmwang](https://github.com/mingmwang)) -- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#260](https://github.com/apache/datafusion-ballista/pull/260) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add a feature based object store provider [\#258](https://github.com/apache/datafusion-ballista/pull/258) ([yahoNanJing](https://github.com/yahoNanJing)) -- Make fetch shuffle partition data in parallel [\#256](https://github.com/apache/datafusion-ballista/pull/256) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add LaunchMultiTask rpc interface for executor [\#255](https://github.com/apache/datafusion-ballista/pull/255) ([yahoNanJing](https://github.com/yahoNanJing)) -- CLI uses ballista context instead of datafusion context in local mode [\#252](https://github.com/apache/datafusion-ballista/pull/252) ([r4ntix](https://github.com/r4ntix)) -- Fix Scheduler UI in Docker image [\#251](https://github.com/apache/datafusion-ballista/pull/251) ([andygrove](https://github.com/andygrove)) -- Generate into source folder to make IDEs happy [\#247](https://github.com/apache/datafusion-ballista/pull/247) ([avantgardnerio](https://github.com/avantgardnerio)) -- Use bind host rather than the external host for starting a local executor service [\#245](https://github.com/apache/datafusion-ballista/pull/245) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add REST endpoint to get DOT graph of a job [\#242](https://github.com/apache/datafusion-ballista/pull/242) ([andygrove](https://github.com/andygrove)) -- Add list of jobs to scheduler UI [\#241](https://github.com/apache/datafusion-ballista/pull/241) ([andygrove](https://github.com/andygrove)) -- Clean up job data on both Scheduler and Executor [\#188](https://github.com/apache/datafusion-ballista/pull/188) ([mingmwang](https://github.com/mingmwang)) -- Update etcd-client requirement from 0.9 to 0.10 [\#111](https://github.com/apache/datafusion-ballista/pull/111) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump terser from 4.8.0 to 4.8.1 in /ballista/ui/scheduler [\#91](https://github.com/apache/datafusion-ballista/pull/91) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump jsdom from 16.4.0 to 16.7.0 in /ballista/ui/scheduler [\#74](https://github.com/apache/datafusion-ballista/pull/74) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump numpy from 1.21.3 to 1.22.0 in /python [\#72](https://github.com/apache/datafusion-ballista/pull/72) ([dependabot[bot]](https://github.com/apps/dependabot)) - -## [0.8.0](https://github.com/apache/datafusion-ballista/tree/0.8.0) (2022-09-16) - -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.7.0...0.8.0) - -**Implemented enhancements:** - -- Executor should use all available cores by default [\#218](https://github.com/apache/datafusion-ballista/issues/218) -- Update task status to the task job curator scheduler [\#179](https://github.com/apache/datafusion-ballista/issues/179) -- update datafusion and arrow to 20.0.0 [\#176](https://github.com/apache/datafusion-ballista/issues/176) -- No scheduler logs when deployed to k8s [\#165](https://github.com/apache/datafusion-ballista/issues/165) -- Upgrade to DataFusion 11.0.0 [\#163](https://github.com/apache/datafusion-ballista/issues/163) -- Better encapsulation for ExecutionGraph [\#149](https://github.com/apache/datafusion-ballista/issues/149) -- A stage may act as the input of multiple stages [\#144](https://github.com/apache/datafusion-ballista/issues/144) -- Executor Lost handling [\#143](https://github.com/apache/datafusion-ballista/issues/143) -- Cancel a running query. [\#139](https://github.com/apache/datafusion-ballista/issues/139) -- Ignore the previous job_id inside fill_reservations\(\) [\#138](https://github.com/apache/datafusion-ballista/issues/138) -- Normalize the serialization and deserialization places of protobuf structs [\#137](https://github.com/apache/datafusion-ballista/issues/137) -- Remove revive offer event loop [\#136](https://github.com/apache/datafusion-ballista/issues/136) -- Remove Keyspace::QueuedJobs [\#133](https://github.com/apache/datafusion-ballista/issues/133) -- Spawn a thread for execution plan generation [\#131](https://github.com/apache/datafusion-ballista/issues/131) -- Introduce CuratorTaskManager for make an active job be curated by only one scheduler [\#130](https://github.com/apache/datafusion-ballista/issues/130) -- Using tokio tracing for log file [\#122](https://github.com/apache/datafusion-ballista/issues/122) -- Ballista Executor report plan/operators metrics to Ballista Scheduler when task finish [\#116](https://github.com/apache/datafusion-ballista/issues/116) -- Add timeout settings for Grpc Client [\#114](https://github.com/apache/datafusion-ballista/issues/114) -- Add log level config in ballista [\#102](https://github.com/apache/datafusion-ballista/issues/102) -- Use another channel to update the status of a task set for executor [\#96](https://github.com/apache/datafusion-ballista/issues/96) -- Add config for concurrent_task in executor [\#94](https://github.com/apache/datafusion-ballista/issues/94) -- Ballista should support Arrow FlightSQL [\#92](https://github.com/apache/datafusion-ballista/issues/92) -- Why not include the `ballista-cli` in the member of workspace [\#88](https://github.com/apache/datafusion-ballista/issues/88) -- Upgrade dependency of arrow-datafusion to commit d0d5564b8f689a01e542b8c1df829d74d0fab2b0 [\#84](https://github.com/apache/datafusion-ballista/issues/84) -- Support sled path in config file. [\#79](https://github.com/apache/datafusion-ballista/issues/79) -- Support for multi-scheduler deployments [\#39](https://github.com/apache/datafusion-ballista/issues/39) -- Ballista 0.7.0 Release [\#126](https://github.com/apache/datafusion-ballista/issues/126) -- Improvements to Ballista extensibility [\#8](https://github.com/apache/datafusion-ballista/issues/8) -- Implement Python bindings for BallistaContext [\#15](https://github.com/apache/datafusion-ballista/issues/15) - -**Fixed bugs:** - -- Run example fails via PushStaged mode [\#214](https://github.com/apache/datafusion-ballista/issues/214) -- Config settings in BallistaContext do not get passed to DataFusion context [\#213](https://github.com/apache/datafusion-ballista/issues/213) -- Start scheduler fails with arguments "-s PushStaged" [\#207](https://github.com/apache/datafusion-ballista/issues/207) -- FlightSQL is broken and CI isn't catching it [\#190](https://github.com/apache/datafusion-ballista/issues/190) -- Query fails with "NULL is invalid as a DataFusion scalar value" [\#180](https://github.com/apache/datafusion-ballista/issues/180) -- Executor doesn't compile, missing `tokio::signal` [\#171](https://github.com/apache/datafusion-ballista/issues/171) -- Unable to build master [\#76](https://github.com/apache/datafusion-ballista/issues/76) - -## [ballista-0.7.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.7.0) (2022-05-12) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.1.0-rc1...ballista-0.7.0) - -**Breaking changes:** - -- Make `ExecutionPlan::execute` Sync [\#2434](https://github.com/apache/arrow-datafusion/pull/2434) ([tustvold](https://github.com/tustvold)) -- Add `Expr::Exists` to represent EXISTS subquery expression [\#2339](https://github.com/apache/arrow-datafusion/pull/2339) ([andygrove](https://github.com/andygrove)) -- Remove dependency from `LogicalPlan::TableScan` to `ExecutionPlan` [\#2284](https://github.com/apache/arrow-datafusion/pull/2284) ([andygrove](https://github.com/andygrove)) -- Move logical expression type-coercion code from `physical-expr` crate to `expr` crate [\#2257](https://github.com/apache/arrow-datafusion/pull/2257) ([andygrove](https://github.com/andygrove)) -- feat: 2061 create external table ddl table partition cols [\#2099](https://github.com/apache/arrow-datafusion/pull/2099) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jychen7](https://github.com/jychen7)) -- Reorganize the project folders [\#2081](https://github.com/apache/arrow-datafusion/pull/2081) ([yahoNanJing](https://github.com/yahoNanJing)) -- Support more ScalarFunction in Ballista [\#2008](https://github.com/apache/arrow-datafusion/pull/2008) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Merge dataframe and dataframe imp [\#1998](https://github.com/apache/arrow-datafusion/pull/1998) ([vchag](https://github.com/vchag)) -- Rename `ExecutionContext` to `SessionContext`, `ExecutionContextState` to `SessionState`, add `TaskContext` to support multi-tenancy configurations - Part 1 [\#1987](https://github.com/apache/arrow-datafusion/pull/1987) ([mingmwang](https://github.com/mingmwang)) -- Add Coalesce function [\#1969](https://github.com/apache/arrow-datafusion/pull/1969) ([msathis](https://github.com/msathis)) -- Add Create Schema functionality in SQL [\#1959](https://github.com/apache/arrow-datafusion/pull/1959) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner)) -- remove sync constraint of SendableRecordBatchStream [\#1884](https://github.com/apache/arrow-datafusion/pull/1884) ([doki23](https://github.com/doki23)) - -**Implemented enhancements:** - -- Add `CREATE VIEW` [\#2279](https://github.com/apache/arrow-datafusion/pull/2279) ([matthewmturner](https://github.com/matthewmturner)) -- \[Ballista\] Support Union in ballista. [\#2098](https://github.com/apache/arrow-datafusion/pull/2098) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Add missing aggr_expr to PhysicalExprNode for Ballista. [\#1989](https://github.com/apache/arrow-datafusion/pull/1989) ([Ted-Jiang](https://github.com/Ted-Jiang)) - -**Fixed bugs:** - -- Ballista integration tests no longer work [\#2440](https://github.com/apache/arrow-datafusion/issues/2440) -- Ballista crates cannot be released from DafaFusion 7.0.0 source release [\#1980](https://github.com/apache/arrow-datafusion/issues/1980) -- protobuf OctetLength should be deserialized as octet_length, not length [\#1834](https://github.com/apache/arrow-datafusion/pull/1834) ([carols10cents](https://github.com/carols10cents)) - -**Documentation updates:** - -- MINOR: Make crate READMEs consistent [\#2437](https://github.com/apache/arrow-datafusion/pull/2437) ([andygrove](https://github.com/andygrove)) -- docs: Update the Ballista dev env instructions [\#2419](https://github.com/apache/arrow-datafusion/pull/2419) ([haoxins](https://github.com/haoxins)) -- Revise document of installing ballista pinned to specified version [\#2034](https://github.com/apache/arrow-datafusion/pull/2034) ([WinkerDu](https://github.com/WinkerDu)) -- Fix typos \(Datafusion -\> DataFusion\) [\#1993](https://github.com/apache/arrow-datafusion/pull/1993) ([andygrove](https://github.com/andygrove)) - -**Performance improvements:** - -- Introduce StageManager for managing tasks stage by stage [\#1983](https://github.com/apache/arrow-datafusion/pull/1983) ([yahoNanJing](https://github.com/yahoNanJing)) - -**Closed issues:** - -- Make expected result string in unit tests more readable [\#2412](https://github.com/apache/arrow-datafusion/issues/2412) -- remove duplicated `fn aggregate()` in aggregate expression tests [\#2399](https://github.com/apache/arrow-datafusion/issues/2399) -- split `distinct_expression.rs` into `count_distinct.rs` and `array_agg_distinct.rs` [\#2385](https://github.com/apache/arrow-datafusion/issues/2385) -- move sql tests in `context.rs` to corresponding test files in `datafustion/core/tests/sql` [\#2328](https://github.com/apache/arrow-datafusion/issues/2328) -- Date32/Date64 as join keys for merge join [\#2314](https://github.com/apache/arrow-datafusion/issues/2314) -- Error precision and scale for decimal coercion in logic comparison [\#2232](https://github.com/apache/arrow-datafusion/issues/2232) -- Support Multiple row layout [\#2188](https://github.com/apache/arrow-datafusion/issues/2188) -- Discussion: Is Ballista a standalone system or framework [\#1916](https://github.com/apache/arrow-datafusion/issues/1916) - -**Merged pull requests:** - -- MINOR: Enable multi-statement benchmark queries [\#2507](https://github.com/apache/arrow-datafusion/pull/2507) ([andygrove](https://github.com/andygrove)) -- Persist session configs in scheduler [\#2501](https://github.com/apache/arrow-datafusion/pull/2501) ([thinkharderdev](https://github.com/thinkharderdev)) -- Update to `sqlparser` `0.17.0` [\#2500](https://github.com/apache/arrow-datafusion/pull/2500) ([alamb](https://github.com/alamb)) -- Limit cpu cores used when generating changelog [\#2494](https://github.com/apache/arrow-datafusion/pull/2494) ([andygrove](https://github.com/andygrove)) -- MINOR: Parameterize changelog script [\#2484](https://github.com/apache/arrow-datafusion/pull/2484) ([jychen7](https://github.com/jychen7)) -- Fix stage key extraction [\#2472](https://github.com/apache/arrow-datafusion/pull/2472) ([thinkharderdev](https://github.com/thinkharderdev)) -- Add support for list_dir\(\) on local fs [\#2467](https://github.com/apache/arrow-datafusion/pull/2467) ([wjones127](https://github.com/wjones127)) -- minor: update versions and paths in changelog scripts [\#2429](https://github.com/apache/arrow-datafusion/pull/2429) ([andygrove](https://github.com/andygrove)) -- Fix Ballista executing during plan [\#2428](https://github.com/apache/arrow-datafusion/pull/2428) ([tustvold](https://github.com/tustvold)) -- Re-organize and rename aggregates physical plan [\#2388](https://github.com/apache/arrow-datafusion/pull/2388) ([yjshen](https://github.com/yjshen)) -- Upgrade to arrow 13 [\#2382](https://github.com/apache/arrow-datafusion/pull/2382) ([alamb](https://github.com/alamb)) -- Grouped Aggregate in row format [\#2375](https://github.com/apache/arrow-datafusion/pull/2375) ([yjshen](https://github.com/yjshen)) -- Stop optimizing queries twice [\#2369](https://github.com/apache/arrow-datafusion/pull/2369) ([andygrove](https://github.com/andygrove)) -- Bump follow-redirects from 1.13.2 to 1.14.9 in /ballista/ui/scheduler [\#2325](https://github.com/apache/arrow-datafusion/pull/2325) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Move FileType enum from sql module to logical_plan module [\#2290](https://github.com/apache/arrow-datafusion/pull/2290) ([andygrove](https://github.com/andygrove)) -- Add BatchPartitioner \(\#2285\) [\#2287](https://github.com/apache/arrow-datafusion/pull/2287) ([tustvold](https://github.com/tustvold)) -- Update uuid requirement from 0.8 to 1.0 [\#2280](https://github.com/apache/arrow-datafusion/pull/2280) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump async from 2.6.3 to 2.6.4 in /ballista/ui/scheduler [\#2277](https://github.com/apache/arrow-datafusion/pull/2277) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump minimist from 1.2.5 to 1.2.6 in /ballista/ui/scheduler [\#2276](https://github.com/apache/arrow-datafusion/pull/2276) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump url-parse from 1.5.1 to 1.5.10 in /ballista/ui/scheduler [\#2275](https://github.com/apache/arrow-datafusion/pull/2275) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump nanoid from 3.1.20 to 3.3.3 in /ballista/ui/scheduler [\#2274](https://github.com/apache/arrow-datafusion/pull/2274) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Update to Arrow 12.0.0, update tonic and prost [\#2253](https://github.com/apache/arrow-datafusion/pull/2253) ([alamb](https://github.com/alamb)) -- Add ExecutorMetricsCollector interface [\#2234](https://github.com/apache/arrow-datafusion/pull/2234) ([thinkharderdev](https://github.com/thinkharderdev)) -- minor: add editor config file [\#2224](https://github.com/apache/arrow-datafusion/pull/2224) ([jackwener](https://github.com/jackwener)) -- \[Ballista\] Enable ApproxPercentileWithWeight in Ballista and fill UT [\#2192](https://github.com/apache/arrow-datafusion/pull/2192) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- make nightly clippy happy [\#2186](https://github.com/apache/arrow-datafusion/pull/2186) ([xudong963](https://github.com/xudong963)) -- \[Ballista\]Make PhysicalAggregateExprNode has repeated PhysicalExprNode [\#2184](https://github.com/apache/arrow-datafusion/pull/2184) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Add LogicalPlan::SubqueryAlias [\#2172](https://github.com/apache/arrow-datafusion/pull/2172) ([andygrove](https://github.com/andygrove)) -- Implement fast path of with_new_children\(\) in ExecutionPlan [\#2168](https://github.com/apache/arrow-datafusion/pull/2168) ([mingmwang](https://github.com/mingmwang)) -- \[MINOR\] ignore suspicious slow test in Ballista [\#2167](https://github.com/apache/arrow-datafusion/pull/2167) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- enable explain for ballista [\#2163](https://github.com/apache/arrow-datafusion/pull/2163) ([doki23](https://github.com/doki23)) -- Add delimiter for create external table [\#2162](https://github.com/apache/arrow-datafusion/pull/2162) ([matthewmturner](https://github.com/matthewmturner)) -- Update sqlparser requirement from 0.15 to 0.16 [\#2152](https://github.com/apache/arrow-datafusion/pull/2152) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Add IF NOT EXISTS to `CREATE TABLE` and `CREATE EXTERNAL TABLE` [\#2143](https://github.com/apache/arrow-datafusion/pull/2143) ([matthewmturner](https://github.com/matthewmturner)) -- Update quarterly roadmap for Q2 [\#2133](https://github.com/apache/arrow-datafusion/pull/2133) ([matthewmturner](https://github.com/matthewmturner)) -- \[Ballista\] Add ballista plugin manager and UDF plugin [\#2131](https://github.com/apache/arrow-datafusion/pull/2131) ([gaojun2048](https://github.com/gaojun2048)) -- Serialize scalar UDFs in physical plan [\#2130](https://github.com/apache/arrow-datafusion/pull/2130) ([thinkharderdev](https://github.com/thinkharderdev)) -- doc: update release schedule [\#2110](https://github.com/apache/arrow-datafusion/pull/2110) ([jychen7](https://github.com/jychen7)) -- Reduce repetition in Decimal binary kernels, upgrade to arrow 11.1 [\#2107](https://github.com/apache/arrow-datafusion/pull/2107) ([alamb](https://github.com/alamb)) -- update zlib version to 1.2.12 [\#2106](https://github.com/apache/arrow-datafusion/pull/2106) ([waitingkuo](https://github.com/waitingkuo)) -- Add CREATE DATABASE command to SQL [\#2094](https://github.com/apache/arrow-datafusion/pull/2094) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner)) -- Refactor SessionContext, BallistaContext to support multi-tenancy configurations - Part 3 [\#2091](https://github.com/apache/arrow-datafusion/pull/2091) ([mingmwang](https://github.com/mingmwang)) -- Remove dependency of common for the storage crate [\#2076](https://github.com/apache/arrow-datafusion/pull/2076) ([yahoNanJing](https://github.com/yahoNanJing)) -- [MINOR] fix doc in `EXTRACT\(field FROM source\) [\#2074](https://github.com/apache/arrow-datafusion/pull/2074) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- \[Bug\]\[Datafusion\] fix TaskContext session_config bug [\#2070](https://github.com/apache/arrow-datafusion/pull/2070) ([gaojun2048](https://github.com/gaojun2048)) -- Short-circuit evaluation for `CaseWhen` [\#2068](https://github.com/apache/arrow-datafusion/pull/2068) ([yjshen](https://github.com/yjshen)) -- split datafusion-object-store module [\#2065](https://github.com/apache/arrow-datafusion/pull/2065) ([yahoNanJing](https://github.com/yahoNanJing)) -- Change log level for noisy logs [\#2060](https://github.com/apache/arrow-datafusion/pull/2060) ([thinkharderdev](https://github.com/thinkharderdev)) -- Update to arrow/parquet 11.0 [\#2048](https://github.com/apache/arrow-datafusion/pull/2048) ([alamb](https://github.com/alamb)) -- minor: format comments \(`//` to `//`\) [\#2047](https://github.com/apache/arrow-datafusion/pull/2047) ([jackwener](https://github.com/jackwener)) -- use cargo-tomlfmt to check Cargo.toml formatting in CI [\#2033](https://github.com/apache/arrow-datafusion/pull/2033) ([WinkerDu](https://github.com/WinkerDu)) -- Refactor SessionContext, SessionState and SessionConfig to support multi-tenancy configurations - Part 2 [\#2029](https://github.com/apache/arrow-datafusion/pull/2029) ([mingmwang](https://github.com/mingmwang)) -- Simplify prerequisites for running examples [\#2028](https://github.com/apache/arrow-datafusion/pull/2028) ([doki23](https://github.com/doki23)) -- Use SessionContext to parse Expr protobuf [\#2024](https://github.com/apache/arrow-datafusion/pull/2024) ([thinkharderdev](https://github.com/thinkharderdev)) -- Fix stuck issue for the load testing of Push-based task scheduling [\#2006](https://github.com/apache/arrow-datafusion/pull/2006) ([yahoNanJing](https://github.com/yahoNanJing)) -- Fixing a typo in documentation [\#1997](https://github.com/apache/arrow-datafusion/pull/1997) ([psvri](https://github.com/psvri)) -- Fix minor clippy issue [\#1995](https://github.com/apache/arrow-datafusion/pull/1995) ([alamb](https://github.com/alamb)) -- Make it possible to only scan part of a parquet file in a partition [\#1990](https://github.com/apache/arrow-datafusion/pull/1990) ([yjshen](https://github.com/yjshen)) -- Update Dockerfile to fix integration tests [\#1982](https://github.com/apache/arrow-datafusion/pull/1982) ([andygrove](https://github.com/andygrove)) -- Update sqlparser requirement from 0.14 to 0.15 [\#1966](https://github.com/apache/arrow-datafusion/pull/1966) ([dependabot[bot]](https://github.com/apps/dependabot)) -- fix logical conflict with protobuf [\#1958](https://github.com/apache/arrow-datafusion/pull/1958) ([alamb](https://github.com/alamb)) -- Update to arrow 10.0.0, pyo3 0.16 [\#1957](https://github.com/apache/arrow-datafusion/pull/1957) ([alamb](https://github.com/alamb)) -- update jit-related dependencies [\#1953](https://github.com/apache/arrow-datafusion/pull/1953) ([xudong963](https://github.com/xudong963)) -- Allow different types of query variables \(`@@var`\) rather than just string [\#1943](https://github.com/apache/arrow-datafusion/pull/1943) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([maxburke](https://github.com/maxburke)) -- Pruning serialization [\#1941](https://github.com/apache/arrow-datafusion/pull/1941) ([thinkharderdev](https://github.com/thinkharderdev)) -- Fix select from EmptyExec always return 0 row after optimizer passes [\#1938](https://github.com/apache/arrow-datafusion/pull/1938) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Introduce Ballista query stage scheduler [\#1935](https://github.com/apache/arrow-datafusion/pull/1935) ([yahoNanJing](https://github.com/yahoNanJing)) -- Add db benchmark script [\#1928](https://github.com/apache/arrow-datafusion/pull/1928) ([matthewmturner](https://github.com/matthewmturner)) -- fix a typo [\#1919](https://github.com/apache/arrow-datafusion/pull/1919) ([vchag](https://github.com/vchag)) -- \[MINOR\] Update copyright year in Docs [\#1918](https://github.com/apache/arrow-datafusion/pull/1918) ([alamb](https://github.com/alamb)) -- add metadata to DFSchema, close \#1806. [\#1914](https://github.com/apache/arrow-datafusion/pull/1914) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jiacai2050](https://github.com/jiacai2050)) -- Refactor scheduler state mod [\#1913](https://github.com/apache/arrow-datafusion/pull/1913) ([yahoNanJing](https://github.com/yahoNanJing)) -- Refactor the event channel [\#1912](https://github.com/apache/arrow-datafusion/pull/1912) ([yahoNanJing](https://github.com/yahoNanJing)) -- Refactor scheduler server [\#1911](https://github.com/apache/arrow-datafusion/pull/1911) ([yahoNanJing](https://github.com/yahoNanJing)) -- Clippy fix on nightly [\#1907](https://github.com/apache/arrow-datafusion/pull/1907) ([yjshen](https://github.com/yjshen)) -- Updated Rust version to 1.59 in all the files [\#1903](https://github.com/apache/arrow-datafusion/pull/1903) ([NaincyKumariKnoldus](https://github.com/NaincyKumariKnoldus)) -- Remove uneeded Mutex in Ballista Client [\#1898](https://github.com/apache/arrow-datafusion/pull/1898) ([alamb](https://github.com/alamb)) -- Create a `datafusion-proto` crate for datafusion protobuf serialization [\#1887](https://github.com/apache/arrow-datafusion/pull/1887) ([carols10cents](https://github.com/carols10cents)) -- Fix clippy lints [\#1885](https://github.com/apache/arrow-datafusion/pull/1885) ([HaoYang670](https://github.com/HaoYang670)) -- Separate cpu-bound \(query-execution\) and IO-bound\(heartbeat\) to … [\#1883](https://github.com/apache/arrow-datafusion/pull/1883) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- \[Minor\] Clean up DecimalArray API Usage [\#1869](https://github.com/apache/arrow-datafusion/pull/1869) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) -- Changes after went through "Datafusion as a library section" [\#1868](https://github.com/apache/arrow-datafusion/pull/1868) ([nonontb](https://github.com/nonontb)) -- Remove allow unused imports from ballista-core, then fix all warnings [\#1853](https://github.com/apache/arrow-datafusion/pull/1853) ([carols10cents](https://github.com/carols10cents)) -- Update to arrow 9.1.0 [\#1851](https://github.com/apache/arrow-datafusion/pull/1851) ([alamb](https://github.com/alamb)) -- move some tests out of context and into sql [\#1846](https://github.com/apache/arrow-datafusion/pull/1846) ([alamb](https://github.com/alamb)) -- Fix compiling ballista in standalone mode, add build to CI [\#1839](https://github.com/apache/arrow-datafusion/pull/1839) ([alamb](https://github.com/alamb)) -- Update documentation example for change in API [\#1812](https://github.com/apache/arrow-datafusion/pull/1812) ([alamb](https://github.com/alamb)) -- Refactor scheduler state with different management policy for volatile and stable states [\#1810](https://github.com/apache/arrow-datafusion/pull/1810) ([yahoNanJing](https://github.com/yahoNanJing)) -- DataFusion + Conbench Integration [\#1791](https://github.com/apache/arrow-datafusion/pull/1791) ([dianaclarke](https://github.com/dianaclarke)) -- Enable periodic cleanup of work_dir directories in ballista executor [\#1783](https://github.com/apache/arrow-datafusion/pull/1783) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Use`eq_dyn`, `neq_dyn`, `lt_dyn`, `lt_eq_dyn`, `gt_dyn`, `gt_eq_dyn` kernels from arrow [\#1475](https://github.com/apache/arrow-datafusion/pull/1475) ([alamb](https://github.com/alamb)) - -## [7.1.0-rc1](https://github.com/apache/arrow-datafusion/tree/7.1.0-rc1) (2022-04-10) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.0.0-rc2...7.1.0-rc1) - -**Implemented enhancements:** - -- Support substring with three arguments: \(str, from, for\) for DataFrame API and Ballista [\#2092](https://github.com/apache/arrow-datafusion/issues/2092) -- UnionAll support for Ballista [\#2032](https://github.com/apache/arrow-datafusion/issues/2032) -- Separate cpu-bound and IO-bound work in ballista-executor by using diff tokio runtime. [\#1770](https://github.com/apache/arrow-datafusion/issues/1770) -- \[Ballista\] Introduce DAGScheduler for better managing the stage-based task scheduling [\#1704](https://github.com/apache/arrow-datafusion/issues/1704) -- \[Ballista\] Support to better manage cluster state, like alive executors, executor available task slots, etc [\#1703](https://github.com/apache/arrow-datafusion/issues/1703) - -**Closed issues:** - -- Optimize memory usage pattern to avoid "double memory" behavior [\#2149](https://github.com/apache/arrow-datafusion/issues/2149) -- Document approx_percentile_cont_with_weight in users guide [\#2078](https://github.com/apache/arrow-datafusion/issues/2078) -- \[follow up\]cleaning up statements.remove\(0\) [\#1986](https://github.com/apache/arrow-datafusion/issues/1986) -- Formatting error on documentation for Python [\#1873](https://github.com/apache/arrow-datafusion/issues/1873) -- Remove duplicate tests from `test_const_evaluator_scalar_functions` [\#1727](https://github.com/apache/arrow-datafusion/issues/1727) -- Question: Is the Ballista project providing value to the overall DataFusion project? [\#1273](https://github.com/apache/arrow-datafusion/issues/1273) - -## [7.0.0-rc2](https://github.com/apache/arrow-datafusion/tree/7.0.0-rc2) (2022-02-14) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.0.0...7.0.0-rc2) - -## [7.0.0](https://github.com/apache/arrow-datafusion/tree/7.0.0) (2022-02-14) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/6.0.0-rc0...7.0.0) - -**Breaking changes:** - -- Update `ExecutionPlan` to know about sortedness and repartitioning optimizer pass respect the invariants [\#1776](https://github.com/apache/arrow-datafusion/pull/1776) ([alamb](https://github.com/alamb)) -- Update to `arrow 8.0.0` [\#1673](https://github.com/apache/arrow-datafusion/pull/1673) ([alamb](https://github.com/alamb)) - -**Implemented enhancements:** - -- Task assignment between Scheduler and Executors [\#1221](https://github.com/apache/arrow-datafusion/issues/1221) -- Add `approx_median()` aggregate function [\#1729](https://github.com/apache/arrow-datafusion/pull/1729) ([realno](https://github.com/realno)) -- \[Ballista\] Add Decimal128, Date64, TimestampSecond, TimestampMillisecond, Interv… [\#1659](https://github.com/apache/arrow-datafusion/pull/1659) ([gaojun2048](https://github.com/gaojun2048)) -- Add `corr` aggregate function [\#1561](https://github.com/apache/arrow-datafusion/pull/1561) ([realno](https://github.com/realno)) -- Add `covar`, `covar_pop` and `covar_samp` aggregate functions [\#1551](https://github.com/apache/arrow-datafusion/pull/1551) ([realno](https://github.com/realno)) -- Add `approx_quantile()` aggregation function [\#1539](https://github.com/apache/arrow-datafusion/pull/1539) ([domodwyer](https://github.com/domodwyer)) -- Initial MemoryManager and DiskManager APIs for query execution + External Sort implementation [\#1526](https://github.com/apache/arrow-datafusion/pull/1526) ([yjshen](https://github.com/yjshen)) -- Add `stddev` and `variance` [\#1525](https://github.com/apache/arrow-datafusion/pull/1525) ([realno](https://github.com/realno)) -- Add `rem` operation for Expr [\#1467](https://github.com/apache/arrow-datafusion/pull/1467) ([liukun4515](https://github.com/liukun4515)) -- Implement `array_agg` aggregate function [\#1300](https://github.com/apache/arrow-datafusion/pull/1300) ([viirya](https://github.com/viirya)) - -**Fixed bugs:** - -- Ballista context::tests::test_standalone_mode test fails [\#1020](https://github.com/apache/arrow-datafusion/issues/1020) -- \[Ballista\] Fix scheduler state mod bug [\#1655](https://github.com/apache/arrow-datafusion/pull/1655) ([gaojun2048](https://github.com/gaojun2048)) -- Pass local address host so we do not get mismatch between IPv4 and IP… [\#1466](https://github.com/apache/arrow-datafusion/pull/1466) ([thinkharderdev](https://github.com/thinkharderdev)) -- Add Timezone to Scalar::Time\* types, and better timezone awareness to Datafusion's time types [\#1455](https://github.com/apache/arrow-datafusion/pull/1455) ([maxburke](https://github.com/maxburke)) - -**Documentation updates:** - -- Add dependencies to ballista example documentation [\#1346](https://github.com/apache/arrow-datafusion/pull/1346) ([jgoday](https://github.com/jgoday)) -- \[MINOR\] Fix some typos. [\#1310](https://github.com/apache/arrow-datafusion/pull/1310) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- fix some clippy warnings from nightly channel [\#1277](https://github.com/apache/arrow-datafusion/pull/1277) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist)) - -**Performance improvements:** - -- Introduce push-based task scheduling for Ballista [\#1560](https://github.com/apache/arrow-datafusion/pull/1560) ([yahoNanJing](https://github.com/yahoNanJing)) - -**Closed issues:** - -- Track memory usage in Non Limited Operators [\#1569](https://github.com/apache/arrow-datafusion/issues/1569) -- \[Question\] Why does ballista store tables in the client instead of in the SchedulerServer [\#1473](https://github.com/apache/arrow-datafusion/issues/1473) -- Why use the expr types before coercion to get the result type? [\#1358](https://github.com/apache/arrow-datafusion/issues/1358) -- A problem about the projection_push_down optimizer gathers valid columns [\#1312](https://github.com/apache/arrow-datafusion/issues/1312) -- apply constant folding to `LogicalPlan::Values` [\#1170](https://github.com/apache/arrow-datafusion/issues/1170) -- reduce usage of `IntoIterator` in logical plan builder window fn [\#372](https://github.com/apache/arrow-datafusion/issues/372) - -**Merged pull requests:** - -- Fix verification scripts for 7.0.0 release [\#1830](https://github.com/apache/arrow-datafusion/pull/1830) ([alamb](https://github.com/alamb)) -- update README for ballista [\#1817](https://github.com/apache/arrow-datafusion/pull/1817) ([liukun4515](https://github.com/liukun4515)) -- Fix logical conflict [\#1801](https://github.com/apache/arrow-datafusion/pull/1801) ([alamb](https://github.com/alamb)) -- Improve the error message and UX of tpch benchmark program [\#1800](https://github.com/apache/arrow-datafusion/pull/1800) ([alamb](https://github.com/alamb)) -- Update to sqlparser 0.14 [\#1796](https://github.com/apache/arrow-datafusion/pull/1796) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) -- Update datafusion versions [\#1793](https://github.com/apache/arrow-datafusion/pull/1793) ([matthewmturner](https://github.com/matthewmturner)) -- Update datafusion to use arrow 9.0.0 [\#1775](https://github.com/apache/arrow-datafusion/pull/1775) ([alamb](https://github.com/alamb)) -- Update parking_lot requirement from 0.11 to 0.12 [\#1735](https://github.com/apache/arrow-datafusion/pull/1735) ([dependabot[bot]](https://github.com/apps/dependabot)) -- substitute `parking_lot::Mutex` for `std::sync::Mutex` [\#1720](https://github.com/apache/arrow-datafusion/pull/1720) ([xudong963](https://github.com/xudong963)) -- Create ListingTableConfig which includes file format and schema inference [\#1715](https://github.com/apache/arrow-datafusion/pull/1715) ([matthewmturner](https://github.com/matthewmturner)) -- Support `create_physical_expr` and `ExecutionContextState` or `DefaultPhysicalPlanner` for faster speed [\#1700](https://github.com/apache/arrow-datafusion/pull/1700) ([alamb](https://github.com/alamb)) -- Use NamedTempFile rather than `String` in DiskManager [\#1680](https://github.com/apache/arrow-datafusion/pull/1680) ([alamb](https://github.com/alamb)) -- Abstract over logical and physical plan representations in Ballista [\#1677](https://github.com/apache/arrow-datafusion/pull/1677) ([thinkharderdev](https://github.com/thinkharderdev)) -- upgrade clap to version 3 [\#1672](https://github.com/apache/arrow-datafusion/pull/1672) ([Jimexist](https://github.com/Jimexist)) -- Improve configuration and resource use of `MemoryManager` and `DiskManager` [\#1668](https://github.com/apache/arrow-datafusion/pull/1668) ([alamb](https://github.com/alamb)) -- Make `MemoryManager` and `MemoryStream` public [\#1664](https://github.com/apache/arrow-datafusion/pull/1664) ([yjshen](https://github.com/yjshen)) -- Consolidate Schema and RecordBatch projection [\#1638](https://github.com/apache/arrow-datafusion/pull/1638) ([alamb](https://github.com/alamb)) -- Update hashbrown requirement from 0.11 to 0.12 [\#1631](https://github.com/apache/arrow-datafusion/pull/1631) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Update etcd-client requirement from 0.7 to 0.8 [\#1626](https://github.com/apache/arrow-datafusion/pull/1626) ([dependabot[bot]](https://github.com/apps/dependabot)) -- update nightly version [\#1597](https://github.com/apache/arrow-datafusion/pull/1597) ([Jimexist](https://github.com/Jimexist)) -- Add support show tables and show columns for ballista [\#1593](https://github.com/apache/arrow-datafusion/pull/1593) ([gaojun2048](https://github.com/gaojun2048)) -- minor: improve the benchmark readme [\#1567](https://github.com/apache/arrow-datafusion/pull/1567) ([xudong963](https://github.com/xudong963)) -- Consolidate `batch_size` configuration in `ExecutionConfig`, `RuntimeConfig` and `PhysicalPlanConfig` [\#1562](https://github.com/apache/arrow-datafusion/pull/1562) ([yjshen](https://github.com/yjshen)) -- Update to rust 1.58 [\#1557](https://github.com/apache/arrow-datafusion/pull/1557) ([xudong963](https://github.com/xudong963)) -- support mathematics operation for decimal data type [\#1554](https://github.com/apache/arrow-datafusion/pull/1554) ([liukun4515](https://github.com/liukun4515)) -- Make call SchedulerServer::new once in ballista-scheduler process [\#1537](https://github.com/apache/arrow-datafusion/pull/1537) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Add load test command in tpch.rs. [\#1530](https://github.com/apache/arrow-datafusion/pull/1530) ([Ted-Jiang](https://github.com/Ted-Jiang)) -- Remove one copy of ballista datatype serialization code [\#1524](https://github.com/apache/arrow-datafusion/pull/1524) ([alamb](https://github.com/alamb)) -- Update to arrow-7.0.0 [\#1523](https://github.com/apache/arrow-datafusion/pull/1523) ([alamb](https://github.com/alamb)) -- Workaround build failure: Pin quote to 1.0.10 [\#1499](https://github.com/apache/arrow-datafusion/pull/1499) ([alamb](https://github.com/alamb)) -- add rfcs for datafusion [\#1490](https://github.com/apache/arrow-datafusion/pull/1490) ([xudong963](https://github.com/xudong963)) -- support comparison for decimal data type and refactor the binary coercion rule [\#1483](https://github.com/apache/arrow-datafusion/pull/1483) ([liukun4515](https://github.com/liukun4515)) -- Update arrow-rs to 6.4.0 and replace boolean comparison in datafusion with arrow compute kernel [\#1446](https://github.com/apache/arrow-datafusion/pull/1446) ([xudong963](https://github.com/xudong963)) -- support cast/try_cast for decimal: signed numeric to decimal [\#1442](https://github.com/apache/arrow-datafusion/pull/1442) ([liukun4515](https://github.com/liukun4515)) -- use 0.13 sql parser [\#1435](https://github.com/apache/arrow-datafusion/pull/1435) ([Jimexist](https://github.com/Jimexist)) -- Clarify communication on bi-weekly sync [\#1427](https://github.com/apache/arrow-datafusion/pull/1427) ([alamb](https://github.com/alamb)) -- Minimize features [\#1399](https://github.com/apache/arrow-datafusion/pull/1399) ([carols10cents](https://github.com/carols10cents)) -- Update rust vesion to 1.57 [\#1395](https://github.com/apache/arrow-datafusion/pull/1395) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963)) -- Add coercion rules for AggregateFunctions [\#1387](https://github.com/apache/arrow-datafusion/pull/1387) ([liukun4515](https://github.com/liukun4515)) -- upgrade the arrow-rs version [\#1385](https://github.com/apache/arrow-datafusion/pull/1385) ([liukun4515](https://github.com/liukun4515)) -- Extract logical plan: rename the plan name \(follow up\) [\#1354](https://github.com/apache/arrow-datafusion/pull/1354) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([liukun4515](https://github.com/liukun4515)) -- upgrade arrow-rs to 6.2.0 [\#1334](https://github.com/apache/arrow-datafusion/pull/1334) ([liukun4515](https://github.com/liukun4515)) -- Update release instructions [\#1331](https://github.com/apache/arrow-datafusion/pull/1331) ([alamb](https://github.com/alamb)) -- Extract Aggregate, Sort, and Join to struct from AggregatePlan [\#1326](https://github.com/apache/arrow-datafusion/pull/1326) ([matthewmturner](https://github.com/matthewmturner)) -- Extract `EmptyRelation`, `Limit`, `Values` from `LogicalPlan` [\#1325](https://github.com/apache/arrow-datafusion/pull/1325) ([liukun4515](https://github.com/liukun4515)) -- Extract CrossJoin, Repartition, Union in LogicalPlan [\#1322](https://github.com/apache/arrow-datafusion/pull/1322) ([liukun4515](https://github.com/liukun4515)) -- Extract Explain, Analyze, Extension in LogicalPlan as independent struct [\#1317](https://github.com/apache/arrow-datafusion/pull/1317) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963)) -- Extract CreateMemoryTable, DropTable, CreateExternalTable in LogicalPlan as independent struct [\#1311](https://github.com/apache/arrow-datafusion/pull/1311) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([liukun4515](https://github.com/liukun4515)) -- Extract Projection, Filter, Window in LogicalPlan as independent struct [\#1309](https://github.com/apache/arrow-datafusion/pull/1309) ([ic4y](https://github.com/ic4y)) -- Add PSQL comparison tests for except, intersect [\#1292](https://github.com/apache/arrow-datafusion/pull/1292) ([mrob95](https://github.com/mrob95)) -- Extract logical plans in LogicalPlan as independent struct: TableScan [\#1290](https://github.com/apache/arrow-datafusion/pull/1290) ([xudong963](https://github.com/xudong963)) - -## [6.0.0-rc0](https://github.com/apache/arrow-datafusion/tree/6.0.0-rc0) (2021-11-14) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/6.0.0...6.0.0-rc0) - -## [6.0.0](https://github.com/apache/arrow-datafusion/tree/6.0.0) (2021-11-14) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.6.0...6.0.0) - -## [ballista-0.6.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.6.0) (2021-11-13) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.5.0...ballista-0.6.0) - -**Breaking changes:** - -- File partitioning for ListingTable [\#1141](https://github.com/apache/arrow-datafusion/pull/1141) ([rdettai](https://github.com/rdettai)) -- Register tables in BallistaContext using TableProviders instead of Dataframe [\#1028](https://github.com/apache/arrow-datafusion/pull/1028) ([rdettai](https://github.com/rdettai)) -- Make TableProvider.scan\(\) and PhysicalPlanner::create_physical_plan\(\) async [\#1013](https://github.com/apache/arrow-datafusion/pull/1013) ([rdettai](https://github.com/rdettai)) -- Reorganize table providers by table format [\#1010](https://github.com/apache/arrow-datafusion/pull/1010) ([rdettai](https://github.com/rdettai)) -- Move CBOs and Statistics to physical plan [\#965](https://github.com/apache/arrow-datafusion/pull/965) ([rdettai](https://github.com/rdettai)) -- Update to sqlparser v 0.10.0 [\#934](https://github.com/apache/arrow-datafusion/pull/934) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) -- FilePartition and PartitionedFile for scanning flexibility [\#932](https://github.com/apache/arrow-datafusion/pull/932) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([yjshen](https://github.com/yjshen)) -- Improve SQLMetric APIs, port existing metrics [\#908](https://github.com/apache/arrow-datafusion/pull/908) ([alamb](https://github.com/alamb)) -- Add support for EXPLAIN ANALYZE [\#858](https://github.com/apache/arrow-datafusion/pull/858) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) -- Rename concurrency to target_partitions [\#706](https://github.com/apache/arrow-datafusion/pull/706) ([andygrove](https://github.com/andygrove)) - -**Implemented enhancements:** - -- Update datafusion-cli to support Ballista, or implement new ballista-cli [\#886](https://github.com/apache/arrow-datafusion/issues/886) -- Prepare Ballista crates for publishing [\#509](https://github.com/apache/arrow-datafusion/issues/509) -- Add drop table support [\#1266](https://github.com/apache/arrow-datafusion/pull/1266) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([viirya](https://github.com/viirya)) -- use arrow 6.1.0 [\#1255](https://github.com/apache/arrow-datafusion/pull/1255) ([Jimexist](https://github.com/Jimexist)) -- Add support for `create table as` via MemTable [\#1243](https://github.com/apache/arrow-datafusion/pull/1243) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan)) -- add values list expression [\#1165](https://github.com/apache/arrow-datafusion/pull/1165) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist)) -- Multiple files per partitions for CSV Avro Json [\#1138](https://github.com/apache/arrow-datafusion/pull/1138) ([rdettai](https://github.com/rdettai)) -- Implement INTERSECT & INTERSECT DISTINCT [\#1135](https://github.com/apache/arrow-datafusion/pull/1135) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963)) -- Simplify file struct abstractions [\#1120](https://github.com/apache/arrow-datafusion/pull/1120) ([rdettai](https://github.com/rdettai)) -- Implement `is [not] distinct from` [\#1117](https://github.com/apache/arrow-datafusion/pull/1117) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan)) -- add digest\(utf8, method\) function and refactor all current hash digest functions [\#1090](https://github.com/apache/arrow-datafusion/pull/1090) ([Jimexist](https://github.com/Jimexist)) -- \[crypto\] add `blake3` algorithm to `digest` function [\#1086](https://github.com/apache/arrow-datafusion/pull/1086) ([Jimexist](https://github.com/Jimexist)) -- \[crypto\] add blake2b and blake2s functions [\#1081](https://github.com/apache/arrow-datafusion/pull/1081) ([Jimexist](https://github.com/Jimexist)) -- Update sqlparser-rs to 0.11 [\#1052](https://github.com/apache/arrow-datafusion/pull/1052) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) -- remove hard coded partition count in ballista logicalplan deserialization [\#1044](https://github.com/apache/arrow-datafusion/pull/1044) ([xudong963](https://github.com/xudong963)) -- Indexed field access for List [\#1006](https://github.com/apache/arrow-datafusion/pull/1006) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki)) -- Update DataFusion to arrow 6.0 [\#984](https://github.com/apache/arrow-datafusion/pull/984) ([alamb](https://github.com/alamb)) -- Implement Display for Expr, improve operator display [\#971](https://github.com/apache/arrow-datafusion/pull/971) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner)) -- ObjectStore API to read from remote storage systems [\#950](https://github.com/apache/arrow-datafusion/pull/950) ([yjshen](https://github.com/yjshen)) -- fixes \#933 replace placeholder fmt_as fr ExecutionPlan impls [\#939](https://github.com/apache/arrow-datafusion/pull/939) ([tiphaineruy](https://github.com/tiphaineruy)) -- Support `NotLike` in Ballista [\#916](https://github.com/apache/arrow-datafusion/pull/916) ([Dandandan](https://github.com/Dandandan)) -- Avro Table Provider [\#910](https://github.com/apache/arrow-datafusion/pull/910) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki)) -- Add BaselineMetrics, Timestamp metrics, add for `CoalescePartitionsExec`, rename output_time -\> elapsed_compute [\#909](https://github.com/apache/arrow-datafusion/pull/909) ([alamb](https://github.com/alamb)) -- \[Ballista\] Add executor last seen info to the ui [\#895](https://github.com/apache/arrow-datafusion/pull/895) ([msathis](https://github.com/msathis)) -- add cross join support to ballista [\#891](https://github.com/apache/arrow-datafusion/pull/891) ([houqp](https://github.com/houqp)) -- Add Ballista support to DataFusion CLI [\#889](https://github.com/apache/arrow-datafusion/pull/889) ([andygrove](https://github.com/andygrove)) -- Add support for PostgreSQL regex match [\#870](https://github.com/apache/arrow-datafusion/pull/870) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([b41sh](https://github.com/b41sh)) - -**Fixed bugs:** - -- Test execution_plans::shuffle_writer::tests::test Fail [\#1040](https://github.com/apache/arrow-datafusion/issues/1040) -- Integration test fails to build docker images [\#918](https://github.com/apache/arrow-datafusion/issues/918) -- Ballista: Remove hard-coded concurrency from logical plan serde code [\#708](https://github.com/apache/arrow-datafusion/issues/708) -- How can I make ballista distributed compute work? [\#327](https://github.com/apache/arrow-datafusion/issues/327) -- fix subquery alias [\#1067](https://github.com/apache/arrow-datafusion/pull/1067) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963)) -- Fix compilation for ballista in stand-alone mode [\#1008](https://github.com/apache/arrow-datafusion/pull/1008) ([Igosuki](https://github.com/Igosuki)) - -**Documentation updates:** - -- Add Ballista roadmap [\#1166](https://github.com/apache/arrow-datafusion/pull/1166) ([andygrove](https://github.com/andygrove)) -- Adds note on compatible rust version [\#1097](https://github.com/apache/arrow-datafusion/pull/1097) ([1nF0rmed](https://github.com/1nF0rmed)) -- implement `approx_distinct` function using HyperLogLog [\#1087](https://github.com/apache/arrow-datafusion/pull/1087) ([Jimexist](https://github.com/Jimexist)) -- Improve User Guide [\#954](https://github.com/apache/arrow-datafusion/pull/954) ([andygrove](https://github.com/andygrove)) -- Update plan_query_stages doc [\#951](https://github.com/apache/arrow-datafusion/pull/951) ([rdettai](https://github.com/rdettai)) -- \[DataFusion\] - Add show and show_limit function for DataFrame [\#923](https://github.com/apache/arrow-datafusion/pull/923) ([francis-du](https://github.com/francis-du)) -- update docs related to protoc and optional syntax [\#902](https://github.com/apache/arrow-datafusion/pull/902) ([Jimexist](https://github.com/Jimexist)) -- Improve Ballista crate README content [\#878](https://github.com/apache/arrow-datafusion/pull/878) ([andygrove](https://github.com/andygrove)) - -**Performance improvements:** - -- optimize build profile for datafusion python binding, cli and ballista [\#1137](https://github.com/apache/arrow-datafusion/pull/1137) ([houqp](https://github.com/houqp)) - -**Closed issues:** - -- InList expr with NULL literals do not work [\#1190](https://github.com/apache/arrow-datafusion/issues/1190) -- update the homepage README to include values, `approx_distinct`, etc. [\#1171](https://github.com/apache/arrow-datafusion/issues/1171) -- \[Python\]: Inconsistencies with Python package name [\#1011](https://github.com/apache/arrow-datafusion/issues/1011) -- Wanting to contribute to project where to start? [\#983](https://github.com/apache/arrow-datafusion/issues/983) -- delete redundant code [\#973](https://github.com/apache/arrow-datafusion/issues/973) -- How to build DataFusion python wheel [\#853](https://github.com/apache/arrow-datafusion/issues/853) -- Produce a design for a metrics framework [\#21](https://github.com/apache/arrow-datafusion/issues/21) - -**Merged pull requests:** - -- \[nit\] simplify ballista executor `CollectExec` impl codes [\#1140](https://github.com/apache/arrow-datafusion/pull/1140) ([panarch](https://github.com/panarch)) - -For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md) - -## [ballista-0.5.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.5.0) (2021-08-10) - -[Full Changelog](https://github.com/apache/arrow-datafusion/compare/4.0.0...ballista-0.5.0) - -**Breaking changes:** - -- \[ballista\] support date_part and date_turnc ser/de, pass tpch 7 [\#840](https://github.com/apache/arrow-datafusion/pull/840) ([houqp](https://github.com/houqp)) -- Box ScalarValue:Lists, reduce size by half size [\#788](https://github.com/apache/arrow-datafusion/pull/788) ([alamb](https://github.com/alamb)) -- Support DataFrame.collect for Ballista DataFrames [\#785](https://github.com/apache/arrow-datafusion/pull/785) ([andygrove](https://github.com/andygrove)) -- JOIN conditions are order dependent [\#778](https://github.com/apache/arrow-datafusion/pull/778) ([seddonm1](https://github.com/seddonm1)) -- UnresolvedShuffleExec should represent a single shuffle [\#727](https://github.com/apache/arrow-datafusion/pull/727) ([andygrove](https://github.com/andygrove)) -- Ballista: Make shuffle partitions configurable in benchmarks [\#702](https://github.com/apache/arrow-datafusion/pull/702) ([andygrove](https://github.com/andygrove)) -- Rename MergeExec to CoalescePartitionsExec [\#635](https://github.com/apache/arrow-datafusion/pull/635) ([andygrove](https://github.com/andygrove)) -- Ballista: Rename QueryStageExec to ShuffleWriterExec [\#633](https://github.com/apache/arrow-datafusion/pull/633) ([andygrove](https://github.com/andygrove)) -- fix 593, reduce cloning by taking ownership in logical planner's `from` fn [\#610](https://github.com/apache/arrow-datafusion/pull/610) ([Jimexist](https://github.com/Jimexist)) -- fix join column handling logic for `On` and `Using` constraints [\#605](https://github.com/apache/arrow-datafusion/pull/605) ([houqp](https://github.com/houqp)) -- Move ballista standalone mode to client [\#589](https://github.com/apache/arrow-datafusion/pull/589) ([edrevo](https://github.com/edrevo)) -- Ballista: Implement map-side shuffle [\#543](https://github.com/apache/arrow-datafusion/pull/543) ([andygrove](https://github.com/andygrove)) -- ShuffleReaderExec now supports multiple locations per partition [\#541](https://github.com/apache/arrow-datafusion/pull/541) ([andygrove](https://github.com/andygrove)) -- Make external hostname in executor optional [\#232](https://github.com/apache/arrow-datafusion/pull/232) ([edrevo](https://github.com/edrevo)) -- Remove namespace from executors [\#75](https://github.com/apache/arrow-datafusion/pull/75) ([edrevo](https://github.com/edrevo)) -- Support qualified columns in queries [\#55](https://github.com/apache/arrow-datafusion/pull/55) ([houqp](https://github.com/houqp)) -- Read CSV format text from stdin or memory [\#54](https://github.com/apache/arrow-datafusion/pull/54) ([heymind](https://github.com/heymind)) -- Remove Ballista DataFrame [\#48](https://github.com/apache/arrow-datafusion/pull/48) ([andygrove](https://github.com/andygrove)) -- Use atomics for SQLMetric implementation, remove unused name field [\#25](https://github.com/apache/arrow-datafusion/pull/25) ([returnString](https://github.com/returnString)) - -**Implemented enhancements:** - -- Add crate documentation for Ballista crates [\#830](https://github.com/apache/arrow-datafusion/issues/830) -- Support DataFrame.collect for Ballista DataFrames [\#787](https://github.com/apache/arrow-datafusion/issues/787) -- Ballista: Prep for supporting shuffle correctly, part one [\#736](https://github.com/apache/arrow-datafusion/issues/736) -- Ballista: Implement physical plan serde for ShuffleWriterExec [\#710](https://github.com/apache/arrow-datafusion/issues/710) -- Ballista: Finish implementing shuffle mechanism [\#707](https://github.com/apache/arrow-datafusion/issues/707) -- Rename QueryStageExec to ShuffleWriterExec [\#542](https://github.com/apache/arrow-datafusion/issues/542) -- Ballista ShuffleReaderExec should be able to read from multiple locations per partition [\#540](https://github.com/apache/arrow-datafusion/issues/540) -- \[Ballista\] Use deployments in k8s user guide [\#473](https://github.com/apache/arrow-datafusion/issues/473) -- Ballista refactor QueryStageExec in preparation for map-side shuffle [\#458](https://github.com/apache/arrow-datafusion/issues/458) -- Ballista: Implement map-side of shuffle [\#456](https://github.com/apache/arrow-datafusion/issues/456) -- Refactor Ballista to separate Flight logic from execution logic [\#449](https://github.com/apache/arrow-datafusion/issues/449) -- Use published versions of arrow rather than github shas [\#393](https://github.com/apache/arrow-datafusion/issues/393) -- BallistaContext::collect\(\) logging is too noisy [\#352](https://github.com/apache/arrow-datafusion/issues/352) -- Update Ballista to use new physical plan formatter utility [\#343](https://github.com/apache/arrow-datafusion/issues/343) -- Add Ballista Getting Started documentation [\#329](https://github.com/apache/arrow-datafusion/issues/329) -- Remove references to ballistacompute Docker Hub repo [\#325](https://github.com/apache/arrow-datafusion/issues/325) -- Implement scalable distributed joins [\#63](https://github.com/apache/arrow-datafusion/issues/63) -- Remove hard-coded Ballista version from scripts [\#32](https://github.com/apache/arrow-datafusion/issues/32) -- Implement streaming versions of Dataframe.collect methods [\#789](https://github.com/apache/arrow-datafusion/pull/789) ([andygrove](https://github.com/andygrove)) -- Ballista shuffle is finally working as intended, providing scalable distributed joins [\#750](https://github.com/apache/arrow-datafusion/pull/750) ([andygrove](https://github.com/andygrove)) -- Update to use arrow 5.0 [\#721](https://github.com/apache/arrow-datafusion/pull/721) ([alamb](https://github.com/alamb)) -- Implement serde for ShuffleWriterExec [\#712](https://github.com/apache/arrow-datafusion/pull/712) ([andygrove](https://github.com/andygrove)) -- dedup using join column in wildcard expansion [\#678](https://github.com/apache/arrow-datafusion/pull/678) ([houqp](https://github.com/houqp)) -- Implement metrics for shuffle read and write [\#676](https://github.com/apache/arrow-datafusion/pull/676) ([andygrove](https://github.com/andygrove)) -- Remove hard-coded PartitionMode from Ballista serde [\#637](https://github.com/apache/arrow-datafusion/pull/637) ([andygrove](https://github.com/andygrove)) -- Ballista: Implement scalable distributed joins [\#634](https://github.com/apache/arrow-datafusion/pull/634) ([andygrove](https://github.com/andygrove)) -- Add Keda autoscaling for ballista in k8s [\#586](https://github.com/apache/arrow-datafusion/pull/586) ([edrevo](https://github.com/edrevo)) -- Add some resiliency to lost executors [\#568](https://github.com/apache/arrow-datafusion/pull/568) ([edrevo](https://github.com/edrevo)) -- Add `partition by` constructs in window functions and modify logical planning [\#501](https://github.com/apache/arrow-datafusion/pull/501) ([Jimexist](https://github.com/Jimexist)) -- Support anti join [\#482](https://github.com/apache/arrow-datafusion/pull/482) ([Dandandan](https://github.com/Dandandan)) -- add `order by` construct in window function and logical plans [\#463](https://github.com/apache/arrow-datafusion/pull/463) ([Jimexist](https://github.com/Jimexist)) -- Refactor Ballista executor so that FlightService delegates to an Executor struct [\#450](https://github.com/apache/arrow-datafusion/pull/450) ([andygrove](https://github.com/andygrove)) -- implement lead and lag built-in window function [\#429](https://github.com/apache/arrow-datafusion/pull/429) ([Jimexist](https://github.com/Jimexist)) -- Implement fmt_as for ShuffleReaderExec [\#400](https://github.com/apache/arrow-datafusion/pull/400) ([andygrove](https://github.com/andygrove)) -- Add window expression part 1 - logical and physical planning, structure, to/from proto, and explain, for empty over clause only [\#334](https://github.com/apache/arrow-datafusion/pull/334) ([Jimexist](https://github.com/Jimexist)) -- \[breaking change\] fix 265, log should be log10, and add ln [\#271](https://github.com/apache/arrow-datafusion/pull/271) ([Jimexist](https://github.com/Jimexist)) -- Allow table providers to indicate their type for catalog metadata [\#205](https://github.com/apache/arrow-datafusion/pull/205) ([returnString](https://github.com/returnString)) -- Add query 19 to TPC-H regression tests [\#59](https://github.com/apache/arrow-datafusion/pull/59) ([Dandandan](https://github.com/Dandandan)) -- Use arrow eq kernels in CaseWhen expression evaluation [\#52](https://github.com/apache/arrow-datafusion/pull/52) ([Dandandan](https://github.com/Dandandan)) -- Add option param for standalone mode [\#42](https://github.com/apache/arrow-datafusion/pull/42) ([djKooks](https://github.com/djKooks)) -- \[DataFusion\] Optimize hash join inner workings, null handling fix [\#24](https://github.com/apache/arrow-datafusion/pull/24) ([Dandandan](https://github.com/Dandandan)) -- \[Ballista\] Docker files for ui [\#22](https://github.com/apache/arrow-datafusion/pull/22) ([msathis](https://github.com/msathis)) - -**Fixed bugs:** - -- Ballista: TPC-H q3 @ SF=1000 never completes [\#835](https://github.com/apache/arrow-datafusion/issues/835) -- Ballista does not support MIN/MAX aggregate functions [\#832](https://github.com/apache/arrow-datafusion/issues/832) -- Ballista docker images fail to build [\#828](https://github.com/apache/arrow-datafusion/issues/828) -- Ballista: UnresolvedShuffleExec should only have a single stage_id [\#726](https://github.com/apache/arrow-datafusion/issues/726) -- Ballista integration tests are failing [\#623](https://github.com/apache/arrow-datafusion/issues/623) -- Integration test build failure due to arrow-rs using unstable feature [\#596](https://github.com/apache/arrow-datafusion/issues/596) -- `cargo build` cannot build the project [\#531](https://github.com/apache/arrow-datafusion/issues/531) -- ShuffleReaderExec does not get formatted correctly in displayable physical plan [\#399](https://github.com/apache/arrow-datafusion/issues/399) -- Implement serde for MIN and MAX [\#833](https://github.com/apache/arrow-datafusion/pull/833) ([andygrove](https://github.com/andygrove)) -- Ballista: Prep for fixing shuffle mechansim, part 1 [\#738](https://github.com/apache/arrow-datafusion/pull/738) ([andygrove](https://github.com/andygrove)) -- Ballista: Shuffle write bug fix [\#714](https://github.com/apache/arrow-datafusion/pull/714) ([andygrove](https://github.com/andygrove)) -- honor table name for csv/parquet scan in ballista plan serde [\#629](https://github.com/apache/arrow-datafusion/pull/629) ([houqp](https://github.com/houqp)) -- MINOR: Fix integration tests by adding datafusion-cli module to docker image [\#322](https://github.com/apache/arrow-datafusion/pull/322) ([andygrove](https://github.com/andygrove)) - -**Documentation updates:** - -- Add minimal crate documentation for Ballista crates [\#831](https://github.com/apache/arrow-datafusion/pull/831) ([andygrove](https://github.com/andygrove)) -- Add Ballista examples [\#775](https://github.com/apache/arrow-datafusion/pull/775) ([andygrove](https://github.com/andygrove)) -- Update ballista.proto link in architecture doc [\#502](https://github.com/apache/arrow-datafusion/pull/502) ([terrycorley](https://github.com/terrycorley)) -- Update k8s user guide to use deployments [\#474](https://github.com/apache/arrow-datafusion/pull/474) ([edrevo](https://github.com/edrevo)) -- use prettier to format md files [\#367](https://github.com/apache/arrow-datafusion/pull/367) ([Jimexist](https://github.com/Jimexist)) -- Make it easier for developers to find Ballista documentation [\#330](https://github.com/apache/arrow-datafusion/pull/330) ([andygrove](https://github.com/andygrove)) -- Instructions for cross-compiling Ballista to the Raspberry Pi [\#263](https://github.com/apache/arrow-datafusion/pull/263) ([andygrove](https://github.com/andygrove)) -- Add install guide in README [\#236](https://github.com/apache/arrow-datafusion/pull/236) ([djKooks](https://github.com/djKooks)) - -**Performance improvements:** - -- Ballista: Avoid sleeping between polling for tasks [\#698](https://github.com/apache/arrow-datafusion/pull/698) ([Dandandan](https://github.com/Dandandan)) -- Make BallistaContext::collect streaming [\#535](https://github.com/apache/arrow-datafusion/pull/535) ([edrevo](https://github.com/edrevo)) - -**Closed issues:** - -- Confirm git tagging strategy for releases [\#770](https://github.com/apache/arrow-datafusion/issues/770) -- arrow::util::pretty::pretty_format_batches missing [\#769](https://github.com/apache/arrow-datafusion/issues/769) -- move the `assert_batches_eq!` macros to a non part of datafusion [\#745](https://github.com/apache/arrow-datafusion/issues/745) -- fix an issue where aliases are not respected in generating downstream schemas in window expr [\#592](https://github.com/apache/arrow-datafusion/issues/592) -- make the planner to print more succinct and useful information in window function explain clause [\#526](https://github.com/apache/arrow-datafusion/issues/526) -- move window frame module to be in `logical_plan` [\#517](https://github.com/apache/arrow-datafusion/issues/517) -- use a more rust idiomatic way of handling nth_value [\#448](https://github.com/apache/arrow-datafusion/issues/448) -- Make Ballista not depend on arrow directly [\#446](https://github.com/apache/arrow-datafusion/issues/446) -- create a test with more than one partition for window functions [\#435](https://github.com/apache/arrow-datafusion/issues/435) -- Implement hash-partitioned hash aggregate [\#27](https://github.com/apache/arrow-datafusion/issues/27) -- Consider using GitHub pages for DataFusion/Ballista documentation [\#18](https://github.com/apache/arrow-datafusion/issues/18) -- Add Ballista to default cargo workspace [\#17](https://github.com/apache/arrow-datafusion/issues/17) -- Update "repository" in Cargo.toml [\#16](https://github.com/apache/arrow-datafusion/issues/16) -- Consolidate TPC-H benchmarks [\#6](https://github.com/apache/arrow-datafusion/issues/6) -- \[Ballista\] Fix integration test script [\#4](https://github.com/apache/arrow-datafusion/issues/4) -- Ballista should not have separate DataFrame implementation [\#2](https://github.com/apache/arrow-datafusion/issues/2) - -**Merged pull requests:** - -- Change datatype of tpch keys from Int32 to UInt64 to support sf=1000 [\#836](https://github.com/apache/arrow-datafusion/pull/836) ([andygrove](https://github.com/andygrove)) -- Add ballista-examples to docker build [\#829](https://github.com/apache/arrow-datafusion/pull/829) ([andygrove](https://github.com/andygrove)) -- Update dependencies: prost to 0.8 and tonic to 0.5 [\#818](https://github.com/apache/arrow-datafusion/pull/818) ([alamb](https://github.com/alamb)) -- Move `hash_array` into hash_utils.rs [\#807](https://github.com/apache/arrow-datafusion/pull/807) ([alamb](https://github.com/alamb)) -- Fix: Update clippy lints for Rust 1.54 [\#794](https://github.com/apache/arrow-datafusion/pull/794) ([alamb](https://github.com/alamb)) -- MINOR: Remove unused Ballista query execution code path [\#732](https://github.com/apache/arrow-datafusion/pull/732) ([andygrove](https://github.com/andygrove)) -- \[fix\] benchmark run with compose [\#666](https://github.com/apache/arrow-datafusion/pull/666) ([rdettai](https://github.com/rdettai)) -- bring back dev scripts for ballista [\#648](https://github.com/apache/arrow-datafusion/pull/648) ([Jimexist](https://github.com/Jimexist)) -- Remove unnecessary mutex [\#639](https://github.com/apache/arrow-datafusion/pull/639) ([edrevo](https://github.com/edrevo)) -- round trip TPCH queries in tests [\#630](https://github.com/apache/arrow-datafusion/pull/630) ([houqp](https://github.com/houqp)) -- Fix build [\#627](https://github.com/apache/arrow-datafusion/pull/627) ([andygrove](https://github.com/andygrove)) -- in ballista also check for UI prettier changes [\#578](https://github.com/apache/arrow-datafusion/pull/578) ([Jimexist](https://github.com/Jimexist)) -- turn on clippy rule for needless borrow [\#545](https://github.com/apache/arrow-datafusion/pull/545) ([Jimexist](https://github.com/Jimexist)) -- reuse datafusion physical planner in ballista building from protobuf [\#532](https://github.com/apache/arrow-datafusion/pull/532) ([Jimexist](https://github.com/Jimexist)) -- update cargo.toml in python crate and fix unit test due to hash joins [\#483](https://github.com/apache/arrow-datafusion/pull/483) ([Jimexist](https://github.com/Jimexist)) -- make `VOLUME` declaration in tpch datagen docker absolute [\#466](https://github.com/apache/arrow-datafusion/pull/466) ([crepererum](https://github.com/crepererum)) -- Refactor QueryStageExec in preparation for implementing map-side shuffle [\#459](https://github.com/apache/arrow-datafusion/pull/459) ([andygrove](https://github.com/andygrove)) -- Simplified usage of `use arrow` in ballista. [\#447](https://github.com/apache/arrow-datafusion/pull/447) ([jorgecarleitao](https://github.com/jorgecarleitao)) -- Benchmark subcommand to distinguish between DataFusion and Ballista [\#402](https://github.com/apache/arrow-datafusion/pull/402) ([jgoday](https://github.com/jgoday)) -- \#352: BallistaContext::collect\(\) logging is too noisy [\#394](https://github.com/apache/arrow-datafusion/pull/394) ([jgoday](https://github.com/jgoday)) -- cleanup function return type fn [\#350](https://github.com/apache/arrow-datafusion/pull/350) ([Jimexist](https://github.com/Jimexist)) -- Update Ballista to use new physical plan formatter utility [\#344](https://github.com/apache/arrow-datafusion/pull/344) ([andygrove](https://github.com/andygrove)) -- Update arrow dependencies again [\#341](https://github.com/apache/arrow-datafusion/pull/341) ([alamb](https://github.com/alamb)) -- Remove references to Ballista Docker images published to ballistacompute Docker Hub repo [\#326](https://github.com/apache/arrow-datafusion/pull/326) ([andygrove](https://github.com/andygrove)) -- Update arrow-rs deps [\#317](https://github.com/apache/arrow-datafusion/pull/317) ([alamb](https://github.com/alamb)) -- Update arrow deps [\#269](https://github.com/apache/arrow-datafusion/pull/269) ([alamb](https://github.com/alamb)) -- Enable redundant_field_names clippy lint [\#261](https://github.com/apache/arrow-datafusion/pull/261) ([Dandandan](https://github.com/Dandandan)) -- Update arrow-rs deps \(to fix build due to flatbuffers update\) [\#224](https://github.com/apache/arrow-datafusion/pull/224) ([alamb](https://github.com/alamb)) -- update arrow-rs deps to latest master [\#216](https://github.com/apache/arrow-datafusion/pull/216) ([alamb](https://github.com/alamb)) - -\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ +Per-release changelogs are available under +[`docs/source/changelog/`](docs/source/changelog/index.md). diff --git a/dev/release/README.md b/dev/release/README.md index 2b43e77b55..3f34ba10e5 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -88,48 +88,47 @@ GitHub dependency. ### Change Log -We maintain a `CHANGELOG.md` so our users know what has been changed between releases. +We maintain per-release changelogs under +[`docs/source/changelog/`](../../docs/source/changelog/). They are surfaced +in the Sphinx site through `docs/source/changelog/index.md`. -You will need a GitHub Personal Access Token for the following steps. Follow +You will need a GitHub Personal Access Token for the following steps. +Follow [these instructions](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) -to generate one if you do not already have one. - -The changelog is generated using a Python script. There is a dependency on `PyGitHub`, which can be installed using pip: +to generate one if you do not already have one. The changelog script +depends on `PyGitHub`: ```bash pip3 install PyGitHub ``` -Run the following command to generate the changelog content. +Run the generator from the repo root, pointing at the previous release +tag and the new release tag (or `HEAD`): ```bash -$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/datafusion-ballista 0.11.0 HEAD > 0.12.0.md +GITHUB_TOKEN= ./dev/release/generate-changelog.py \ + 52.0.0 HEAD 53.0.0 \ + > docs/source/changelog/53.0.0.md ``` -This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for -titles starting with `feat:`, `fix:`, or `docs:` . The script will produce output similar to: - -``` -Fetching list of commits between 0.11.0 and HEAD -Fetching pull requests -Categorizing pull requests -Generating changelog content -``` +The script writes a fully-formed file: ASF header, version title, commit / +contributor summary, categorized PR list, and a Credits section. The only +remaining manual step is to prepend the new version to the toctree at the +top of `docs/source/changelog/index.md`: -This process is not fully automated, so there are some additional manual steps: - -- Add the ASF header to the generated file -- Add the following content (copy from the previous version's changelog and update as appropriate: - -``` -## [0.12.0](https://github.com/apache/datafusion-ballista/tree/0.12.0) (2024-01-14) +```` +```{toctree} +:maxdepth: 1 -[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.11.0...0.12.0) +53.0.0 +52.0.0 +... ``` +```` -Send a PR to get these changes merged into the release branch (e.g. `branch-0.12`). If new commits that could change the -change log content landed in the release branch before you could merge the PR, you need to rerun the changelog update -script to regenerate the changelog and update the PR accordingly. +Send a PR with the new file and the updated index to the release branch +(e.g. `branch-53`). If new commits land in the release branch before +merge, rerun the generator to refresh the file. ## Prepare release candidate artifacts diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 3a33b331af..613c651d07 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -20,7 +20,7 @@ from github import Github import os import re - +import subprocess def print_pulls(repo_name, title, pulls): if len(pulls) > 0: @@ -32,7 +32,7 @@ def print_pulls(repo_name, title, pulls): print() -def generate_changelog(repo, repo_name, tag1, tag2): +def generate_changelog(repo, repo_name, tag1, tag2, version): # get a list of commits between two tags print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) @@ -52,49 +52,113 @@ def generate_changelog(repo, repo_name, tag1, tag2): all_pulls.append((pull, commit)) # we split the pulls into categories - #TODO: make categories configurable breaking = [] bugs = [] docs = [] enhancements = [] performance = [] + other = [] # categorize the pull requests based on GitHub labels print("Categorizing pull requests", file=sys.stderr) for (pull, commit) in all_pulls: + labels = [label.name for label in pull.labels] + + # skip Dependabot dependency-bump PRs (labeled 'auto-dependencies') + if 'auto-dependencies' in labels: + continue + # see if PR title uses Conventional Commits cc_type = '' cc_scope = '' cc_breaking = '' - parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + parts = re.findall(r'^([a-zA-Z]+)(\([a-zA-Z0-9_-]+\))?(!)?:', pull.title) if len(parts) == 1: parts_tuple = parts[0] cc_type = parts_tuple[0] # fix, feat, docs, chore cc_scope = parts_tuple[1] # component within project cc_breaking = parts_tuple[2] == '!' - labels = [label.name for label in pull.labels] - #print(pull.number, labels, parts, file=sys.stderr) if 'api change' in labels or cc_breaking: breaking.append((pull, commit)) - elif 'bug' in labels or cc_type == 'fix': - bugs.append((pull, commit)) elif 'performance' in labels or cc_type == 'perf': performance.append((pull, commit)) + elif 'bug' in labels or cc_type == 'fix': + bugs.append((pull, commit)) elif 'enhancement' in labels or cc_type == 'feat': enhancements.append((pull, commit)) - elif 'documentation' in labels or cc_type == 'docs': + elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc': docs.append((pull, commit)) + else: + other.append((pull, commit)) # produce the changelog content print("Generating changelog content", file=sys.stderr) + + # ASF header + print("""\n""") + + print(f"# Apache DataFusion Ballista {version} Changelog\n") + + # get the number of commits + commit_count = subprocess.check_output(["git", "rev-list", "--count", f"{tag1}..{tag2}"], text=True).strip() + + # get number of contributors + shortlog_output = subprocess.check_output(["git", "shortlog", "-sn", f"{tag1}..{tag2}"], text=True) + contributor_count = len(shortlog_output.strip().splitlines()) + + print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " + f"See credits at the end of this changelog for more information.\n") + print_pulls(repo_name, "Breaking changes", breaking) + print_pulls(repo_name, "Fixed bugs", bugs) print_pulls(repo_name, "Performance related", performance) print_pulls(repo_name, "Implemented enhancements", enhancements) - print_pulls(repo_name, "Fixed bugs", bugs) print_pulls(repo_name, "Documentation updates", docs) - print_pulls(repo_name, "Merged pull requests", all_pulls) + print_pulls(repo_name, "Other", other) + + # show code contributions + credits = shortlog_output.rstrip() + + print("## Credits\n") + print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " + "per contributor.\n") + print("```") + print(credits) + print("```\n") + + print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " + "PRs, and providing feedback on this release.\n") + +def resolve_ref(ref): + """Resolve a git ref (e.g. HEAD, branch name) to a full commit SHA.""" + try: + return subprocess.check_output( + ["git", "rev-parse", ref], text=True + ).strip() + except subprocess.CalledProcessError: + # If it can't be resolved locally, return as-is (e.g. a remote tag). + # The GitHub API will attempt to resolve it. + print(f"Note: Could not resolve '{ref}' locally; passing to GitHub API as-is", file=sys.stderr) + return ref def cli(args=None): @@ -103,16 +167,23 @@ def cli(args=None): args = sys.argv[1:] parser = argparse.ArgumentParser() - parser.add_argument("project", help="The project name e.g. apache/datafusion-ballista") - parser.add_argument("tag1", help="The previous release tag") - parser.add_argument("tag2", help="The current release tag") + parser.add_argument("tag1", help="The previous commit or tag (e.g. 52.0.0)") + parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") + parser.add_argument("version", help="The version number to include in the changelog") args = parser.parse_args() + # Resolve refs to SHAs so the GitHub API compares the same commits + # as the local git log. Without this, refs like HEAD get resolved by + # the GitHub API to the default branch instead of the current branch. + tag1 = resolve_ref(args.tag1) + tag2 = resolve_ref(args.tag2) + token = os.getenv("GITHUB_TOKEN") + project = "apache/datafusion-ballista" g = Github(token) - repo = g.get_repo(args.project) - generate_changelog(repo, args.project, args.tag1, args.tag2) + repo = g.get_repo(project) + generate_changelog(repo, project, tag1, tag2, args.version) if __name__ == "__main__": - cli() \ No newline at end of file + cli() diff --git a/docs/source/changelog/0.10.0.md b/docs/source/changelog/0.10.0.md new file mode 100644 index 0000000000..2984a207bb --- /dev/null +++ b/docs/source/changelog/0.10.0.md @@ -0,0 +1,95 @@ + + +# Apache DataFusion Ballista 0.10.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.9.0...0.10.0) + +**Implemented enhancements:** + +- Add user guide section on prometheus metrics [\#507](https://github.com/apache/datafusion-ballista/issues/507) +- Don't throw error when job path not exist in remove_job_data [\#502](https://github.com/apache/datafusion-ballista/issues/502) +- Fix clippy warning [\#494](https://github.com/apache/datafusion-ballista/issues/494) +- Use job_data_clean_up_interval_seconds == 0 to indicate executor_cleanup_enable [\#488](https://github.com/apache/datafusion-ballista/issues/488) +- Add a config for tracing log rolling policy for both scheduler and executor [\#486](https://github.com/apache/datafusion-ballista/issues/486) +- Set up repo where we can push benchmark results [\#473](https://github.com/apache/datafusion-ballista/issues/473) +- Make the delayed time interval for cleanup job data in both scheduler and executor configurable [\#469](https://github.com/apache/datafusion-ballista/issues/469) +- Add some validation for the remove_job_data grpc service [\#467](https://github.com/apache/datafusion-ballista/issues/467) +- Add ability to build docker images using `release-lto` profile [\#463](https://github.com/apache/datafusion-ballista/issues/463) +- Suggest users download \(rather than build\) the FlightSQL JDBC Driver [\#460](https://github.com/apache/datafusion-ballista/issues/460) +- Clean up legacy job shuffle data [\#459](https://github.com/apache/datafusion-ballista/issues/459) +- Add grpc service for the scheduler to make it able to be triggered by client explicitly [\#458](https://github.com/apache/datafusion-ballista/issues/458) +- Replace Mutex\ by using DashMap [\#448](https://github.com/apache/datafusion-ballista/issues/448) +- Refine log level [\#446](https://github.com/apache/datafusion-ballista/issues/446) +- Upgrade to DataFusion 14.0.0 [\#445](https://github.com/apache/datafusion-ballista/issues/445) +- Add a feature for hdfs3 [\#419](https://github.com/apache/datafusion-ballista/issues/419) +- Add optional flag which advertises host for Arrow Flight SQL [\#418](https://github.com/apache/datafusion-ballista/issues/418) +- Partitioning reasoning in DataFusion and Ballista [\#284](https://github.com/apache/datafusion-ballista/issues/284) +- Stop wasting time in CI on MIRI runs [\#283](https://github.com/apache/datafusion-ballista/issues/283) +- Publish Docker images as part of each release [\#236](https://github.com/apache/datafusion-ballista/issues/236) +- Cleanup job/stage status from TaskManager and clean up shuffle data after a period after JobFinished [\#185](https://github.com/apache/datafusion-ballista/issues/185) + +**Fixed bugs:** + +- build broken: configure_me_codegen retroactively reserved `bind_host` [\#519](https://github.com/apache/datafusion-ballista/issues/519) +- Return empty results for SQLs with order by [\#451](https://github.com/apache/datafusion-ballista/issues/451) +- ballista scheduler is not taken inline parameters into account [\#443](https://github.com/apache/datafusion-ballista/issues/443) +- \[FlightSQL\] Cannot connect with Tableau Desktop [\#428](https://github.com/apache/datafusion-ballista/issues/428) +- Benchmark q15 fails [\#372](https://github.com/apache/datafusion-ballista/issues/372) +- Incorrect documentation for building Ballista on Linux when using docker-compose [\#362](https://github.com/apache/datafusion-ballista/issues/362) +- Scheduler silently replaces `ParquetExec` with `EmptyExec` if data path is not correctly mounted in container [\#353](https://github.com/apache/datafusion-ballista/issues/353) +- SQL with order by limit returns nothing [\#334](https://github.com/apache/datafusion-ballista/issues/334) + +**Documentation updates:** + +- README updates [\#433](https://github.com/apache/datafusion-ballista/pull/433) ([andygrove](https://github.com/andygrove)) + +**Merged pull requests:** + +- configure_me_codegen retroactively reserved on our `bind_host` parame… [\#520](https://github.com/apache/datafusion-ballista/pull/520) ([avantgardnerio](https://github.com/avantgardnerio)) +- Bump actions/cache from 2 to 3 [\#517](https://github.com/apache/datafusion-ballista/pull/517) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Update graphviz-rust requirement from 0.3.0 to 0.4.0 [\#515](https://github.com/apache/datafusion-ballista/pull/515) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add Prometheus metrics endpoint [\#511](https://github.com/apache/datafusion-ballista/pull/511) ([thinkharderdev](https://github.com/thinkharderdev)) +- Enable tests that work since upgrading to DataFusion 14 [\#510](https://github.com/apache/datafusion-ballista/pull/510) ([andygrove](https://github.com/andygrove)) +- Update hashbrown requirement from 0.12 to 0.13 [\#506](https://github.com/apache/datafusion-ballista/pull/506) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Don't throw error when job shuffle data path not exist in executor [\#503](https://github.com/apache/datafusion-ballista/pull/503) ([yahoNanJing](https://github.com/yahoNanJing)) +- Upgrade to DataFusion 14.0.0 and Arrow 26.0.0 [\#499](https://github.com/apache/datafusion-ballista/pull/499) ([andygrove](https://github.com/andygrove)) +- Fix clippy warning [\#495](https://github.com/apache/datafusion-ballista/pull/495) ([yahoNanJing](https://github.com/yahoNanJing)) +- Stop wasting time in CI on MIRI runs [\#491](https://github.com/apache/datafusion-ballista/pull/491) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Remove executor config executor_cleanup_enable and make the configuation name for executor cleanup more intuitive [\#489](https://github.com/apache/datafusion-ballista/pull/489) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add a config for tracing log rolling policy for both scheduler and executor [\#487](https://github.com/apache/datafusion-ballista/pull/487) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add grpc service of cleaning up job shuffle data for the scheduler to make it able to be triggered by client explicitly [\#485](https://github.com/apache/datafusion-ballista/pull/485) ([yahoNanJing](https://github.com/yahoNanJing)) +- \[Minor\] Bump DataFusion [\#480](https://github.com/apache/datafusion-ballista/pull/480) ([Dandandan](https://github.com/Dandandan)) +- Remove benchmark results from README [\#478](https://github.com/apache/datafusion-ballista/pull/478) ([andygrove](https://github.com/andygrove)) +- Update `flightsql.md` to provide correct instruction [\#476](https://github.com/apache/datafusion-ballista/pull/476) ([iajoiner](https://github.com/iajoiner)) +- Add support for Tableau [\#475](https://github.com/apache/datafusion-ballista/pull/475) ([avantgardnerio](https://github.com/avantgardnerio)) +- Add SchedulerConfig for the scheduler configurations, like event_loop_buffer_size, finished_job_data_clean_up_interval_seconds, finished_job_state_clean_up_interval_seconds [\#472](https://github.com/apache/datafusion-ballista/pull/472) ([yahoNanJing](https://github.com/yahoNanJing)) +- Bump DataFusion [\#471](https://github.com/apache/datafusion-ballista/pull/471) ([Dandandan](https://github.com/Dandandan)) +- Add some validation for remove_job_data in the executor server [\#468](https://github.com/apache/datafusion-ballista/pull/468) ([yahoNanJing](https://github.com/yahoNanJing)) +- Update documentation to reflect the release of the FlightSQL JDBC Driver [\#461](https://github.com/apache/datafusion-ballista/pull/461) ([avantgardnerio](https://github.com/avantgardnerio)) +- Bump DataFusion version [\#453](https://github.com/apache/datafusion-ballista/pull/453) ([andygrove](https://github.com/andygrove)) +- Add shuffle for SortPreservingMergeExec physical operator [\#452](https://github.com/apache/datafusion-ballista/pull/452) ([yahoNanJing](https://github.com/yahoNanJing)) +- Replace Mutex\ by using DashMap [\#449](https://github.com/apache/datafusion-ballista/pull/449) ([yahoNanJing](https://github.com/yahoNanJing)) +- Refine log level for trial info and periodically invoked places [\#447](https://github.com/apache/datafusion-ballista/pull/447) ([yahoNanJing](https://github.com/yahoNanJing)) +- MINOR: Add `set -e` to scripts, fix a typo [\#444](https://github.com/apache/datafusion-ballista/pull/444) ([andygrove](https://github.com/andygrove)) +- Add optional flag which advertises host for Arrow Flight SQL \#418 [\#442](https://github.com/apache/datafusion-ballista/pull/442) ([DaltonModlin](https://github.com/DaltonModlin)) +- Reorder joins after resolving stage inputs [\#441](https://github.com/apache/datafusion-ballista/pull/441) ([Dandandan](https://github.com/Dandandan)) +- Add a feature for hdfs3 [\#439](https://github.com/apache/datafusion-ballista/pull/439) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add Spark benchmarks [\#438](https://github.com/apache/datafusion-ballista/pull/438) ([andygrove](https://github.com/andygrove)) +- scheduler now verifies that `file://` ListingTable URLs are accessible [\#414](https://github.com/apache/datafusion-ballista/pull/414) ([andygrove](https://github.com/andygrove)) diff --git a/docs/source/changelog/0.11.0.md b/docs/source/changelog/0.11.0.md new file mode 100644 index 0000000000..7dc454b048 --- /dev/null +++ b/docs/source/changelog/0.11.0.md @@ -0,0 +1,99 @@ + + +# Apache DataFusion Ballista 0.11.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.10.0...0.11.0) + +**Implemented enhancements:** + +- Remove `python` since it has been moved to its own repo, `datafusion-ballista-python` [\#653](https://github.com/apache/datafusion-ballista/issues/653) +- Add executor self-registration mechanism in the heartbeat service [\#648](https://github.com/apache/datafusion-ballista/issues/648) +- Upgrade to DataFusion 17 [\#638](https://github.com/apache/datafusion-ballista/issues/638) +- Move Python bindings to separate repo? [\#635](https://github.com/apache/datafusion-ballista/issues/635) +- Implement new release process [\#622](https://github.com/apache/datafusion-ballista/issues/622) +- Change default branch name from master to main [\#618](https://github.com/apache/datafusion-ballista/issues/618) +- Update latest datafusion dependency [\#610](https://github.com/apache/datafusion-ballista/issues/610) +- Implement optimizer rule to remove redundant repartitioning [\#608](https://github.com/apache/datafusion-ballista/issues/608) +- ballista-cli as \(docker\) images [\#600](https://github.com/apache/datafusion-ballista/issues/600) +- Update contributor guide [\#598](https://github.com/apache/datafusion-ballista/issues/598) +- Fix cargo clippy [\#570](https://github.com/apache/datafusion-ballista/issues/570) +- Support Alibaba Cloud OSS with ObjectStore [\#566](https://github.com/apache/datafusion-ballista/issues/566) +- Refactor `StateBackendClient` to be a higher-level interface [\#554](https://github.com/apache/datafusion-ballista/issues/554) +- Make it concurrently to launch tasks to executors [\#544](https://github.com/apache/datafusion-ballista/issues/544) +- Simplify docs [\#531](https://github.com/apache/datafusion-ballista/issues/531) +- Provide an in-memory StateBackend [\#505](https://github.com/apache/datafusion-ballista/issues/505) +- Add support for Azure blob storage [\#294](https://github.com/apache/datafusion-ballista/issues/294) +- Add a workflow to build the image and publish it to the package [\#71](https://github.com/apache/datafusion-ballista/issues/71) + +**Fixed bugs:** + +- Rust / Check Cargo.toml formatting \(amd64, stable\) \(pull_request\) Failing [\#662](https://github.com/apache/datafusion-ballista/issues/662) +- Protobuf parsing error [\#646](https://github.com/apache/datafusion-ballista/issues/646) +- jobs from python client not showing up in Scheduler UI [\#625](https://github.com/apache/datafusion-ballista/issues/625) +- ballista ui fails to build [\#594](https://github.com/apache/datafusion-ballista/issues/594) +- cargo build --release fails for ballista-scheduler [\#590](https://github.com/apache/datafusion-ballista/issues/590) +- docker build fails [\#589](https://github.com/apache/datafusion-ballista/issues/589) +- Multi-scheduler Job Starvation [\#585](https://github.com/apache/datafusion-ballista/issues/585) +- Cannot query file from S3 [\#559](https://github.com/apache/datafusion-ballista/issues/559) +- Benchmark q16 fails [\#373](https://github.com/apache/datafusion-ballista/issues/373) + +**Documentation updates:** + +- Check in benchmark image [\#647](https://github.com/apache/datafusion-ballista/pull/647) ([andygrove](https://github.com/andygrove)) +- MINOR: Fix benchmark image link [\#596](https://github.com/apache/datafusion-ballista/pull/596) ([andygrove](https://github.com/andygrove)) + +**Merged pull requests:** + +- Upgrade to DataFusion 18 [\#668](https://github.com/apache/datafusion-ballista/pull/668) ([andygrove](https://github.com/andygrove)) +- Enable physical plan round-trip tests [\#666](https://github.com/apache/datafusion-ballista/pull/666) ([andygrove](https://github.com/andygrove)) +- Upgrade to DataFusion 18.0.0-rc1 [\#664](https://github.com/apache/datafusion-ballista/pull/664) ([andygrove](https://github.com/andygrove)) +- add test_util to make examples work well [\#661](https://github.com/apache/datafusion-ballista/pull/661) ([jiangzhx](https://github.com/jiangzhx)) +- Minor refactor to reduce duplicate code [\#659](https://github.com/apache/datafusion-ballista/pull/659) ([andygrove](https://github.com/andygrove)) +- Cluster state refactor Part 2 [\#658](https://github.com/apache/datafusion-ballista/pull/658) ([thinkharderdev](https://github.com/thinkharderdev)) +- Remove `python` dir & python-related workflows [\#654](https://github.com/apache/datafusion-ballista/pull/654) ([iajoiner](https://github.com/iajoiner)) +- Add executor self-registration mechanism in the heartbeat service [\#649](https://github.com/apache/datafusion-ballista/pull/649) ([yahoNanJing](https://github.com/yahoNanJing)) +- Upgrade to DataFusion 17 [\#639](https://github.com/apache/datafusion-ballista/pull/639) ([avantgardnerio](https://github.com/avantgardnerio)) +- Upgrade to DataFusion 16 \(again\) [\#636](https://github.com/apache/datafusion-ballista/pull/636) ([avantgardnerio](https://github.com/avantgardnerio)) +- Update release process documentation [\#632](https://github.com/apache/datafusion-ballista/pull/632) ([andygrove](https://github.com/andygrove)) +- Implement new release process [\#623](https://github.com/apache/datafusion-ballista/pull/623) ([andygrove](https://github.com/andygrove)) +- Update contributor guide [\#617](https://github.com/apache/datafusion-ballista/pull/617) ([andygrove](https://github.com/andygrove)) +- Fix Cargo.toml format issue [\#616](https://github.com/apache/datafusion-ballista/pull/616) ([andygrove](https://github.com/andygrove)) +- Refactor scheduler main [\#615](https://github.com/apache/datafusion-ballista/pull/615) ([andygrove](https://github.com/andygrove)) +- Refactor executor main [\#614](https://github.com/apache/datafusion-ballista/pull/614) ([andygrove](https://github.com/andygrove)) +- Update datafusion dependency to the latest version [\#612](https://github.com/apache/datafusion-ballista/pull/612) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add support for Azure Blob Storage [\#599](https://github.com/apache/datafusion-ballista/pull/599) ([aidankovacic-8451](https://github.com/aidankovacic-8451)) +- Python: add method to get explain output as a string [\#593](https://github.com/apache/datafusion-ballista/pull/593) ([andygrove](https://github.com/andygrove)) +- Handle job resubmission [\#586](https://github.com/apache/datafusion-ballista/pull/586) ([thinkharderdev](https://github.com/thinkharderdev)) +- updated readme to contain correct versions of dependencies. [\#580](https://github.com/apache/datafusion-ballista/pull/580) ([saikrishna1-bidgely](https://github.com/saikrishna1-bidgely)) +- Update graphviz-rust requirement from 0.4.0 to 0.5.0 [\#574](https://github.com/apache/datafusion-ballista/pull/574) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Super minor spelling error [\#573](https://github.com/apache/datafusion-ballista/pull/573) ([jdye64](https://github.com/jdye64)) +- Fix cargo clippy [\#571](https://github.com/apache/datafusion-ballista/pull/571) ([yahoNanJing](https://github.com/yahoNanJing)) +- Support Alibaba Cloud OSS with ObjectStore [\#567](https://github.com/apache/datafusion-ballista/pull/567) ([r4ntix](https://github.com/r4ntix)) +- fix\(ui\): fix last seen [\#562](https://github.com/apache/datafusion-ballista/pull/562) ([duyet](https://github.com/duyet)) +- Cluster state refactor part 1 [\#560](https://github.com/apache/datafusion-ballista/pull/560) ([thinkharderdev](https://github.com/thinkharderdev)) +- Make it concurrently to launch tasks to executors [\#557](https://github.com/apache/datafusion-ballista/pull/557) ([yahoNanJing](https://github.com/yahoNanJing)) +- Update datafusion requirement from 14.0.0 to 15.0.0 [\#552](https://github.com/apache/datafusion-ballista/pull/552) ([yahoNanJing](https://github.com/yahoNanJing)) +- docs: fix style in the Helm readme [\#551](https://github.com/apache/datafusion-ballista/pull/551) ([haoxins](https://github.com/haoxins)) +- Fix Helm chart's image format [\#550](https://github.com/apache/datafusion-ballista/pull/550) ([haoxins](https://github.com/haoxins)) +- Update env_logger requirement from 0.9 to 0.10 [\#539](https://github.com/apache/datafusion-ballista/pull/539) ([dependabot[bot]](https://github.com/apps/dependabot)) +- only build docker images on rc tags [\#535](https://github.com/apache/datafusion-ballista/pull/535) ([andygrove](https://github.com/andygrove)) +- Remove `--locked` when building Python wheels [\#533](https://github.com/apache/datafusion-ballista/pull/533) ([andygrove](https://github.com/andygrove)) +- Bump actions/labeler from 4.0.2 to 4.1.0 [\#525](https://github.com/apache/datafusion-ballista/pull/525) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Provide a memory StateBackendClient [\#523](https://github.com/apache/datafusion-ballista/pull/523) ([yahoNanJing](https://github.com/yahoNanJing)) diff --git a/docs/source/changelog/0.12.0.md b/docs/source/changelog/0.12.0.md new file mode 100644 index 0000000000..8c271e04dc --- /dev/null +++ b/docs/source/changelog/0.12.0.md @@ -0,0 +1,119 @@ + + +# Apache DataFusion Ballista 0.12.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.11.0...0.12.0) + +**Documentation updates:** + +- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins) + +**Merged pull requests:** + +- [minor] remove outdate todo [#683](https://github.com/apache/datafusion-ballista/pull/683) (Ted-Jiang) +- Add executor terminating status for graceful shutdown [#667](https://github.com/apache/datafusion-ballista/pull/667) (thinkharderdev) +- Allow `BallistaContext::read_*` methods to read multiple paths. [#679](https://github.com/apache/datafusion-ballista/pull/679) (luckylsk34) +- Update scheduler.md [#657](https://github.com/apache/datafusion-ballista/pull/657) (psvri) +- Mark `SchedulerState` as pub [#688](https://github.com/apache/datafusion-ballista/pull/688) (Dandandan) +- Update graphviz-rust requirement from 0.5.0 to 0.6.1 [#651](https://github.com/apache/datafusion-ballista/pull/651) (dependabot[bot]) +- Upgrade DataFusion to 19.0.0 [#691](https://github.com/apache/datafusion-ballista/pull/691) (r4ntix) +- Update release docs [#692](https://github.com/apache/datafusion-ballista/pull/692) (andygrove) +- Mark `SchedulerServer::with_task_launcher` as pub [#695](https://github.com/apache/datafusion-ballista/pull/695) (Dandandan) +- Make task_manager pub [#698](https://github.com/apache/datafusion-ballista/pull/698) (Dandandan) +- Add ExecutionEngine abstraction [#687](https://github.com/apache/datafusion-ballista/pull/687) (andygrove) +- Allow accessing s3 locations in client mode [#700](https://github.com/apache/datafusion-ballista/pull/700) (luckylsk34) +- git clone branch incorrect [#699](https://github.com/apache/datafusion-ballista/pull/699) (BubbaJoe) +- Fix for error message during testing [#707](https://github.com/apache/datafusion-ballista/pull/707) (yahoNanJing) +- Upgrade datafusion to 20.0.0 & sqlparser to to 0.32.0 [#711](https://github.com/apache/datafusion-ballista/pull/711) (r4ntix) +- Update README.md [#729](https://github.com/apache/datafusion-ballista/pull/729) (jiangzhx) +- Update link to scheduler proto file in dev docs [#713](https://github.com/apache/datafusion-ballista/pull/713) (JAicewizard) +- Fix `show tables` fails [#715](https://github.com/apache/datafusion-ballista/pull/715) (r4ntix) +- Remove redundant fields in ExecutorManager [#728](https://github.com/apache/datafusion-ballista/pull/728) (yahoNanJing) +- Fix parameter '--config-backend' to '--cluster-backend' [#720](https://github.com/apache/datafusion-ballista/pull/720) (paolorechia) +- Upgrade DataFusion to 21.0.0 [#727](https://github.com/apache/datafusion-ballista/pull/727) (r4ntix) +- [minor] remove useless brackets [#739](https://github.com/apache/datafusion-ballista/pull/739) (Ted-Jiang) +- Only decode plan in `LaunchMultiTaskParams` once [#743](https://github.com/apache/datafusion-ballista/pull/743) (Dandandan) +- Upgrade DataFusion to 22.0.0 [#740](https://github.com/apache/datafusion-ballista/pull/740) (r4ntix) +- [feature] support shuffle read with retry when facing IO error. [#738](https://github.com/apache/datafusion-ballista/pull/738) (Ted-Jiang) +- [log] Print long running task status. [#750](https://github.com/apache/datafusion-ballista/pull/750) (Ted-Jiang) +- Upgrade DataFusion to 23.0.0 [#755](https://github.com/apache/datafusion-ballista/pull/755) (yahoNanJing) +- Fix plan metrics length and stage metrics length not match [#764](https://github.com/apache/datafusion-ballista/pull/764) (yahoNanJing) +- added match arms to create ClusterStorageConfig [#766](https://github.com/apache/datafusion-ballista/pull/766) (BokarevNik) +- [Improve] refactor the offer_reservation avoid wait result [#760](https://github.com/apache/datafusion-ballista/pull/760) (Ted-Jiang) +- [fea] Avoid multithreaded write lock conflicts in event queue. [#754](https://github.com/apache/datafusion-ballista/pull/754) (Ted-Jiang) +- Upgrade DataFusion to 24.0.0, Object_Store to 0.5.6 [#769](https://github.com/apache/datafusion-ballista/pull/769) (r4ntix) +- Refine create_datafusion_context() [#778](https://github.com/apache/datafusion-ballista/pull/778) (yahoNanJing) +- Remove output_partitioning for task definition [#776](https://github.com/apache/datafusion-ballista/pull/776) (yahoNanJing) +- Upgrade DataFusion to 25.0.0 [#779](https://github.com/apache/datafusion-ballista/pull/779) (r4ntix) +- Disable the ansi feature of tracing-subscriber [#784](https://github.com/apache/datafusion-ballista/pull/784) (yahoNanJing) +- Add config grpc_server_max_decoding_message_size to make the maximum size of a decoded message at the grpc server side configurable [#782](https://github.com/apache/datafusion-ballista/pull/782) (yahoNanJing) +- Fix nodejs issues in Docker build [#731](https://github.com/apache/datafusion-ballista/pull/731) (jnaous) +- Upgrade node version to fix build in `main` [#794](https://github.com/apache/datafusion-ballista/pull/794) (avantgardnerio) +- Remove redundant mod session_registry [#792](https://github.com/apache/datafusion-ballista/pull/792) (yahoNanJing) +- Make last_seen_ts_threshold for getting alive executor at the scheduler side larger than the heartbeat time interval [#786](https://github.com/apache/datafusion-ballista/pull/786) (yahoNanJing) +- Remove the prometheus-metrics from the default feature [#788](https://github.com/apache/datafusion-ballista/pull/788) (yahoNanJing) +- Refine the ExecuteQuery grpc interface [#790](https://github.com/apache/datafusion-ballista/pull/790) (yahoNanJing) +- Add config to collect statistics, enable in TPC-H benchmark [#796](https://github.com/apache/datafusion-ballista/pull/796) (Dandandan) +- Add support for GCS data sources [#805](https://github.com/apache/datafusion-ballista/pull/805) (haoxins) +- Update DataFusion to 26 [#798](https://github.com/apache/datafusion-ballista/pull/798) (Dandandan) +- Issue 162 build docker image in ci [#716](https://github.com/apache/datafusion-ballista/pull/716) (paolorechia) +- Fix index out of bounds panic [#819](https://github.com/apache/datafusion-ballista/pull/819) (yahoNanJing) +- Refactor the TaskDefinition by changing encoding execution plan to the decoded one [#817](https://github.com/apache/datafusion-ballista/pull/817) (yahoNanJing) +- Fix ballista-cli docs [#800](https://github.com/apache/datafusion-ballista/pull/800) (jonahgao) +- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins) +- Implement the with_new_children for ShuffleReaderExec [#821](https://github.com/apache/datafusion-ballista/pull/821) (yahoNanJing) +- Update to point to the correct documentation [#838](https://github.com/apache/datafusion-ballista/pull/838) (dadepo) +- Remove ExecutorReservation and change the task assignment philosophy from executor first to task first [#823](https://github.com/apache/datafusion-ballista/pull/823) (yahoNanJing) +- Upgrade DataFusion to 27.0.0 [#834](https://github.com/apache/datafusion-ballista/pull/834) (r4ntix) +- Reduce the number of calls to `create_logical_plan` [#842](https://github.com/apache/datafusion-ballista/pull/842) (jonahgao) +- Bump semver from 5.7.1 to 5.7.2 in /ballista/scheduler/ui [#843](https://github.com/apache/datafusion-ballista/pull/843) (dependabot[bot]) +- Bump actions/labeler from 4.1.0 to 4.3.0 [#841](https://github.com/apache/datafusion-ballista/pull/841) (dependabot[bot]) +- Bump tough-cookie from 4.1.2 to 4.1.3 in /ballista/scheduler/ui [#840](https://github.com/apache/datafusion-ballista/pull/840) (dependabot[bot]) +- Update flatbuffers requirement from 22.9.29 to 23.5.26 [#801](https://github.com/apache/datafusion-ballista/pull/801) (dependabot[bot]) +- Update dirs requirement from 4.0.0 to 5.0.1 [#767](https://github.com/apache/datafusion-ballista/pull/767) (dependabot[bot]) +- Update libloading requirement from 0.7.3 to 0.8.0 [#761](https://github.com/apache/datafusion-ballista/pull/761) (dependabot[bot]) +- Introduce a cache crate supporting concurrent cache value loading [#825](https://github.com/apache/datafusion-ballista/pull/825) (yahoNanJing) +- Fix cargo clippy for latest rust version [#848](https://github.com/apache/datafusion-ballista/pull/848) (yahoNanJing) +- Introduce CachedBasedObjectStoreRegistry to use data source cache transparently [#827](https://github.com/apache/datafusion-ballista/pull/827) (yahoNanJing) +- Add ConsistentHash for node topology management [#830](https://github.com/apache/datafusion-ballista/pull/830) (yahoNanJing) +- Implement 3-phase consistent hash based task assignment policy [#833](https://github.com/apache/datafusion-ballista/pull/833) (yahoNanJing) +- Update tonic requirement from 0.8 to 0.9 [#733](https://github.com/apache/datafusion-ballista/pull/733) (dependabot[bot]) +- Update itertools requirement from 0.10 to 0.11 [#844](https://github.com/apache/datafusion-ballista/pull/844) (dependabot[bot]) +- Update etcd-client requirement from 0.10 to 0.11 [#845](https://github.com/apache/datafusion-ballista/pull/845) (dependabot[bot]) +- Update hashbrown requirement from 0.13 to 0.14 [#846](https://github.com/apache/datafusion-ballista/pull/846) (dependabot[bot]) +- Bump word-wrap from 1.2.3 to 1.2.4 in /ballista/scheduler/ui [#849](https://github.com/apache/datafusion-ballista/pull/849) (dependabot[bot]) +- Update hdfs requirement from 0.1.1 to 0.1.4 [#856](https://github.com/apache/datafusion-ballista/pull/856) (yahoNanJing) +- Update to DataFusion 28 [#858](https://github.com/apache/datafusion-ballista/pull/858) (Dandandan) +- Upgrade datafusion to 30.0.0 [#866](https://github.com/apache/datafusion-ballista/pull/866) (r4ntix) +- refactor: port get_scan_files to Ballista [#877](https://github.com/apache/datafusion-ballista/pull/877) (alamb) +- Upgrade datafusion to 31.0.0 [#878](https://github.com/apache/datafusion-ballista/pull/878) (r4ntix) +- Upgrade datafusion to 32.0.0 [#899](https://github.com/apache/datafusion-ballista/pull/899) (r4ntix) +- Update to DataFusion 33 [#900](https://github.com/apache/datafusion-ballista/pull/900) (Dandandan) +- Refactor lru mod, remove linked_hash_map [#918](https://github.com/apache/datafusion-ballista/pull/918) (PsiACE) +- Dynamically optimize aggregate (count) based on shuffle stats [#919](https://github.com/apache/datafusion-ballista/pull/919) (Dandandan) +- Use lz4 compression for shuffle files & flight stream, refactoring / improvements [#920](https://github.com/apache/datafusion-ballista/pull/920) (Dandandan) +- Make max encoding message size configurable [#928](https://github.com/apache/datafusion-ballista/pull/928) (andygrove) +- Set max message size to 16MB in gRPC clients [#931](https://github.com/apache/datafusion-ballista/pull/931) (andygrove) +- Upgrade to DataFusion 34.0.0-rc1 [#927](https://github.com/apache/datafusion-ballista/pull/927) (andygrove) +- Use official DF 34 release [#939](https://github.com/apache/datafusion-ballista/pull/939) (andygrove) +- Use StreamWriter instead of FileWriter [#943](https://github.com/apache/datafusion-ballista/pull/943) (avantgardnerio) +- Remove some TODO comments related to context fetching schemas from scheduler [#946](https://github.com/apache/datafusion-ballista/pull/946) (andygrove) +- Fix Docker build [#947](https://github.com/apache/datafusion-ballista/pull/947) (andygrove) +- Fix regression in DataFrame.write_xxx [#945](https://github.com/apache/datafusion-ballista/pull/945) (andygrove) diff --git a/docs/source/changelog/0.5.0.md b/docs/source/changelog/0.5.0.md new file mode 100644 index 0000000000..38d0355dae --- /dev/null +++ b/docs/source/changelog/0.5.0.md @@ -0,0 +1,174 @@ + + +# Apache DataFusion Ballista 0.5.0 Changelog + +[Full Changelog](https://github.com/apache/arrow-datafusion/compare/4.0.0...ballista-0.5.0) + +**Breaking changes:** + +- \[ballista\] support date_part and date_turnc ser/de, pass tpch 7 [\#840](https://github.com/apache/arrow-datafusion/pull/840) ([houqp](https://github.com/houqp)) +- Box ScalarValue:Lists, reduce size by half size [\#788](https://github.com/apache/arrow-datafusion/pull/788) ([alamb](https://github.com/alamb)) +- Support DataFrame.collect for Ballista DataFrames [\#785](https://github.com/apache/arrow-datafusion/pull/785) ([andygrove](https://github.com/andygrove)) +- JOIN conditions are order dependent [\#778](https://github.com/apache/arrow-datafusion/pull/778) ([seddonm1](https://github.com/seddonm1)) +- UnresolvedShuffleExec should represent a single shuffle [\#727](https://github.com/apache/arrow-datafusion/pull/727) ([andygrove](https://github.com/andygrove)) +- Ballista: Make shuffle partitions configurable in benchmarks [\#702](https://github.com/apache/arrow-datafusion/pull/702) ([andygrove](https://github.com/andygrove)) +- Rename MergeExec to CoalescePartitionsExec [\#635](https://github.com/apache/arrow-datafusion/pull/635) ([andygrove](https://github.com/andygrove)) +- Ballista: Rename QueryStageExec to ShuffleWriterExec [\#633](https://github.com/apache/arrow-datafusion/pull/633) ([andygrove](https://github.com/andygrove)) +- fix 593, reduce cloning by taking ownership in logical planner's `from` fn [\#610](https://github.com/apache/arrow-datafusion/pull/610) ([Jimexist](https://github.com/Jimexist)) +- fix join column handling logic for `On` and `Using` constraints [\#605](https://github.com/apache/arrow-datafusion/pull/605) ([houqp](https://github.com/houqp)) +- Move ballista standalone mode to client [\#589](https://github.com/apache/arrow-datafusion/pull/589) ([edrevo](https://github.com/edrevo)) +- Ballista: Implement map-side shuffle [\#543](https://github.com/apache/arrow-datafusion/pull/543) ([andygrove](https://github.com/andygrove)) +- ShuffleReaderExec now supports multiple locations per partition [\#541](https://github.com/apache/arrow-datafusion/pull/541) ([andygrove](https://github.com/andygrove)) +- Make external hostname in executor optional [\#232](https://github.com/apache/arrow-datafusion/pull/232) ([edrevo](https://github.com/edrevo)) +- Remove namespace from executors [\#75](https://github.com/apache/arrow-datafusion/pull/75) ([edrevo](https://github.com/edrevo)) +- Support qualified columns in queries [\#55](https://github.com/apache/arrow-datafusion/pull/55) ([houqp](https://github.com/houqp)) +- Read CSV format text from stdin or memory [\#54](https://github.com/apache/arrow-datafusion/pull/54) ([heymind](https://github.com/heymind)) +- Remove Ballista DataFrame [\#48](https://github.com/apache/arrow-datafusion/pull/48) ([andygrove](https://github.com/andygrove)) +- Use atomics for SQLMetric implementation, remove unused name field [\#25](https://github.com/apache/arrow-datafusion/pull/25) ([returnString](https://github.com/returnString)) + +**Implemented enhancements:** + +- Add crate documentation for Ballista crates [\#830](https://github.com/apache/arrow-datafusion/issues/830) +- Support DataFrame.collect for Ballista DataFrames [\#787](https://github.com/apache/arrow-datafusion/issues/787) +- Ballista: Prep for supporting shuffle correctly, part one [\#736](https://github.com/apache/arrow-datafusion/issues/736) +- Ballista: Implement physical plan serde for ShuffleWriterExec [\#710](https://github.com/apache/arrow-datafusion/issues/710) +- Ballista: Finish implementing shuffle mechanism [\#707](https://github.com/apache/arrow-datafusion/issues/707) +- Rename QueryStageExec to ShuffleWriterExec [\#542](https://github.com/apache/arrow-datafusion/issues/542) +- Ballista ShuffleReaderExec should be able to read from multiple locations per partition [\#540](https://github.com/apache/arrow-datafusion/issues/540) +- \[Ballista\] Use deployments in k8s user guide [\#473](https://github.com/apache/arrow-datafusion/issues/473) +- Ballista refactor QueryStageExec in preparation for map-side shuffle [\#458](https://github.com/apache/arrow-datafusion/issues/458) +- Ballista: Implement map-side of shuffle [\#456](https://github.com/apache/arrow-datafusion/issues/456) +- Refactor Ballista to separate Flight logic from execution logic [\#449](https://github.com/apache/arrow-datafusion/issues/449) +- Use published versions of arrow rather than github shas [\#393](https://github.com/apache/arrow-datafusion/issues/393) +- BallistaContext::collect\(\) logging is too noisy [\#352](https://github.com/apache/arrow-datafusion/issues/352) +- Update Ballista to use new physical plan formatter utility [\#343](https://github.com/apache/arrow-datafusion/issues/343) +- Add Ballista Getting Started documentation [\#329](https://github.com/apache/arrow-datafusion/issues/329) +- Remove references to ballistacompute Docker Hub repo [\#325](https://github.com/apache/arrow-datafusion/issues/325) +- Implement scalable distributed joins [\#63](https://github.com/apache/arrow-datafusion/issues/63) +- Remove hard-coded Ballista version from scripts [\#32](https://github.com/apache/arrow-datafusion/issues/32) +- Implement streaming versions of Dataframe.collect methods [\#789](https://github.com/apache/arrow-datafusion/pull/789) ([andygrove](https://github.com/andygrove)) +- Ballista shuffle is finally working as intended, providing scalable distributed joins [\#750](https://github.com/apache/arrow-datafusion/pull/750) ([andygrove](https://github.com/andygrove)) +- Update to use arrow 5.0 [\#721](https://github.com/apache/arrow-datafusion/pull/721) ([alamb](https://github.com/alamb)) +- Implement serde for ShuffleWriterExec [\#712](https://github.com/apache/arrow-datafusion/pull/712) ([andygrove](https://github.com/andygrove)) +- dedup using join column in wildcard expansion [\#678](https://github.com/apache/arrow-datafusion/pull/678) ([houqp](https://github.com/houqp)) +- Implement metrics for shuffle read and write [\#676](https://github.com/apache/arrow-datafusion/pull/676) ([andygrove](https://github.com/andygrove)) +- Remove hard-coded PartitionMode from Ballista serde [\#637](https://github.com/apache/arrow-datafusion/pull/637) ([andygrove](https://github.com/andygrove)) +- Ballista: Implement scalable distributed joins [\#634](https://github.com/apache/arrow-datafusion/pull/634) ([andygrove](https://github.com/andygrove)) +- Add Keda autoscaling for ballista in k8s [\#586](https://github.com/apache/arrow-datafusion/pull/586) ([edrevo](https://github.com/edrevo)) +- Add some resiliency to lost executors [\#568](https://github.com/apache/arrow-datafusion/pull/568) ([edrevo](https://github.com/edrevo)) +- Add `partition by` constructs in window functions and modify logical planning [\#501](https://github.com/apache/arrow-datafusion/pull/501) ([Jimexist](https://github.com/Jimexist)) +- Support anti join [\#482](https://github.com/apache/arrow-datafusion/pull/482) ([Dandandan](https://github.com/Dandandan)) +- add `order by` construct in window function and logical plans [\#463](https://github.com/apache/arrow-datafusion/pull/463) ([Jimexist](https://github.com/Jimexist)) +- Refactor Ballista executor so that FlightService delegates to an Executor struct [\#450](https://github.com/apache/arrow-datafusion/pull/450) ([andygrove](https://github.com/andygrove)) +- implement lead and lag built-in window function [\#429](https://github.com/apache/arrow-datafusion/pull/429) ([Jimexist](https://github.com/Jimexist)) +- Implement fmt_as for ShuffleReaderExec [\#400](https://github.com/apache/arrow-datafusion/pull/400) ([andygrove](https://github.com/andygrove)) +- Add window expression part 1 - logical and physical planning, structure, to/from proto, and explain, for empty over clause only [\#334](https://github.com/apache/arrow-datafusion/pull/334) ([Jimexist](https://github.com/Jimexist)) +- \[breaking change\] fix 265, log should be log10, and add ln [\#271](https://github.com/apache/arrow-datafusion/pull/271) ([Jimexist](https://github.com/Jimexist)) +- Allow table providers to indicate their type for catalog metadata [\#205](https://github.com/apache/arrow-datafusion/pull/205) ([returnString](https://github.com/returnString)) +- Add query 19 to TPC-H regression tests [\#59](https://github.com/apache/arrow-datafusion/pull/59) ([Dandandan](https://github.com/Dandandan)) +- Use arrow eq kernels in CaseWhen expression evaluation [\#52](https://github.com/apache/arrow-datafusion/pull/52) ([Dandandan](https://github.com/Dandandan)) +- Add option param for standalone mode [\#42](https://github.com/apache/arrow-datafusion/pull/42) ([djKooks](https://github.com/djKooks)) +- \[DataFusion\] Optimize hash join inner workings, null handling fix [\#24](https://github.com/apache/arrow-datafusion/pull/24) ([Dandandan](https://github.com/Dandandan)) +- \[Ballista\] Docker files for ui [\#22](https://github.com/apache/arrow-datafusion/pull/22) ([msathis](https://github.com/msathis)) + +**Fixed bugs:** + +- Ballista: TPC-H q3 @ SF=1000 never completes [\#835](https://github.com/apache/arrow-datafusion/issues/835) +- Ballista does not support MIN/MAX aggregate functions [\#832](https://github.com/apache/arrow-datafusion/issues/832) +- Ballista docker images fail to build [\#828](https://github.com/apache/arrow-datafusion/issues/828) +- Ballista: UnresolvedShuffleExec should only have a single stage_id [\#726](https://github.com/apache/arrow-datafusion/issues/726) +- Ballista integration tests are failing [\#623](https://github.com/apache/arrow-datafusion/issues/623) +- Integration test build failure due to arrow-rs using unstable feature [\#596](https://github.com/apache/arrow-datafusion/issues/596) +- `cargo build` cannot build the project [\#531](https://github.com/apache/arrow-datafusion/issues/531) +- ShuffleReaderExec does not get formatted correctly in displayable physical plan [\#399](https://github.com/apache/arrow-datafusion/issues/399) +- Implement serde for MIN and MAX [\#833](https://github.com/apache/arrow-datafusion/pull/833) ([andygrove](https://github.com/andygrove)) +- Ballista: Prep for fixing shuffle mechansim, part 1 [\#738](https://github.com/apache/arrow-datafusion/pull/738) ([andygrove](https://github.com/andygrove)) +- Ballista: Shuffle write bug fix [\#714](https://github.com/apache/arrow-datafusion/pull/714) ([andygrove](https://github.com/andygrove)) +- honor table name for csv/parquet scan in ballista plan serde [\#629](https://github.com/apache/arrow-datafusion/pull/629) ([houqp](https://github.com/houqp)) +- MINOR: Fix integration tests by adding datafusion-cli module to docker image [\#322](https://github.com/apache/arrow-datafusion/pull/322) ([andygrove](https://github.com/andygrove)) + +**Documentation updates:** + +- Add minimal crate documentation for Ballista crates [\#831](https://github.com/apache/arrow-datafusion/pull/831) ([andygrove](https://github.com/andygrove)) +- Add Ballista examples [\#775](https://github.com/apache/arrow-datafusion/pull/775) ([andygrove](https://github.com/andygrove)) +- Update ballista.proto link in architecture doc [\#502](https://github.com/apache/arrow-datafusion/pull/502) ([terrycorley](https://github.com/terrycorley)) +- Update k8s user guide to use deployments [\#474](https://github.com/apache/arrow-datafusion/pull/474) ([edrevo](https://github.com/edrevo)) +- use prettier to format md files [\#367](https://github.com/apache/arrow-datafusion/pull/367) ([Jimexist](https://github.com/Jimexist)) +- Make it easier for developers to find Ballista documentation [\#330](https://github.com/apache/arrow-datafusion/pull/330) ([andygrove](https://github.com/andygrove)) +- Instructions for cross-compiling Ballista to the Raspberry Pi [\#263](https://github.com/apache/arrow-datafusion/pull/263) ([andygrove](https://github.com/andygrove)) +- Add install guide in README [\#236](https://github.com/apache/arrow-datafusion/pull/236) ([djKooks](https://github.com/djKooks)) + +**Performance improvements:** + +- Ballista: Avoid sleeping between polling for tasks [\#698](https://github.com/apache/arrow-datafusion/pull/698) ([Dandandan](https://github.com/Dandandan)) +- Make BallistaContext::collect streaming [\#535](https://github.com/apache/arrow-datafusion/pull/535) ([edrevo](https://github.com/edrevo)) + +**Closed issues:** + +- Confirm git tagging strategy for releases [\#770](https://github.com/apache/arrow-datafusion/issues/770) +- arrow::util::pretty::pretty_format_batches missing [\#769](https://github.com/apache/arrow-datafusion/issues/769) +- move the `assert_batches_eq!` macros to a non part of datafusion [\#745](https://github.com/apache/arrow-datafusion/issues/745) +- fix an issue where aliases are not respected in generating downstream schemas in window expr [\#592](https://github.com/apache/arrow-datafusion/issues/592) +- make the planner to print more succinct and useful information in window function explain clause [\#526](https://github.com/apache/arrow-datafusion/issues/526) +- move window frame module to be in `logical_plan` [\#517](https://github.com/apache/arrow-datafusion/issues/517) +- use a more rust idiomatic way of handling nth_value [\#448](https://github.com/apache/arrow-datafusion/issues/448) +- Make Ballista not depend on arrow directly [\#446](https://github.com/apache/arrow-datafusion/issues/446) +- create a test with more than one partition for window functions [\#435](https://github.com/apache/arrow-datafusion/issues/435) +- Implement hash-partitioned hash aggregate [\#27](https://github.com/apache/arrow-datafusion/issues/27) +- Consider using GitHub pages for DataFusion/Ballista documentation [\#18](https://github.com/apache/arrow-datafusion/issues/18) +- Add Ballista to default cargo workspace [\#17](https://github.com/apache/arrow-datafusion/issues/17) +- Update "repository" in Cargo.toml [\#16](https://github.com/apache/arrow-datafusion/issues/16) +- Consolidate TPC-H benchmarks [\#6](https://github.com/apache/arrow-datafusion/issues/6) +- \[Ballista\] Fix integration test script [\#4](https://github.com/apache/arrow-datafusion/issues/4) +- Ballista should not have separate DataFrame implementation [\#2](https://github.com/apache/arrow-datafusion/issues/2) + +**Merged pull requests:** + +- Change datatype of tpch keys from Int32 to UInt64 to support sf=1000 [\#836](https://github.com/apache/arrow-datafusion/pull/836) ([andygrove](https://github.com/andygrove)) +- Add ballista-examples to docker build [\#829](https://github.com/apache/arrow-datafusion/pull/829) ([andygrove](https://github.com/andygrove)) +- Update dependencies: prost to 0.8 and tonic to 0.5 [\#818](https://github.com/apache/arrow-datafusion/pull/818) ([alamb](https://github.com/alamb)) +- Move `hash_array` into hash_utils.rs [\#807](https://github.com/apache/arrow-datafusion/pull/807) ([alamb](https://github.com/alamb)) +- Fix: Update clippy lints for Rust 1.54 [\#794](https://github.com/apache/arrow-datafusion/pull/794) ([alamb](https://github.com/alamb)) +- MINOR: Remove unused Ballista query execution code path [\#732](https://github.com/apache/arrow-datafusion/pull/732) ([andygrove](https://github.com/andygrove)) +- \[fix\] benchmark run with compose [\#666](https://github.com/apache/arrow-datafusion/pull/666) ([rdettai](https://github.com/rdettai)) +- bring back dev scripts for ballista [\#648](https://github.com/apache/arrow-datafusion/pull/648) ([Jimexist](https://github.com/Jimexist)) +- Remove unnecessary mutex [\#639](https://github.com/apache/arrow-datafusion/pull/639) ([edrevo](https://github.com/edrevo)) +- round trip TPCH queries in tests [\#630](https://github.com/apache/arrow-datafusion/pull/630) ([houqp](https://github.com/houqp)) +- Fix build [\#627](https://github.com/apache/arrow-datafusion/pull/627) ([andygrove](https://github.com/andygrove)) +- in ballista also check for UI prettier changes [\#578](https://github.com/apache/arrow-datafusion/pull/578) ([Jimexist](https://github.com/Jimexist)) +- turn on clippy rule for needless borrow [\#545](https://github.com/apache/arrow-datafusion/pull/545) ([Jimexist](https://github.com/Jimexist)) +- reuse datafusion physical planner in ballista building from protobuf [\#532](https://github.com/apache/arrow-datafusion/pull/532) ([Jimexist](https://github.com/Jimexist)) +- update cargo.toml in python crate and fix unit test due to hash joins [\#483](https://github.com/apache/arrow-datafusion/pull/483) ([Jimexist](https://github.com/Jimexist)) +- make `VOLUME` declaration in tpch datagen docker absolute [\#466](https://github.com/apache/arrow-datafusion/pull/466) ([crepererum](https://github.com/crepererum)) +- Refactor QueryStageExec in preparation for implementing map-side shuffle [\#459](https://github.com/apache/arrow-datafusion/pull/459) ([andygrove](https://github.com/andygrove)) +- Simplified usage of `use arrow` in ballista. [\#447](https://github.com/apache/arrow-datafusion/pull/447) ([jorgecarleitao](https://github.com/jorgecarleitao)) +- Benchmark subcommand to distinguish between DataFusion and Ballista [\#402](https://github.com/apache/arrow-datafusion/pull/402) ([jgoday](https://github.com/jgoday)) +- \#352: BallistaContext::collect\(\) logging is too noisy [\#394](https://github.com/apache/arrow-datafusion/pull/394) ([jgoday](https://github.com/jgoday)) +- cleanup function return type fn [\#350](https://github.com/apache/arrow-datafusion/pull/350) ([Jimexist](https://github.com/Jimexist)) +- Update Ballista to use new physical plan formatter utility [\#344](https://github.com/apache/arrow-datafusion/pull/344) ([andygrove](https://github.com/andygrove)) +- Update arrow dependencies again [\#341](https://github.com/apache/arrow-datafusion/pull/341) ([alamb](https://github.com/alamb)) +- Remove references to Ballista Docker images published to ballistacompute Docker Hub repo [\#326](https://github.com/apache/arrow-datafusion/pull/326) ([andygrove](https://github.com/andygrove)) +- Update arrow-rs deps [\#317](https://github.com/apache/arrow-datafusion/pull/317) ([alamb](https://github.com/alamb)) +- Update arrow deps [\#269](https://github.com/apache/arrow-datafusion/pull/269) ([alamb](https://github.com/alamb)) +- Enable redundant_field_names clippy lint [\#261](https://github.com/apache/arrow-datafusion/pull/261) ([Dandandan](https://github.com/Dandandan)) +- Update arrow-rs deps \(to fix build due to flatbuffers update\) [\#224](https://github.com/apache/arrow-datafusion/pull/224) ([alamb](https://github.com/alamb)) +- update arrow-rs deps to latest master [\#216](https://github.com/apache/arrow-datafusion/pull/216) ([alamb](https://github.com/alamb)) + +\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ diff --git a/docs/source/changelog/0.6.0.md b/docs/source/changelog/0.6.0.md new file mode 100644 index 0000000000..9657586118 --- /dev/null +++ b/docs/source/changelog/0.6.0.md @@ -0,0 +1,105 @@ + + +# Apache DataFusion Ballista 0.6.0 Changelog + +[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.5.0...ballista-0.6.0) + +**Breaking changes:** + +- File partitioning for ListingTable [\#1141](https://github.com/apache/arrow-datafusion/pull/1141) ([rdettai](https://github.com/rdettai)) +- Register tables in BallistaContext using TableProviders instead of Dataframe [\#1028](https://github.com/apache/arrow-datafusion/pull/1028) ([rdettai](https://github.com/rdettai)) +- Make TableProvider.scan\(\) and PhysicalPlanner::create_physical_plan\(\) async [\#1013](https://github.com/apache/arrow-datafusion/pull/1013) ([rdettai](https://github.com/rdettai)) +- Reorganize table providers by table format [\#1010](https://github.com/apache/arrow-datafusion/pull/1010) ([rdettai](https://github.com/rdettai)) +- Move CBOs and Statistics to physical plan [\#965](https://github.com/apache/arrow-datafusion/pull/965) ([rdettai](https://github.com/rdettai)) +- Update to sqlparser v 0.10.0 [\#934](https://github.com/apache/arrow-datafusion/pull/934) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) +- FilePartition and PartitionedFile for scanning flexibility [\#932](https://github.com/apache/arrow-datafusion/pull/932) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([yjshen](https://github.com/yjshen)) +- Improve SQLMetric APIs, port existing metrics [\#908](https://github.com/apache/arrow-datafusion/pull/908) ([alamb](https://github.com/alamb)) +- Add support for EXPLAIN ANALYZE [\#858](https://github.com/apache/arrow-datafusion/pull/858) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) +- Rename concurrency to target_partitions [\#706](https://github.com/apache/arrow-datafusion/pull/706) ([andygrove](https://github.com/andygrove)) + +**Implemented enhancements:** + +- Update datafusion-cli to support Ballista, or implement new ballista-cli [\#886](https://github.com/apache/arrow-datafusion/issues/886) +- Prepare Ballista crates for publishing [\#509](https://github.com/apache/arrow-datafusion/issues/509) +- Add drop table support [\#1266](https://github.com/apache/arrow-datafusion/pull/1266) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([viirya](https://github.com/viirya)) +- use arrow 6.1.0 [\#1255](https://github.com/apache/arrow-datafusion/pull/1255) ([Jimexist](https://github.com/Jimexist)) +- Add support for `create table as` via MemTable [\#1243](https://github.com/apache/arrow-datafusion/pull/1243) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan)) +- add values list expression [\#1165](https://github.com/apache/arrow-datafusion/pull/1165) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist)) +- Multiple files per partitions for CSV Avro Json [\#1138](https://github.com/apache/arrow-datafusion/pull/1138) ([rdettai](https://github.com/rdettai)) +- Implement INTERSECT & INTERSECT DISTINCT [\#1135](https://github.com/apache/arrow-datafusion/pull/1135) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963)) +- Simplify file struct abstractions [\#1120](https://github.com/apache/arrow-datafusion/pull/1120) ([rdettai](https://github.com/rdettai)) +- Implement `is [not] distinct from` [\#1117](https://github.com/apache/arrow-datafusion/pull/1117) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan)) +- add digest\(utf8, method\) function and refactor all current hash digest functions [\#1090](https://github.com/apache/arrow-datafusion/pull/1090) ([Jimexist](https://github.com/Jimexist)) +- \[crypto\] add `blake3` algorithm to `digest` function [\#1086](https://github.com/apache/arrow-datafusion/pull/1086) ([Jimexist](https://github.com/Jimexist)) +- \[crypto\] add blake2b and blake2s functions [\#1081](https://github.com/apache/arrow-datafusion/pull/1081) ([Jimexist](https://github.com/Jimexist)) +- Update sqlparser-rs to 0.11 [\#1052](https://github.com/apache/arrow-datafusion/pull/1052) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) +- remove hard coded partition count in ballista logicalplan deserialization [\#1044](https://github.com/apache/arrow-datafusion/pull/1044) ([xudong963](https://github.com/xudong963)) +- Indexed field access for List [\#1006](https://github.com/apache/arrow-datafusion/pull/1006) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki)) +- Update DataFusion to arrow 6.0 [\#984](https://github.com/apache/arrow-datafusion/pull/984) ([alamb](https://github.com/alamb)) +- Implement Display for Expr, improve operator display [\#971](https://github.com/apache/arrow-datafusion/pull/971) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner)) +- ObjectStore API to read from remote storage systems [\#950](https://github.com/apache/arrow-datafusion/pull/950) ([yjshen](https://github.com/yjshen)) +- fixes \#933 replace placeholder fmt_as fr ExecutionPlan impls [\#939](https://github.com/apache/arrow-datafusion/pull/939) ([tiphaineruy](https://github.com/tiphaineruy)) +- Support `NotLike` in Ballista [\#916](https://github.com/apache/arrow-datafusion/pull/916) ([Dandandan](https://github.com/Dandandan)) +- Avro Table Provider [\#910](https://github.com/apache/arrow-datafusion/pull/910) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki)) +- Add BaselineMetrics, Timestamp metrics, add for `CoalescePartitionsExec`, rename output_time -\> elapsed_compute [\#909](https://github.com/apache/arrow-datafusion/pull/909) ([alamb](https://github.com/alamb)) +- \[Ballista\] Add executor last seen info to the ui [\#895](https://github.com/apache/arrow-datafusion/pull/895) ([msathis](https://github.com/msathis)) +- add cross join support to ballista [\#891](https://github.com/apache/arrow-datafusion/pull/891) ([houqp](https://github.com/houqp)) +- Add Ballista support to DataFusion CLI [\#889](https://github.com/apache/arrow-datafusion/pull/889) ([andygrove](https://github.com/andygrove)) +- Add support for PostgreSQL regex match [\#870](https://github.com/apache/arrow-datafusion/pull/870) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([b41sh](https://github.com/b41sh)) + +**Fixed bugs:** + +- Test execution_plans::shuffle_writer::tests::test Fail [\#1040](https://github.com/apache/arrow-datafusion/issues/1040) +- Integration test fails to build docker images [\#918](https://github.com/apache/arrow-datafusion/issues/918) +- Ballista: Remove hard-coded concurrency from logical plan serde code [\#708](https://github.com/apache/arrow-datafusion/issues/708) +- How can I make ballista distributed compute work? [\#327](https://github.com/apache/arrow-datafusion/issues/327) +- fix subquery alias [\#1067](https://github.com/apache/arrow-datafusion/pull/1067) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963)) +- Fix compilation for ballista in stand-alone mode [\#1008](https://github.com/apache/arrow-datafusion/pull/1008) ([Igosuki](https://github.com/Igosuki)) + +**Documentation updates:** + +- Add Ballista roadmap [\#1166](https://github.com/apache/arrow-datafusion/pull/1166) ([andygrove](https://github.com/andygrove)) +- Adds note on compatible rust version [\#1097](https://github.com/apache/arrow-datafusion/pull/1097) ([1nF0rmed](https://github.com/1nF0rmed)) +- implement `approx_distinct` function using HyperLogLog [\#1087](https://github.com/apache/arrow-datafusion/pull/1087) ([Jimexist](https://github.com/Jimexist)) +- Improve User Guide [\#954](https://github.com/apache/arrow-datafusion/pull/954) ([andygrove](https://github.com/andygrove)) +- Update plan_query_stages doc [\#951](https://github.com/apache/arrow-datafusion/pull/951) ([rdettai](https://github.com/rdettai)) +- \[DataFusion\] - Add show and show_limit function for DataFrame [\#923](https://github.com/apache/arrow-datafusion/pull/923) ([francis-du](https://github.com/francis-du)) +- update docs related to protoc and optional syntax [\#902](https://github.com/apache/arrow-datafusion/pull/902) ([Jimexist](https://github.com/Jimexist)) +- Improve Ballista crate README content [\#878](https://github.com/apache/arrow-datafusion/pull/878) ([andygrove](https://github.com/andygrove)) + +**Performance improvements:** + +- optimize build profile for datafusion python binding, cli and ballista [\#1137](https://github.com/apache/arrow-datafusion/pull/1137) ([houqp](https://github.com/houqp)) + +**Closed issues:** + +- InList expr with NULL literals do not work [\#1190](https://github.com/apache/arrow-datafusion/issues/1190) +- update the homepage README to include values, `approx_distinct`, etc. [\#1171](https://github.com/apache/arrow-datafusion/issues/1171) +- \[Python\]: Inconsistencies with Python package name [\#1011](https://github.com/apache/arrow-datafusion/issues/1011) +- Wanting to contribute to project where to start? [\#983](https://github.com/apache/arrow-datafusion/issues/983) +- delete redundant code [\#973](https://github.com/apache/arrow-datafusion/issues/973) +- How to build DataFusion python wheel [\#853](https://github.com/apache/arrow-datafusion/issues/853) +- Produce a design for a metrics framework [\#21](https://github.com/apache/arrow-datafusion/issues/21) + +**Merged pull requests:** + +- \[nit\] simplify ballista executor `CollectExec` impl codes [\#1140](https://github.com/apache/arrow-datafusion/pull/1140) ([panarch](https://github.com/panarch)) + +For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md) diff --git a/docs/source/changelog/0.7.0.md b/docs/source/changelog/0.7.0.md new file mode 100644 index 0000000000..7fffd781f9 --- /dev/null +++ b/docs/source/changelog/0.7.0.md @@ -0,0 +1,165 @@ + + +# Apache DataFusion Ballista 0.7.0 Changelog + +[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.1.0-rc1...ballista-0.7.0) + +**Breaking changes:** + +- Make `ExecutionPlan::execute` Sync [\#2434](https://github.com/apache/arrow-datafusion/pull/2434) ([tustvold](https://github.com/tustvold)) +- Add `Expr::Exists` to represent EXISTS subquery expression [\#2339](https://github.com/apache/arrow-datafusion/pull/2339) ([andygrove](https://github.com/andygrove)) +- Remove dependency from `LogicalPlan::TableScan` to `ExecutionPlan` [\#2284](https://github.com/apache/arrow-datafusion/pull/2284) ([andygrove](https://github.com/andygrove)) +- Move logical expression type-coercion code from `physical-expr` crate to `expr` crate [\#2257](https://github.com/apache/arrow-datafusion/pull/2257) ([andygrove](https://github.com/andygrove)) +- feat: 2061 create external table ddl table partition cols [\#2099](https://github.com/apache/arrow-datafusion/pull/2099) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jychen7](https://github.com/jychen7)) +- Reorganize the project folders [\#2081](https://github.com/apache/arrow-datafusion/pull/2081) ([yahoNanJing](https://github.com/yahoNanJing)) +- Support more ScalarFunction in Ballista [\#2008](https://github.com/apache/arrow-datafusion/pull/2008) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Merge dataframe and dataframe imp [\#1998](https://github.com/apache/arrow-datafusion/pull/1998) ([vchag](https://github.com/vchag)) +- Rename `ExecutionContext` to `SessionContext`, `ExecutionContextState` to `SessionState`, add `TaskContext` to support multi-tenancy configurations - Part 1 [\#1987](https://github.com/apache/arrow-datafusion/pull/1987) ([mingmwang](https://github.com/mingmwang)) +- Add Coalesce function [\#1969](https://github.com/apache/arrow-datafusion/pull/1969) ([msathis](https://github.com/msathis)) +- Add Create Schema functionality in SQL [\#1959](https://github.com/apache/arrow-datafusion/pull/1959) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner)) +- remove sync constraint of SendableRecordBatchStream [\#1884](https://github.com/apache/arrow-datafusion/pull/1884) ([doki23](https://github.com/doki23)) + +**Implemented enhancements:** + +- Add `CREATE VIEW` [\#2279](https://github.com/apache/arrow-datafusion/pull/2279) ([matthewmturner](https://github.com/matthewmturner)) +- \[Ballista\] Support Union in ballista. [\#2098](https://github.com/apache/arrow-datafusion/pull/2098) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Add missing aggr_expr to PhysicalExprNode for Ballista. [\#1989](https://github.com/apache/arrow-datafusion/pull/1989) ([Ted-Jiang](https://github.com/Ted-Jiang)) + +**Fixed bugs:** + +- Ballista integration tests no longer work [\#2440](https://github.com/apache/arrow-datafusion/issues/2440) +- Ballista crates cannot be released from DafaFusion 7.0.0 source release [\#1980](https://github.com/apache/arrow-datafusion/issues/1980) +- protobuf OctetLength should be deserialized as octet_length, not length [\#1834](https://github.com/apache/arrow-datafusion/pull/1834) ([carols10cents](https://github.com/carols10cents)) + +**Documentation updates:** + +- MINOR: Make crate READMEs consistent [\#2437](https://github.com/apache/arrow-datafusion/pull/2437) ([andygrove](https://github.com/andygrove)) +- docs: Update the Ballista dev env instructions [\#2419](https://github.com/apache/arrow-datafusion/pull/2419) ([haoxins](https://github.com/haoxins)) +- Revise document of installing ballista pinned to specified version [\#2034](https://github.com/apache/arrow-datafusion/pull/2034) ([WinkerDu](https://github.com/WinkerDu)) +- Fix typos \(Datafusion -\> DataFusion\) [\#1993](https://github.com/apache/arrow-datafusion/pull/1993) ([andygrove](https://github.com/andygrove)) + +**Performance improvements:** + +- Introduce StageManager for managing tasks stage by stage [\#1983](https://github.com/apache/arrow-datafusion/pull/1983) ([yahoNanJing](https://github.com/yahoNanJing)) + +**Closed issues:** + +- Make expected result string in unit tests more readable [\#2412](https://github.com/apache/arrow-datafusion/issues/2412) +- remove duplicated `fn aggregate()` in aggregate expression tests [\#2399](https://github.com/apache/arrow-datafusion/issues/2399) +- split `distinct_expression.rs` into `count_distinct.rs` and `array_agg_distinct.rs` [\#2385](https://github.com/apache/arrow-datafusion/issues/2385) +- move sql tests in `context.rs` to corresponding test files in `datafustion/core/tests/sql` [\#2328](https://github.com/apache/arrow-datafusion/issues/2328) +- Date32/Date64 as join keys for merge join [\#2314](https://github.com/apache/arrow-datafusion/issues/2314) +- Error precision and scale for decimal coercion in logic comparison [\#2232](https://github.com/apache/arrow-datafusion/issues/2232) +- Support Multiple row layout [\#2188](https://github.com/apache/arrow-datafusion/issues/2188) +- Discussion: Is Ballista a standalone system or framework [\#1916](https://github.com/apache/arrow-datafusion/issues/1916) + +**Merged pull requests:** + +- MINOR: Enable multi-statement benchmark queries [\#2507](https://github.com/apache/arrow-datafusion/pull/2507) ([andygrove](https://github.com/andygrove)) +- Persist session configs in scheduler [\#2501](https://github.com/apache/arrow-datafusion/pull/2501) ([thinkharderdev](https://github.com/thinkharderdev)) +- Update to `sqlparser` `0.17.0` [\#2500](https://github.com/apache/arrow-datafusion/pull/2500) ([alamb](https://github.com/alamb)) +- Limit cpu cores used when generating changelog [\#2494](https://github.com/apache/arrow-datafusion/pull/2494) ([andygrove](https://github.com/andygrove)) +- MINOR: Parameterize changelog script [\#2484](https://github.com/apache/arrow-datafusion/pull/2484) ([jychen7](https://github.com/jychen7)) +- Fix stage key extraction [\#2472](https://github.com/apache/arrow-datafusion/pull/2472) ([thinkharderdev](https://github.com/thinkharderdev)) +- Add support for list_dir\(\) on local fs [\#2467](https://github.com/apache/arrow-datafusion/pull/2467) ([wjones127](https://github.com/wjones127)) +- minor: update versions and paths in changelog scripts [\#2429](https://github.com/apache/arrow-datafusion/pull/2429) ([andygrove](https://github.com/andygrove)) +- Fix Ballista executing during plan [\#2428](https://github.com/apache/arrow-datafusion/pull/2428) ([tustvold](https://github.com/tustvold)) +- Re-organize and rename aggregates physical plan [\#2388](https://github.com/apache/arrow-datafusion/pull/2388) ([yjshen](https://github.com/yjshen)) +- Upgrade to arrow 13 [\#2382](https://github.com/apache/arrow-datafusion/pull/2382) ([alamb](https://github.com/alamb)) +- Grouped Aggregate in row format [\#2375](https://github.com/apache/arrow-datafusion/pull/2375) ([yjshen](https://github.com/yjshen)) +- Stop optimizing queries twice [\#2369](https://github.com/apache/arrow-datafusion/pull/2369) ([andygrove](https://github.com/andygrove)) +- Bump follow-redirects from 1.13.2 to 1.14.9 in /ballista/ui/scheduler [\#2325](https://github.com/apache/arrow-datafusion/pull/2325) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Move FileType enum from sql module to logical_plan module [\#2290](https://github.com/apache/arrow-datafusion/pull/2290) ([andygrove](https://github.com/andygrove)) +- Add BatchPartitioner \(\#2285\) [\#2287](https://github.com/apache/arrow-datafusion/pull/2287) ([tustvold](https://github.com/tustvold)) +- Update uuid requirement from 0.8 to 1.0 [\#2280](https://github.com/apache/arrow-datafusion/pull/2280) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump async from 2.6.3 to 2.6.4 in /ballista/ui/scheduler [\#2277](https://github.com/apache/arrow-datafusion/pull/2277) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump minimist from 1.2.5 to 1.2.6 in /ballista/ui/scheduler [\#2276](https://github.com/apache/arrow-datafusion/pull/2276) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump url-parse from 1.5.1 to 1.5.10 in /ballista/ui/scheduler [\#2275](https://github.com/apache/arrow-datafusion/pull/2275) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump nanoid from 3.1.20 to 3.3.3 in /ballista/ui/scheduler [\#2274](https://github.com/apache/arrow-datafusion/pull/2274) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Update to Arrow 12.0.0, update tonic and prost [\#2253](https://github.com/apache/arrow-datafusion/pull/2253) ([alamb](https://github.com/alamb)) +- Add ExecutorMetricsCollector interface [\#2234](https://github.com/apache/arrow-datafusion/pull/2234) ([thinkharderdev](https://github.com/thinkharderdev)) +- minor: add editor config file [\#2224](https://github.com/apache/arrow-datafusion/pull/2224) ([jackwener](https://github.com/jackwener)) +- \[Ballista\] Enable ApproxPercentileWithWeight in Ballista and fill UT [\#2192](https://github.com/apache/arrow-datafusion/pull/2192) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- make nightly clippy happy [\#2186](https://github.com/apache/arrow-datafusion/pull/2186) ([xudong963](https://github.com/xudong963)) +- \[Ballista\]Make PhysicalAggregateExprNode has repeated PhysicalExprNode [\#2184](https://github.com/apache/arrow-datafusion/pull/2184) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Add LogicalPlan::SubqueryAlias [\#2172](https://github.com/apache/arrow-datafusion/pull/2172) ([andygrove](https://github.com/andygrove)) +- Implement fast path of with_new_children\(\) in ExecutionPlan [\#2168](https://github.com/apache/arrow-datafusion/pull/2168) ([mingmwang](https://github.com/mingmwang)) +- \[MINOR\] ignore suspicious slow test in Ballista [\#2167](https://github.com/apache/arrow-datafusion/pull/2167) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- enable explain for ballista [\#2163](https://github.com/apache/arrow-datafusion/pull/2163) ([doki23](https://github.com/doki23)) +- Add delimiter for create external table [\#2162](https://github.com/apache/arrow-datafusion/pull/2162) ([matthewmturner](https://github.com/matthewmturner)) +- Update sqlparser requirement from 0.15 to 0.16 [\#2152](https://github.com/apache/arrow-datafusion/pull/2152) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add IF NOT EXISTS to `CREATE TABLE` and `CREATE EXTERNAL TABLE` [\#2143](https://github.com/apache/arrow-datafusion/pull/2143) ([matthewmturner](https://github.com/matthewmturner)) +- Update quarterly roadmap for Q2 [\#2133](https://github.com/apache/arrow-datafusion/pull/2133) ([matthewmturner](https://github.com/matthewmturner)) +- \[Ballista\] Add ballista plugin manager and UDF plugin [\#2131](https://github.com/apache/arrow-datafusion/pull/2131) ([gaojun2048](https://github.com/gaojun2048)) +- Serialize scalar UDFs in physical plan [\#2130](https://github.com/apache/arrow-datafusion/pull/2130) ([thinkharderdev](https://github.com/thinkharderdev)) +- doc: update release schedule [\#2110](https://github.com/apache/arrow-datafusion/pull/2110) ([jychen7](https://github.com/jychen7)) +- Reduce repetition in Decimal binary kernels, upgrade to arrow 11.1 [\#2107](https://github.com/apache/arrow-datafusion/pull/2107) ([alamb](https://github.com/alamb)) +- update zlib version to 1.2.12 [\#2106](https://github.com/apache/arrow-datafusion/pull/2106) ([waitingkuo](https://github.com/waitingkuo)) +- Add CREATE DATABASE command to SQL [\#2094](https://github.com/apache/arrow-datafusion/pull/2094) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner)) +- Refactor SessionContext, BallistaContext to support multi-tenancy configurations - Part 3 [\#2091](https://github.com/apache/arrow-datafusion/pull/2091) ([mingmwang](https://github.com/mingmwang)) +- Remove dependency of common for the storage crate [\#2076](https://github.com/apache/arrow-datafusion/pull/2076) ([yahoNanJing](https://github.com/yahoNanJing)) +- [MINOR] fix doc in `EXTRACT\(field FROM source\) [\#2074](https://github.com/apache/arrow-datafusion/pull/2074) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- \[Bug\]\[Datafusion\] fix TaskContext session_config bug [\#2070](https://github.com/apache/arrow-datafusion/pull/2070) ([gaojun2048](https://github.com/gaojun2048)) +- Short-circuit evaluation for `CaseWhen` [\#2068](https://github.com/apache/arrow-datafusion/pull/2068) ([yjshen](https://github.com/yjshen)) +- split datafusion-object-store module [\#2065](https://github.com/apache/arrow-datafusion/pull/2065) ([yahoNanJing](https://github.com/yahoNanJing)) +- Change log level for noisy logs [\#2060](https://github.com/apache/arrow-datafusion/pull/2060) ([thinkharderdev](https://github.com/thinkharderdev)) +- Update to arrow/parquet 11.0 [\#2048](https://github.com/apache/arrow-datafusion/pull/2048) ([alamb](https://github.com/alamb)) +- minor: format comments \(`//` to `//`\) [\#2047](https://github.com/apache/arrow-datafusion/pull/2047) ([jackwener](https://github.com/jackwener)) +- use cargo-tomlfmt to check Cargo.toml formatting in CI [\#2033](https://github.com/apache/arrow-datafusion/pull/2033) ([WinkerDu](https://github.com/WinkerDu)) +- Refactor SessionContext, SessionState and SessionConfig to support multi-tenancy configurations - Part 2 [\#2029](https://github.com/apache/arrow-datafusion/pull/2029) ([mingmwang](https://github.com/mingmwang)) +- Simplify prerequisites for running examples [\#2028](https://github.com/apache/arrow-datafusion/pull/2028) ([doki23](https://github.com/doki23)) +- Use SessionContext to parse Expr protobuf [\#2024](https://github.com/apache/arrow-datafusion/pull/2024) ([thinkharderdev](https://github.com/thinkharderdev)) +- Fix stuck issue for the load testing of Push-based task scheduling [\#2006](https://github.com/apache/arrow-datafusion/pull/2006) ([yahoNanJing](https://github.com/yahoNanJing)) +- Fixing a typo in documentation [\#1997](https://github.com/apache/arrow-datafusion/pull/1997) ([psvri](https://github.com/psvri)) +- Fix minor clippy issue [\#1995](https://github.com/apache/arrow-datafusion/pull/1995) ([alamb](https://github.com/alamb)) +- Make it possible to only scan part of a parquet file in a partition [\#1990](https://github.com/apache/arrow-datafusion/pull/1990) ([yjshen](https://github.com/yjshen)) +- Update Dockerfile to fix integration tests [\#1982](https://github.com/apache/arrow-datafusion/pull/1982) ([andygrove](https://github.com/andygrove)) +- Update sqlparser requirement from 0.14 to 0.15 [\#1966](https://github.com/apache/arrow-datafusion/pull/1966) ([dependabot[bot]](https://github.com/apps/dependabot)) +- fix logical conflict with protobuf [\#1958](https://github.com/apache/arrow-datafusion/pull/1958) ([alamb](https://github.com/alamb)) +- Update to arrow 10.0.0, pyo3 0.16 [\#1957](https://github.com/apache/arrow-datafusion/pull/1957) ([alamb](https://github.com/alamb)) +- update jit-related dependencies [\#1953](https://github.com/apache/arrow-datafusion/pull/1953) ([xudong963](https://github.com/xudong963)) +- Allow different types of query variables \(`@@var`\) rather than just string [\#1943](https://github.com/apache/arrow-datafusion/pull/1943) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([maxburke](https://github.com/maxburke)) +- Pruning serialization [\#1941](https://github.com/apache/arrow-datafusion/pull/1941) ([thinkharderdev](https://github.com/thinkharderdev)) +- Fix select from EmptyExec always return 0 row after optimizer passes [\#1938](https://github.com/apache/arrow-datafusion/pull/1938) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Introduce Ballista query stage scheduler [\#1935](https://github.com/apache/arrow-datafusion/pull/1935) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add db benchmark script [\#1928](https://github.com/apache/arrow-datafusion/pull/1928) ([matthewmturner](https://github.com/matthewmturner)) +- fix a typo [\#1919](https://github.com/apache/arrow-datafusion/pull/1919) ([vchag](https://github.com/vchag)) +- \[MINOR\] Update copyright year in Docs [\#1918](https://github.com/apache/arrow-datafusion/pull/1918) ([alamb](https://github.com/alamb)) +- add metadata to DFSchema, close \#1806. [\#1914](https://github.com/apache/arrow-datafusion/pull/1914) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jiacai2050](https://github.com/jiacai2050)) +- Refactor scheduler state mod [\#1913](https://github.com/apache/arrow-datafusion/pull/1913) ([yahoNanJing](https://github.com/yahoNanJing)) +- Refactor the event channel [\#1912](https://github.com/apache/arrow-datafusion/pull/1912) ([yahoNanJing](https://github.com/yahoNanJing)) +- Refactor scheduler server [\#1911](https://github.com/apache/arrow-datafusion/pull/1911) ([yahoNanJing](https://github.com/yahoNanJing)) +- Clippy fix on nightly [\#1907](https://github.com/apache/arrow-datafusion/pull/1907) ([yjshen](https://github.com/yjshen)) +- Updated Rust version to 1.59 in all the files [\#1903](https://github.com/apache/arrow-datafusion/pull/1903) ([NaincyKumariKnoldus](https://github.com/NaincyKumariKnoldus)) +- Remove uneeded Mutex in Ballista Client [\#1898](https://github.com/apache/arrow-datafusion/pull/1898) ([alamb](https://github.com/alamb)) +- Create a `datafusion-proto` crate for datafusion protobuf serialization [\#1887](https://github.com/apache/arrow-datafusion/pull/1887) ([carols10cents](https://github.com/carols10cents)) +- Fix clippy lints [\#1885](https://github.com/apache/arrow-datafusion/pull/1885) ([HaoYang670](https://github.com/HaoYang670)) +- Separate cpu-bound \(query-execution\) and IO-bound\(heartbeat\) to … [\#1883](https://github.com/apache/arrow-datafusion/pull/1883) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- \[Minor\] Clean up DecimalArray API Usage [\#1869](https://github.com/apache/arrow-datafusion/pull/1869) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb)) +- Changes after went through "Datafusion as a library section" [\#1868](https://github.com/apache/arrow-datafusion/pull/1868) ([nonontb](https://github.com/nonontb)) +- Remove allow unused imports from ballista-core, then fix all warnings [\#1853](https://github.com/apache/arrow-datafusion/pull/1853) ([carols10cents](https://github.com/carols10cents)) +- Update to arrow 9.1.0 [\#1851](https://github.com/apache/arrow-datafusion/pull/1851) ([alamb](https://github.com/alamb)) +- move some tests out of context and into sql [\#1846](https://github.com/apache/arrow-datafusion/pull/1846) ([alamb](https://github.com/alamb)) +- Fix compiling ballista in standalone mode, add build to CI [\#1839](https://github.com/apache/arrow-datafusion/pull/1839) ([alamb](https://github.com/alamb)) +- Update documentation example for change in API [\#1812](https://github.com/apache/arrow-datafusion/pull/1812) ([alamb](https://github.com/alamb)) +- Refactor scheduler state with different management policy for volatile and stable states [\#1810](https://github.com/apache/arrow-datafusion/pull/1810) ([yahoNanJing](https://github.com/yahoNanJing)) +- DataFusion + Conbench Integration [\#1791](https://github.com/apache/arrow-datafusion/pull/1791) ([dianaclarke](https://github.com/dianaclarke)) +- Enable periodic cleanup of work_dir directories in ballista executor [\#1783](https://github.com/apache/arrow-datafusion/pull/1783) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Use`eq_dyn`, `neq_dyn`, `lt_dyn`, `lt_eq_dyn`, `gt_dyn`, `gt_eq_dyn` kernels from arrow [\#1475](https://github.com/apache/arrow-datafusion/pull/1475) ([alamb](https://github.com/alamb)) diff --git a/docs/source/changelog/0.8.0.md b/docs/source/changelog/0.8.0.md new file mode 100644 index 0000000000..b8e5b73828 --- /dev/null +++ b/docs/source/changelog/0.8.0.md @@ -0,0 +1,64 @@ + + +# Apache DataFusion Ballista 0.8.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.7.0...0.8.0) + +**Implemented enhancements:** + +- Executor should use all available cores by default [\#218](https://github.com/apache/datafusion-ballista/issues/218) +- Update task status to the task job curator scheduler [\#179](https://github.com/apache/datafusion-ballista/issues/179) +- update datafusion and arrow to 20.0.0 [\#176](https://github.com/apache/datafusion-ballista/issues/176) +- No scheduler logs when deployed to k8s [\#165](https://github.com/apache/datafusion-ballista/issues/165) +- Upgrade to DataFusion 11.0.0 [\#163](https://github.com/apache/datafusion-ballista/issues/163) +- Better encapsulation for ExecutionGraph [\#149](https://github.com/apache/datafusion-ballista/issues/149) +- A stage may act as the input of multiple stages [\#144](https://github.com/apache/datafusion-ballista/issues/144) +- Executor Lost handling [\#143](https://github.com/apache/datafusion-ballista/issues/143) +- Cancel a running query. [\#139](https://github.com/apache/datafusion-ballista/issues/139) +- Ignore the previous job_id inside fill_reservations\(\) [\#138](https://github.com/apache/datafusion-ballista/issues/138) +- Normalize the serialization and deserialization places of protobuf structs [\#137](https://github.com/apache/datafusion-ballista/issues/137) +- Remove revive offer event loop [\#136](https://github.com/apache/datafusion-ballista/issues/136) +- Remove Keyspace::QueuedJobs [\#133](https://github.com/apache/datafusion-ballista/issues/133) +- Spawn a thread for execution plan generation [\#131](https://github.com/apache/datafusion-ballista/issues/131) +- Introduce CuratorTaskManager for make an active job be curated by only one scheduler [\#130](https://github.com/apache/datafusion-ballista/issues/130) +- Using tokio tracing for log file [\#122](https://github.com/apache/datafusion-ballista/issues/122) +- Ballista Executor report plan/operators metrics to Ballista Scheduler when task finish [\#116](https://github.com/apache/datafusion-ballista/issues/116) +- Add timeout settings for Grpc Client [\#114](https://github.com/apache/datafusion-ballista/issues/114) +- Add log level config in ballista [\#102](https://github.com/apache/datafusion-ballista/issues/102) +- Use another channel to update the status of a task set for executor [\#96](https://github.com/apache/datafusion-ballista/issues/96) +- Add config for concurrent_task in executor [\#94](https://github.com/apache/datafusion-ballista/issues/94) +- Ballista should support Arrow FlightSQL [\#92](https://github.com/apache/datafusion-ballista/issues/92) +- Why not include the `ballista-cli` in the member of workspace [\#88](https://github.com/apache/datafusion-ballista/issues/88) +- Upgrade dependency of arrow-datafusion to commit d0d5564b8f689a01e542b8c1df829d74d0fab2b0 [\#84](https://github.com/apache/datafusion-ballista/issues/84) +- Support sled path in config file. [\#79](https://github.com/apache/datafusion-ballista/issues/79) +- Support for multi-scheduler deployments [\#39](https://github.com/apache/datafusion-ballista/issues/39) +- Ballista 0.7.0 Release [\#126](https://github.com/apache/datafusion-ballista/issues/126) +- Improvements to Ballista extensibility [\#8](https://github.com/apache/datafusion-ballista/issues/8) +- Implement Python bindings for BallistaContext [\#15](https://github.com/apache/datafusion-ballista/issues/15) + +**Fixed bugs:** + +- Run example fails via PushStaged mode [\#214](https://github.com/apache/datafusion-ballista/issues/214) +- Config settings in BallistaContext do not get passed to DataFusion context [\#213](https://github.com/apache/datafusion-ballista/issues/213) +- Start scheduler fails with arguments "-s PushStaged" [\#207](https://github.com/apache/datafusion-ballista/issues/207) +- FlightSQL is broken and CI isn't catching it [\#190](https://github.com/apache/datafusion-ballista/issues/190) +- Query fails with "NULL is invalid as a DataFusion scalar value" [\#180](https://github.com/apache/datafusion-ballista/issues/180) +- Executor doesn't compile, missing `tokio::signal` [\#171](https://github.com/apache/datafusion-ballista/issues/171) +- Unable to build master [\#76](https://github.com/apache/datafusion-ballista/issues/76) diff --git a/docs/source/changelog/0.9.0.md b/docs/source/changelog/0.9.0.md new file mode 100644 index 0000000000..7f3ba4832a --- /dev/null +++ b/docs/source/changelog/0.9.0.md @@ -0,0 +1,184 @@ + + +# Apache DataFusion Ballista 0.9.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.8.0...0.9.0) + +**Implemented enhancements:** + +- Support count distinct aggregation function [\#411](https://github.com/apache/datafusion-ballista/issues/411) +- Use multi-task definition in pull-based execution loop [\#400](https://github.com/apache/datafusion-ballista/issues/400) +- Make the scheduler event loop buffer size configurable [\#397](https://github.com/apache/datafusion-ballista/issues/397) +- Remove active execution graph when the related job is successful or failed. [\#391](https://github.com/apache/datafusion-ballista/issues/391) +- Improve launch task efficiency by calling LaunchMultiTask [\#389](https://github.com/apache/datafusion-ballista/issues/389) +- Use `tokio::sync::Semaphore` to wait for available task slots [\#388](https://github.com/apache/datafusion-ballista/issues/388) +- stdout and file log level settings are inconsistent [\#385](https://github.com/apache/datafusion-ballista/issues/385) +- Use dedicated executor in pull based loop [\#383](https://github.com/apache/datafusion-ballista/issues/383) +- Avoid calling scheduler when the executor cannot accept new tasks [\#377](https://github.com/apache/datafusion-ballista/issues/377) +- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#371](https://github.com/apache/datafusion-ballista/issues/371) +- Switch to mimalloc and enable by default [\#369](https://github.com/apache/datafusion-ballista/issues/369) +- Integration test script should use docker-compose [\#364](https://github.com/apache/datafusion-ballista/issues/364) +- Use local shuffle reader in containerized environments [\#356](https://github.com/apache/datafusion-ballista/issues/356) +- Add `--ext` option to benchmark [\#352](https://github.com/apache/datafusion-ballista/issues/352) +- Add job cancel in the UI [\#350](https://github.com/apache/datafusion-ballista/issues/350) +- Using local shuffle reader avoid flight rpc call. [\#346](https://github.com/apache/datafusion-ballista/issues/346) +- Add a Helm Chart [\#321](https://github.com/apache/datafusion-ballista/issues/321) +- \[UI\] Show list of query stages with metrics [\#306](https://github.com/apache/datafusion-ballista/issues/306) +- \[UI\] Add ability to specify job name and have it show in the job listing page in the UI [\#277](https://github.com/apache/datafusion-ballista/issues/277) +- \[UI\] Add ability to download query plans in dot format [\#276](https://github.com/apache/datafusion-ballista/issues/276) +- \[UI\] Add ability to render query plans [\#275](https://github.com/apache/datafusion-ballista/issues/275) +- Add REST API documentation to User Guide [\#272](https://github.com/apache/datafusion-ballista/issues/272) +- Graceful shutdown: Handle `SIGTERM` [\#266](https://github.com/apache/datafusion-ballista/issues/266) +- \[EPIC\] Scheduler UI [\#265](https://github.com/apache/datafusion-ballista/issues/265) +- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#259](https://github.com/apache/datafusion-ballista/issues/259) +- Add a feature based object store provider [\#257](https://github.com/apache/datafusion-ballista/issues/257) +- Add docker build files [\#248](https://github.com/apache/datafusion-ballista/issues/248) +- Allow IDEs to recognize generated code [\#246](https://github.com/apache/datafusion-ballista/issues/246) +- Add user guide section on Flight SQL support [\#230](https://github.com/apache/datafusion-ballista/issues/230) +- `dev/release/README.md` is outdated [\#228](https://github.com/apache/datafusion-ballista/issues/228) +- Make ShuffleReaderExec output less verbose [\#211](https://github.com/apache/datafusion-ballista/issues/211) +- Add LaunchMultiTask rpc interface for executor [\#209](https://github.com/apache/datafusion-ballista/issues/209) +- Make executor fetch shuffle partition data in parallel [\#208](https://github.com/apache/datafusion-ballista/issues/208) +- Concurrency control and rate limit during shuffle reader [\#195](https://github.com/apache/datafusion-ballista/issues/195) +- Update User Guide [\#160](https://github.com/apache/datafusion-ballista/issues/160) +- Ballista 0.8.0 Release [\#159](https://github.com/apache/datafusion-ballista/issues/159) +- Save encoded execution plan in the ExecutionStage to reduce cost of task serialization and deserialization [\#142](https://github.com/apache/datafusion-ballista/issues/142) +- Failed task retry [\#140](https://github.com/apache/datafusion-ballista/issues/140) +- Redefine the executor task slots [\#132](https://github.com/apache/datafusion-ballista/issues/132) +- Use ArrowFlight bearer token auth to create a session key for FlightSql clients [\#112](https://github.com/apache/datafusion-ballista/issues/112) +- Leverage Atomic for the in-memory states in Scheduler [\#101](https://github.com/apache/datafusion-ballista/issues/101) +- Introduce the object stores in datafusion-contrib as optional features [\#87](https://github.com/apache/datafusion-ballista/issues/87) +- Support multiple paths for ListingTableScanNode [\#75](https://github.com/apache/datafusion-ballista/issues/75) +- Need clean up intermediate data in Ballista [\#9](https://github.com/apache/datafusion-ballista/issues/9) +- Ballista does not support external file systems [\#10](https://github.com/apache/datafusion-ballista/issues/10) + +**Fixed bugs:** + +- Build errors in ./dev/build-ballista-rust.sh [\#407](https://github.com/apache/datafusion-ballista/issues/407) +- The Ballista Scheduler Dockerfile copies a file that no longer exists [\#402](https://github.com/apache/datafusion-ballista/issues/402) +- Benchmark q20 fails [\#374](https://github.com/apache/datafusion-ballista/issues/374) +- Integration tests fail [\#360](https://github.com/apache/datafusion-ballista/issues/360) +- Helm deploy fails [\#344](https://github.com/apache/datafusion-ballista/issues/344) +- Executor get stopped unexpected [\#333](https://github.com/apache/datafusion-ballista/issues/333) +- Executor poll work loop failure [\#311](https://github.com/apache/datafusion-ballista/issues/311) +- Queries with `LIMIT` are failing with "PhysicalExtensionCodec is not provided" [\#300](https://github.com/apache/datafusion-ballista/issues/300) +- Schema inference does not work in Ballista-cli with a remote context [\#287](https://github.com/apache/datafusion-ballista/issues/287) +- There are bugs in the yarn build github misses but break our internal build [\#270](https://github.com/apache/datafusion-ballista/issues/270) +- Race condition running docker-compose [\#267](https://github.com/apache/datafusion-ballista/issues/267) +- Scheduler UI not working in Docker image [\#250](https://github.com/apache/datafusion-ballista/issues/250) +- Use bind host rather than the external host for starting a local executor service [\#244](https://github.com/apache/datafusion-ballista/issues/244) +- Initial query stages read parquet files and repartition them needlessly [\#243](https://github.com/apache/datafusion-ballista/issues/243) +- Cannot build Docker images on macOS 12.5.1 with M1 chip [\#234](https://github.com/apache/datafusion-ballista/issues/234) +- CLI uses DataFusion context if no host or port are provided [\#219](https://github.com/apache/datafusion-ballista/issues/219) +- Unsupported binary operator `StringConcat` [\#201](https://github.com/apache/datafusion-ballista/issues/201) +- Ballista assumes all aggregate expressions are not DISTINCT [\#5](https://github.com/apache/datafusion-ballista/issues/5) +- Start ballista ui with docker, but it can not found ballista scheduler [\#11](https://github.com/apache/datafusion-ballista/issues/11) +- Cannot build Ballista docker images on Apple silicon [\#17](https://github.com/apache/datafusion-ballista/issues/17) + +**Documentation updates:** + +- Fixup links in README.md [\#366](https://github.com/apache/datafusion-ballista/pull/366) ([romanz](https://github.com/romanz)) +- Update README in preparation for 0.9.0 release [\#318](https://github.com/apache/datafusion-ballista/pull/318) ([andygrove](https://github.com/andygrove)) +- User Guide improvements [\#274](https://github.com/apache/datafusion-ballista/pull/274) ([andygrove](https://github.com/andygrove)) + +**Closed issues:** + +- Automatic version updates for github actions with dependabot [\#127](https://github.com/apache/datafusion-ballista/issues/127) + +**Merged pull requests:** + +- Return multiple tasks in poll_work based on free slots [\#429](https://github.com/apache/datafusion-ballista/pull/429) ([Dandandan](https://github.com/Dandandan)) +- Run integration tests as part of release verification script [\#426](https://github.com/apache/datafusion-ballista/pull/426) ([andygrove](https://github.com/andygrove)) +- Bump actions/setup-node from 2 to 3 [\#424](https://github.com/apache/datafusion-ballista/pull/424) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/setup-python from 2 to 4 [\#423](https://github.com/apache/datafusion-ballista/pull/423) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/checkout from 2 to 3 [\#422](https://github.com/apache/datafusion-ballista/pull/422) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/download-artifact from 2 to 3 [\#421](https://github.com/apache/datafusion-ballista/pull/421) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/upload-artifact from 2 to 3 [\#420](https://github.com/apache/datafusion-ballista/pull/420) ([dependabot[bot]](https://github.com/apps/dependabot)) +- MINOR: Fix yarn warnings [\#415](https://github.com/apache/datafusion-ballista/pull/415) ([andygrove](https://github.com/andygrove)) +- Fix q20 sql typo in benchmarks [\#409](https://github.com/apache/datafusion-ballista/pull/409) ([r4ntix](https://github.com/r4ntix)) +- MINOR: Add notes on Apache Reporter [\#401](https://github.com/apache/datafusion-ballista/pull/401) ([andygrove](https://github.com/andygrove)) +- Use local shuffle reader in containerized environments and some impro… [\#399](https://github.com/apache/datafusion-ballista/pull/399) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Make the scheduler event loop buffer size configurable [\#398](https://github.com/apache/datafusion-ballista/pull/398) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add RoundRobinLocal slots policy for caching executor data to avoid seld persistency [\#396](https://github.com/apache/datafusion-ballista/pull/396) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#395](https://github.com/apache/datafusion-ballista/pull/395) ([yahoNanJing](https://github.com/yahoNanJing)) +- Improve launch task efficiency by calling LaunchMultiTask [\#394](https://github.com/apache/datafusion-ballista/pull/394) ([yahoNanJing](https://github.com/yahoNanJing)) +- Cache encoded stage plan [\#393](https://github.com/apache/datafusion-ballista/pull/393) ([yahoNanJing](https://github.com/yahoNanJing)) +- Remove active execution graph when the related job is successful or failed [\#392](https://github.com/apache/datafusion-ballista/pull/392) ([yahoNanJing](https://github.com/yahoNanJing)) +- Update flatbuffers requirement from 2.1.2 to 22.9.29 [\#390](https://github.com/apache/datafusion-ballista/pull/390) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Unified the log level configuration behavior [\#386](https://github.com/apache/datafusion-ballista/pull/386) ([r4ntix](https://github.com/r4ntix)) +- Add DistinctCount support [\#384](https://github.com/apache/datafusion-ballista/pull/384) ([r4ntix](https://github.com/r4ntix)) +- Pull-based execution loop improvements [\#380](https://github.com/apache/datafusion-ballista/pull/380) ([Dandandan](https://github.com/Dandandan)) +- Fix latest commit [\#379](https://github.com/apache/datafusion-ballista/pull/379) ([Dandandan](https://github.com/Dandandan)) +- Avoid calling scheduler when the executor cannot accept new tasks [\#378](https://github.com/apache/datafusion-ballista/pull/378) ([Dandandan](https://github.com/Dandandan)) +- Switch to mimalloc and enable by default in executor [\#370](https://github.com/apache/datafusion-ballista/pull/370) ([Dandandan](https://github.com/Dandandan)) +- Benchmark looks for path with and without extension [\#354](https://github.com/apache/datafusion-ballista/pull/354) ([andygrove](https://github.com/andygrove)) +- Implement job cancellation in UI [\#349](https://github.com/apache/datafusion-ballista/pull/349) ([Dandandan](https://github.com/Dandandan)) +- Using local shuffle reader avoid flight rpc call. [\#347](https://github.com/apache/datafusion-ballista/pull/347) ([Ted-Jiang](https://github.com/Ted-Jiang)) +- Make helm deployable [\#345](https://github.com/apache/datafusion-ballista/pull/345) ([avantgardnerio](https://github.com/avantgardnerio)) +- Benchmark & UI improvements [\#343](https://github.com/apache/datafusion-ballista/pull/343) ([andygrove](https://github.com/andygrove)) +- Add `cancel_job` REST API [\#340](https://github.com/apache/datafusion-ballista/pull/340) ([tfeda](https://github.com/tfeda)) +- Fix labeler [\#337](https://github.com/apache/datafusion-ballista/pull/337) ([andygrove](https://github.com/andygrove)) +- Upgrade to DataFusion 13.0.0 [\#336](https://github.com/apache/datafusion-ballista/pull/336) ([andygrove](https://github.com/andygrove)) +- Check executor id consistency when receive stop executor request [\#335](https://github.com/apache/datafusion-ballista/pull/335) ([yahoNanJing](https://github.com/yahoNanJing)) +- Enable more benchmark serde tests [\#331](https://github.com/apache/datafusion-ballista/pull/331) ([andygrove](https://github.com/andygrove)) +- Downgrade `docker-compose.yaml` to version 3.3 so that we can support Ubuntu 20.04.4 LTS [\#329](https://github.com/apache/datafusion-ballista/pull/329) ([andygrove](https://github.com/andygrove)) +- update labeler [\#326](https://github.com/apache/datafusion-ballista/pull/326) ([andygrove](https://github.com/andygrove)) +- Upgrade to DataFusion 13.0.0-rc1 [\#325](https://github.com/apache/datafusion-ballista/pull/325) ([andygrove](https://github.com/andygrove)) +- Dependabot stop suggesting arrow and datafusion updates [\#324](https://github.com/apache/datafusion-ballista/pull/324) ([andygrove](https://github.com/andygrove)) +- Show job stages metrics [\#323](https://github.com/apache/datafusion-ballista/pull/323) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) +- Add helm chart [\#322](https://github.com/apache/datafusion-ballista/pull/322) ([avantgardnerio](https://github.com/avantgardnerio)) +- Atomic support for enhancement [\#319](https://github.com/apache/datafusion-ballista/pull/319) ([metesynnada](https://github.com/metesynnada)) +- Allow automatic schema inference when registering csv [\#313](https://github.com/apache/datafusion-ballista/pull/313) ([r4ntix](https://github.com/r4ntix)) +- Add ability to specify job name and have it show in the job listing page in the UI [\#312](https://github.com/apache/datafusion-ballista/pull/312) ([andygrove](https://github.com/andygrove)) +- Add REST API to generate DOT graph for individual query stage [\#310](https://github.com/apache/datafusion-ballista/pull/310) ([andygrove](https://github.com/andygrove)) +- \[UI\] Use tabbed pane with Queries and Executors tabs [\#309](https://github.com/apache/datafusion-ballista/pull/309) ([andygrove](https://github.com/andygrove)) +- REST API to get query stages [\#305](https://github.com/apache/datafusion-ballista/pull/305) ([andygrove](https://github.com/andygrove)) +- Add support for SortPreservingMergeExec; fix LIMIT bug [\#304](https://github.com/apache/datafusion-ballista/pull/304) ([andygrove](https://github.com/andygrove)) +- Add Python script to run benchmarks [\#302](https://github.com/apache/datafusion-ballista/pull/302) ([andygrove](https://github.com/andygrove)) +- \[UI\] Add ability to view query plans directly in the UI [\#301](https://github.com/apache/datafusion-ballista/pull/301) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) +- Update datafusion.proto [\#299](https://github.com/apache/datafusion-ballista/pull/299) ([andygrove](https://github.com/andygrove)) +- Replace function `from_proto_binary_op` from upstream [\#298](https://github.com/apache/datafusion-ballista/pull/298) ([askoa](https://github.com/askoa)) +- Fix dead link in contribution guideline readme file [\#297](https://github.com/apache/datafusion-ballista/pull/297) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) +- UI code cleanup [\#291](https://github.com/apache/datafusion-ballista/pull/291) ([KenSuenobu](https://github.com/KenSuenobu)) +- Add support for S3 data sources [\#290](https://github.com/apache/datafusion-ballista/pull/290) ([andygrove](https://github.com/andygrove)) +- Use latest datafusion [\#289](https://github.com/apache/datafusion-ballista/pull/289) ([andygrove](https://github.com/andygrove)) +- Fix documentation example [\#288](https://github.com/apache/datafusion-ballista/pull/288) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere)) +- Improve formatting of job status in UI [\#286](https://github.com/apache/datafusion-ballista/pull/286) ([andygrove](https://github.com/andygrove)) +- Enabled download of dot files from Download icon [\#279](https://github.com/apache/datafusion-ballista/pull/279) ([KenSuenobu](https://github.com/KenSuenobu)) +- Executor graceful shutdown: Handle SIGTERM [\#278](https://github.com/apache/datafusion-ballista/pull/278) ([mingmwang](https://github.com/mingmwang)) +- Also run yarn build to catch JavaScript errors in CI [\#271](https://github.com/apache/datafusion-ballista/pull/271) ([avantgardnerio](https://github.com/avantgardnerio)) +- Store sessions so users can register tables and query them through flight [\#269](https://github.com/apache/datafusion-ballista/pull/269) ([avantgardnerio](https://github.com/avantgardnerio)) +- Fix compose for Ian [\#268](https://github.com/apache/datafusion-ballista/pull/268) ([avantgardnerio](https://github.com/avantgardnerio)) +- Task level retry and Stage level retry [\#261](https://github.com/apache/datafusion-ballista/pull/261) ([mingmwang](https://github.com/mingmwang)) +- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#260](https://github.com/apache/datafusion-ballista/pull/260) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add a feature based object store provider [\#258](https://github.com/apache/datafusion-ballista/pull/258) ([yahoNanJing](https://github.com/yahoNanJing)) +- Make fetch shuffle partition data in parallel [\#256](https://github.com/apache/datafusion-ballista/pull/256) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add LaunchMultiTask rpc interface for executor [\#255](https://github.com/apache/datafusion-ballista/pull/255) ([yahoNanJing](https://github.com/yahoNanJing)) +- CLI uses ballista context instead of datafusion context in local mode [\#252](https://github.com/apache/datafusion-ballista/pull/252) ([r4ntix](https://github.com/r4ntix)) +- Fix Scheduler UI in Docker image [\#251](https://github.com/apache/datafusion-ballista/pull/251) ([andygrove](https://github.com/andygrove)) +- Generate into source folder to make IDEs happy [\#247](https://github.com/apache/datafusion-ballista/pull/247) ([avantgardnerio](https://github.com/avantgardnerio)) +- Use bind host rather than the external host for starting a local executor service [\#245](https://github.com/apache/datafusion-ballista/pull/245) ([yahoNanJing](https://github.com/yahoNanJing)) +- Add REST endpoint to get DOT graph of a job [\#242](https://github.com/apache/datafusion-ballista/pull/242) ([andygrove](https://github.com/andygrove)) +- Add list of jobs to scheduler UI [\#241](https://github.com/apache/datafusion-ballista/pull/241) ([andygrove](https://github.com/andygrove)) +- Clean up job data on both Scheduler and Executor [\#188](https://github.com/apache/datafusion-ballista/pull/188) ([mingmwang](https://github.com/mingmwang)) +- Update etcd-client requirement from 0.9 to 0.10 [\#111](https://github.com/apache/datafusion-ballista/pull/111) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump terser from 4.8.0 to 4.8.1 in /ballista/ui/scheduler [\#91](https://github.com/apache/datafusion-ballista/pull/91) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump jsdom from 16.4.0 to 16.7.0 in /ballista/ui/scheduler [\#74](https://github.com/apache/datafusion-ballista/pull/74) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump numpy from 1.21.3 to 1.22.0 in /python [\#72](https://github.com/apache/datafusion-ballista/pull/72) ([dependabot[bot]](https://github.com/apps/dependabot)) diff --git a/docs/source/changelog/43.0.0.md b/docs/source/changelog/43.0.0.md new file mode 100644 index 0000000000..a206322d3a --- /dev/null +++ b/docs/source/changelog/43.0.0.md @@ -0,0 +1,146 @@ + + +# Apache DataFusion Ballista 43.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.12.0...43.0.0-rc2) + +**Implemented enhancements:** + +- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov) +- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove) +- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove) +- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm) +- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm) +- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm) + +**Documentation updates:** + +- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd) +- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove) +- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins) +- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov) +- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins) +- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski) +- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove) +- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove) +- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove) +- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove) +- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm) +- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm) +- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm) +- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4) +- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm) +- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove) + +**Merged pull requests:** + +- PyBallista - Python SQL client for Ballista [#970](https://github.com/apache/datafusion-ballista/pull/970) (andygrove) +- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd) +- [Python] Add `read_csv` and `read_parquet` methods [#976](https://github.com/apache/datafusion-ballista/pull/976) (andygrove) +- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove) +- [Python] Add more methods to SessionContext [#978](https://github.com/apache/datafusion-ballista/pull/978) (andygrove) +- [Python] Add `execute_logical_plan` to context [#972](https://github.com/apache/datafusion-ballista/pull/972) (andygrove) +- Use correct product name in docs [#975](https://github.com/apache/datafusion-ballista/pull/975) (andygrove) +- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins) +- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov) +- Upgrade Rust version to 1.72 to keep the same as DataFusion v35 [#982](https://github.com/apache/datafusion-ballista/pull/982) (haoxins) +- build: Fix the ballista-cli Dockerfile [#981](https://github.com/apache/datafusion-ballista/pull/981) (haoxins) +- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov) +- Remove some hard-coded gRPC max message sizes [#984](https://github.com/apache/datafusion-ballista/pull/984) (andygrove) +- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins) +- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski) +- Fix ExecutorLost event debug info [#988](https://github.com/apache/datafusion-ballista/pull/988) (lewiszlw) +- Fix shuffle writer test [#998](https://github.com/apache/datafusion-ballista/pull/998) (Jefffrey) +- Bump graphviz-rust from 0.6.1 to 0.8.0 [#999](https://github.com/apache/datafusion-ballista/pull/999) (Jefffrey) +- Add rust-toolchain.toml for clarity [#1014](https://github.com/apache/datafusion-ballista/pull/1014) (scnerd) +- Fix executor metadata decode bug [#1004](https://github.com/apache/datafusion-ballista/pull/1004) (lewiszlw) +- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove) +- Fix Ballista rust.yml github workflow [#1026](https://github.com/apache/datafusion-ballista/pull/1026) (RaphaelMarinier) +- Bump datafusion to 36.0.0 and make ballista compatible with it. [#1027](https://github.com/apache/datafusion-ballista/pull/1027) (RaphaelMarinier) +- Make Ballista compatible with Datafusion 37.0.0 (from 36.0.0) [#1031](https://github.com/apache/datafusion-ballista/pull/1031) (RaphaelMarinier) +- Fixes Setting Job Name Not Reflected in Ballista UI [#1039](https://github.com/apache/datafusion-ballista/pull/1039) (athultr1997) +- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove) +- [Docs] fix good_first_issue link in the contribution md doc [#1022](https://github.com/apache/datafusion-ballista/pull/1022) (Almaz-KG) +- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove) +- Fix job hangs when partition count of plan is zero [#1024](https://github.com/apache/datafusion-ballista/pull/1024) (lewiszlw) +- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove) +- Fix cargo build [#1045](https://github.com/apache/datafusion-ballista/pull/1045) (andygrove) +- fix docker build in CI [#1046](https://github.com/apache/datafusion-ballista/pull/1046) (andygrove) +- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove) +- Bump actions/setup-node from 3 to 4 [#909](https://github.com/apache/datafusion-ballista/pull/909) (dependabot[bot]) +- Bump actions/cache from 3 to 4 [#958](https://github.com/apache/datafusion-ballista/pull/958) (dependabot[bot]) +- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove) +- Update datafusion protobuf definitions [#1057](https://github.com/apache/datafusion-ballista/pull/1057) (palaska) +- Fix regression with TPC-H benchmark [#1060](https://github.com/apache/datafusion-ballista/pull/1060) (andygrove) +- Upgrade to Datafusion 41 [#1062](https://github.com/apache/datafusion-ballista/pull/1062) (palaska) +- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm) +- Remove plugin subsystem [#1070](https://github.com/apache/datafusion-ballista/pull/1070) (milenkovicm) +- Remove CI folder [#1074](https://github.com/apache/datafusion-ballista/pull/1074) (milenkovicm) +- Code cleanup, move examples, remove unused files [#1075](https://github.com/apache/datafusion-ballista/pull/1075) (milenkovicm) +- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm) +- Remove key-value stores for scheduler persistence [#1077](https://github.com/apache/datafusion-ballista/pull/1077) (milenkovicm) +- Remove cache functionality [#1076](https://github.com/apache/datafusion-ballista/pull/1076) (milenkovicm) +- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm) +- Reorganise and remove dependencies [#1078](https://github.com/apache/datafusion-ballista/pull/1078) (milenkovicm) +- Promote keda and flight-sql to optional features [#1079](https://github.com/apache/datafusion-ballista/pull/1079) (milenkovicm) +- Update to datafusion 42 ... [#1080](https://github.com/apache/datafusion-ballista/pull/1080) (milenkovicm) +- #1086 solve examples errors [#1087](https://github.com/apache/datafusion-ballista/pull/1087) (tbar4) +- fix issue with not building python package ... [#1085](https://github.com/apache/datafusion-ballista/pull/1085) (milenkovicm) +- another round of code cleanup ... [#1089](https://github.com/apache/datafusion-ballista/pull/1089) (milenkovicm) +- Make rest-api optional feature ... [#1084](https://github.com/apache/datafusion-ballista/pull/1084) (milenkovicm) +- fix clippy issues after updating to rust 1.82 [#1090](https://github.com/apache/datafusion-ballista/pull/1090) (milenkovicm) +- Replace BallistaContext with SessionContext [#1088](https://github.com/apache/datafusion-ballista/pull/1088) (milenkovicm) +- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4) +- Ergonomic way to setup/configure `SessionContextExt` [#1096](https://github.com/apache/datafusion-ballista/pull/1096) (milenkovicm) +- Executor configuration extended .. [#1099](https://github.com/apache/datafusion-ballista/pull/1099) (milenkovicm) +- fix issue with executor registration ... [#1101](https://github.com/apache/datafusion-ballista/pull/1101) (milenkovicm) +- Deprecate `BallistaContext` [#1103](https://github.com/apache/datafusion-ballista/pull/1103) (milenkovicm) +- fix imports after un-rebased PR [#1106](https://github.com/apache/datafusion-ballista/pull/1106) (milenkovicm) +- Ballista proto cleanup [#1110](https://github.com/apache/datafusion-ballista/pull/1110) (milenkovicm) +- Update and move deps to workspace [#1109](https://github.com/apache/datafusion-ballista/pull/1109) (milenkovicm) +- Trim down `BallistaConfig` [#1108](https://github.com/apache/datafusion-ballista/pull/1108) (milenkovicm) +- Remove build-in object store registry [#1114](https://github.com/apache/datafusion-ballista/pull/1114) (milenkovicm) +- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm) +- support window functions [#1112](https://github.com/apache/datafusion-ballista/pull/1112) (onursatici) +- added a BallistaContext to ballista to allow for Remote or standalone [#1100](https://github.com/apache/datafusion-ballista/pull/1100) (tbar4) +- Decommission `BallistaContext` [#1119](https://github.com/apache/datafusion-ballista/pull/1119) (milenkovicm) +- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove) +- Make easier to create custom schedulers and executors [#1118](https://github.com/apache/datafusion-ballista/pull/1118) (milenkovicm) +- refactor: Move BallistaRegistry to better location [#1126](https://github.com/apache/datafusion-ballista/pull/1126) (milenkovicm) +- refactor: BallistaLogicalExtensionCodec refactoring and improvements [#1127](https://github.com/apache/datafusion-ballista/pull/1127) (milenkovicm) +- refactor: consolidate ballista tests [#1129](https://github.com/apache/datafusion-ballista/pull/1129) (milenkovicm) +- refactor: SessionStateExt and SessionConfigExt [#1130](https://github.com/apache/datafusion-ballista/pull/1130) (milenkovicm) +- chore: dependancy updates [#1131](https://github.com/apache/datafusion-ballista/pull/1131) (milenkovicm) +- chore: fix warning mimaloc warning when building [#1137](https://github.com/apache/datafusion-ballista/pull/1137) (milenkovicm) +- refactor: SessionBuilder to return Result<\_> [#1138](https://github.com/apache/datafusion-ballista/pull/1138) (milenkovicm) +- chore: remove unused cache\_ options from executor [#1140](https://github.com/apache/datafusion-ballista/pull/1140) (milenkovicm) +- updated maturin version and ccargo build to build yml [#1136](https://github.com/apache/datafusion-ballista/pull/1136) (tbar4) +- chore: Fix clippy issues after rust update (1.83.0) [#1143](https://github.com/apache/datafusion-ballista/pull/1143) (milenkovicm) +- Fix documentation example which still uses BallistaContext [#1145](https://github.com/apache/datafusion-ballista/pull/1145) (milenkovicm) +- Ballista proto cleanup [#1146](https://github.com/apache/datafusion-ballista/pull/1146) (milenkovicm) +- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm) +- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm) +- chore: dependency cleanup [#1150](https://github.com/apache/datafusion-ballista/pull/1150) (milenkovicm) +- Update DataFusion to 43 [#1125](https://github.com/apache/datafusion-ballista/pull/1125) (Dandandan) +- Reinstantiate join order optimization [#1122](https://github.com/apache/datafusion-ballista/pull/1122) (Dandandan) +- add partitioning scheme for unresolved shuffle and shuffle reader exec [#1144](https://github.com/apache/datafusion-ballista/pull/1144) (onursatici) +- chore: update py-df to 43.1 [#1152](https://github.com/apache/datafusion-ballista/pull/1152) (milenkovicm) +- chore: no need to run python test in rust [#1154](https://github.com/apache/datafusion-ballista/pull/1154) (milenkovicm) +- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm) diff --git a/docs/source/changelog/44.0.0.md b/docs/source/changelog/44.0.0.md new file mode 100644 index 0000000000..409717d6ce --- /dev/null +++ b/docs/source/changelog/44.0.0.md @@ -0,0 +1,56 @@ + + +# Apache DataFusion Ballista 44.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/43.0.0...44.0.0) + +**Implemented enhancements:** + +- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm) +- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm) +- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm) +- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm) + +**Fixed bugs:** + +- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm) +- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm) + +**Documentation updates:** + +- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm) + +**Merged pull requests:** + +- chore: planner cleanup and refactor [#1160](https://github.com/apache/datafusion-ballista/pull/1160) (milenkovicm) +- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm) +- chore: fix executor build issue on release [#1167](https://github.com/apache/datafusion-ballista/pull/1167) (milenkovicm) +- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm) +- chore: update to DF.44 [#1153](https://github.com/apache/datafusion-ballista/pull/1153) (milenkovicm) +- chore: publicly expose datafusion in ballista client [#1169](https://github.com/apache/datafusion-ballista/pull/1169) (milenkovicm) +- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm) +- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm) +- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm) +- chore: generate change log for 44.0.0 [#1173](https://github.com/apache/datafusion-ballista/pull/1173) (milenkovicm) +- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm) +- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm) +- chore: fix tpch data generator [#1186](https://github.com/apache/datafusion-ballista/pull/1186) (milenkovicm) +- chore: fix clippy after rust 1.85 update [#1188](https://github.com/apache/datafusion-ballista/pull/1188) (milenkovicm) +- chore: commit `Cargo.lock` file to make builds more predictable [#1190](https://github.com/apache/datafusion-ballista/pull/1190) (milenkovicm) diff --git a/docs/source/changelog/45.0.0.md b/docs/source/changelog/45.0.0.md new file mode 100644 index 0000000000..241144aa1a --- /dev/null +++ b/docs/source/changelog/45.0.0.md @@ -0,0 +1,45 @@ + + +# Apache DataFusion Ballista 45.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/44.0.0...45.0.0) + +**Implemented enhancements:** + +- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm) +- feat: publish docker containers for executor and scheduler [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm) + +**Merged pull requests:** + +- chore: minor release script fix [#1192](https://github.com/apache/datafusion-ballista/pull/1192) (andygrove) +- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm) +- chore: update datafusion to v45 [#1176](https://github.com/apache/datafusion-ballista/pull/1176) (milenkovicm) +- chore: Update changelog for 44.0.0 [#1191](https://github.com/apache/datafusion-ballista/pull/1191) (andygrove) +- minor: fix repo and homepage url in `cargo.toml` [#1196](https://github.com/apache/datafusion-ballista/pull/1196) (milenkovicm) +- chore(deps): bump ring from 0.17.11 to 0.17.13 [#1199](https://github.com/apache/datafusion-ballista/pull/1199) (dependabot[bot]) +- feat: publish docker containers for executor and scheduler [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm) +- minor: make `graphviz-rust` dependency optional [#1203](https://github.com/apache/datafusion-ballista/pull/1203) (milenkovicm) +- doc: update docker related documentation [#1204](https://github.com/apache/datafusion-ballista/pull/1204) (milenkovicm) +- chore: update python dependencies [#1197](https://github.com/apache/datafusion-ballista/pull/1197) (milenkovicm) +- chore(deps): bump ring from 0.17.8 to 0.17.14 in /python [#1206](https://github.com/apache/datafusion-ballista/pull/1206) (dependabot[bot]) +- doc: remove arrow from doc title [#1207](https://github.com/apache/datafusion-ballista/pull/1207) (milenkovicm) +- Fix unit tests in tpch.rs [#1195](https://github.com/apache/datafusion-ballista/pull/1195) (vmingchen) +- documentation :: quick-start.md sample source code correction [#1213](https://github.com/apache/datafusion-ballista/pull/1213) (nj7) +- doc: fix quick-start executor command [#1217](https://github.com/apache/datafusion-ballista/pull/1217) (westhide) diff --git a/docs/source/changelog/46.0.0.md b/docs/source/changelog/46.0.0.md new file mode 100644 index 0000000000..f15227474d --- /dev/null +++ b/docs/source/changelog/46.0.0.md @@ -0,0 +1,58 @@ + + +# Apache DataFusion Ballista 46.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/45.0.0...46.0.0) + +**Implemented enhancements:** + +- feat: make task distribution policies pluggable [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm) +- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm) +- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm) +- feat: make execution_graph.stages() public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm) + +**Fixed bugs:** + +- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii) + +**Merged pull requests:** + +- chore: Remove some arrow references [#1232](https://github.com/apache/datafusion-ballista/pull/1232) (andygrove) +- chore: remove unused executor configuration option [#1229](https://github.com/apache/datafusion-ballista/pull/1229) (milenkovicm) +- chore: return `404` for api requests if path does not exist [#1224](https://github.com/apache/datafusion-ballista/pull/1224) (milenkovicm) +- chore(ci): replace `actions-rs` which are deprecated [#1222](https://github.com/apache/datafusion-ballista/pull/1222) (milenkovicm) +- minor: Decouple `ExecutionGraph` and `DistributedPlanner` [#1221](https://github.com/apache/datafusion-ballista/pull/1221) (milenkovicm) +- chore: update datafusion to 46 [#1201](https://github.com/apache/datafusion-ballista/pull/1201) (milenkovicm) +- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 [#1233](https://github.com/apache/datafusion-ballista/pull/1233) (dependabot[bot]) +- chore(deps): bump tokio from 1.44.1 to 1.44.2 [#1234](https://github.com/apache/datafusion-ballista/pull/1234) (dependabot[bot]) +- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii) +- chore: update python deps to 45 [#1240](https://github.com/apache/datafusion-ballista/pull/1240) (milenkovicm) +- Add S3 object store support to executor and scheduler [#1230](https://github.com/apache/datafusion-ballista/pull/1230) (milenkovicm) +- feat: make task distribution policies pluggable [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm) +- chore: reduce log levels for few log statements [#1237](https://github.com/apache/datafusion-ballista/pull/1237) (milenkovicm) +- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 in /python [#1244](https://github.com/apache/datafusion-ballista/pull/1244) (dependabot[bot]) +- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm) +- minor: `executor_shutdown_while_running` test has race condition [#1248](https://github.com/apache/datafusion-ballista/pull/1248) (milenkovicm) +- bug: build fails with `--no-default-features` [#1255](https://github.com/apache/datafusion-ballista/pull/1255) (milenkovicm) +- doc: update architectural diagram [#1253](https://github.com/apache/datafusion-ballista/pull/1253) (milenkovicm) +- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm) +- chore: add read/write roundtrip tests [#1249](https://github.com/apache/datafusion-ballista/pull/1249) (milenkovicm) +- minor: change log level for object store creation [#1247](https://github.com/apache/datafusion-ballista/pull/1247) (milenkovicm) +- feat: make execution_graph.stages() public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm) diff --git a/docs/source/changelog/47.0.0.md b/docs/source/changelog/47.0.0.md new file mode 100644 index 0000000000..413b235d79 --- /dev/null +++ b/docs/source/changelog/47.0.0.md @@ -0,0 +1,48 @@ + + +# Apache DataFusion Ballista 47.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/46.0.0...47.0.0) + +**Implemented enhancements:** + +- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm) +- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm) +- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm) +- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm) +- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm) +- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm) + +**Fixed bugs:** + +- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm) +- fix: fix tests failing on windows [#1273](https://github.com/apache/datafusion-ballista/pull/1273) (Huy1Ng) + +**Merged pull requests:** + +- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm) +- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm) +- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm) +- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm) +- minor: release docker on when release has been tagged [#1264](https://github.com/apache/datafusion-ballista/pull/1264) (milenkovicm) +- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm) +- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm) +- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm) +- chore(deps): update to datafusion 47.0.0 [#1250](https://github.com/apache/datafusion-ballista/pull/1250) (milenkovicm) diff --git a/docs/source/changelog/48.0.0.md b/docs/source/changelog/48.0.0.md new file mode 100644 index 0000000000..4e1e1ce6bc --- /dev/null +++ b/docs/source/changelog/48.0.0.md @@ -0,0 +1,44 @@ + + +# Apache DataFusion Ballista 48.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/47.0.0...48.0.0) + +**Fixed bugs:** + +- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG) +- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm) +- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm) + +**Documentation updates:** + +- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm) +- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone) + +**Merged pull requests:** + +- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm) +- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone) +- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG) +- improve rust workflows without cache [#1275](https://github.com/apache/datafusion-ballista/pull/1275) (Huy1Ng) +- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm) +- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm) +- chore: update python module to latest ballista release (v.47) [#1279](https://github.com/apache/datafusion-ballista/pull/1279) (milenkovicm) +- chore: update datafusion to 48.0.1 [#1284](https://github.com/apache/datafusion-ballista/pull/1284) (milenkovicm) diff --git a/docs/source/changelog/49.0.0.md b/docs/source/changelog/49.0.0.md new file mode 100644 index 0000000000..770f5f9f69 --- /dev/null +++ b/docs/source/changelog/49.0.0.md @@ -0,0 +1,58 @@ + + +# Apache DataFusion Ballista 49.0.0 Changelog + +[Full Changelog](https://github.com/apache/datafusion-ballista/compare/48.0.0...49.0.0) + +**Implemented enhancements:** + +- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm) +- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay) + +**Fixed bugs:** + +- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm) +- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez) +- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm) +- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm) +- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm) +- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm) + +**Merged pull requests:** + +- chore: update datafusion to 49 [#1285](https://github.com/apache/datafusion-ballista/pull/1285) (milenkovicm) +- chore: Improve GitHub actions/python workflows [#1289](https://github.com/apache/datafusion-ballista/pull/1289) (Huy1Ng) +- chore: update datafusion to 49.0.2 [#1298](https://github.com/apache/datafusion-ballista/pull/1298) (milenkovicm) +- minor: make shuffle exec display consistent [#1299](https://github.com/apache/datafusion-ballista/pull/1299) (milenkovicm) +- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm) +- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez) +- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm) +- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm) +- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm) +- chore: notice and cargo deps cleanup [#1295](https://github.com/apache/datafusion-ballista/pull/1295) (milenkovicm) +- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 [#1303](https://github.com/apache/datafusion-ballista/pull/1303) (dependabot[bot]) +- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 in /python [#1308](https://github.com/apache/datafusion-ballista/pull/1308) (dependabot[bot]) +- refactor: expand ClusterEventSender visibility to public [#1312](https://github.com/apache/datafusion-ballista/pull/1312) (Th824) +- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm) +- minor: enable json write test [#1311](https://github.com/apache/datafusion-ballista/pull/1311) (milenkovicm) +- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm) +- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay) +- feat: Improve Remote Shuffle Read Speed and Resource Utilisation [#1318](https://github.com/apache/datafusion-ballista/pull/1318) (milenkovicm) +- fix: Issue with `JoinSelection` and `CrossJoinExec` when stages have been resoled [#1322](https://github.com/apache/datafusion-ballista/pull/1322) (ZihuanLing) diff --git a/docs/source/changelog/50.0.0.md b/docs/source/changelog/50.0.0.md new file mode 100644 index 0000000000..cbae10092e --- /dev/null +++ b/docs/source/changelog/50.0.0.md @@ -0,0 +1,46 @@ + + +# Apache DataFusion Ballista 50.0.0 Changelog + +**Implemented enhancements:** + +- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan) + +**Fixed bugs:** + +- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear) +- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel) +- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm) + +**Merged pull requests:** + +- infra: macos-13 is deprecated [#1324](https://github.com/apache/datafusion-ballista/pull/1324) (kevinjqliu) +- chore: update to datafusion v50 [#1320](https://github.com/apache/datafusion-ballista/pull/1320) (milenkovicm) +- chore: Update datafusion to 50.2 [#1326](https://github.com/apache/datafusion-ballista/pull/1326) (milenkovicm) +- document private items and update docs CI [#1327](https://github.com/apache/datafusion-ballista/pull/1327) (killzoner) +- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear) +- add msrvcheck [#1328](https://github.com/apache/datafusion-ballista/pull/1328) (killzoner) +- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel) +- chore: Pinning versions of external actions. [#1334](https://github.com/apache/datafusion-ballista/pull/1334) (samueleresca) +- chore: update python deps to 49 [#1335](https://github.com/apache/datafusion-ballista/pull/1335) (milenkovicm) +- chore: update datafusion to 50.3 [#1336](https://github.com/apache/datafusion-ballista/pull/1336) (milenkovicm) +- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm) +- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan) +- minor: change log level [#1340](https://github.com/apache/datafusion-ballista/pull/1340) (milenkovicm) diff --git a/docs/source/changelog/51.0.0.md b/docs/source/changelog/51.0.0.md new file mode 100644 index 0000000000..a6b125d48d --- /dev/null +++ b/docs/source/changelog/51.0.0.md @@ -0,0 +1,45 @@ + + +# Apache DataFusion Ballista 51.0.0 Changelog + +**Implemented enhancements:** + +- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod) +- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner) +- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang) +- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento) +- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento) + +**Merged pull requests:** + +- minor: minor changes to release script/docs [#1342](https://github.com/apache/datafusion-ballista/pull/1342) (andygrove) +- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod) +- chore: update datafusion to 51.0 [#1345](https://github.com/apache/datafusion-ballista/pull/1345) (danielhumanmod) +- doc: Add a note that datafusion may need to be downgraded after installing it [#1348](https://github.com/apache/datafusion-ballista/pull/1348) (martin-g) +- minor: Make `DisplayAs` consistent and more readable for ShuffleExec [#1347](https://github.com/apache/datafusion-ballista/pull/1347) (milenkovicm) +- minor: remove unnecessary clone functions [#1352](https://github.com/apache/datafusion-ballista/pull/1352) (mmooyyii) +- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner) +- chore(doc): Clean up deployment docs. [#1354](https://github.com/apache/datafusion-ballista/pull/1354) (LouisBurke) +- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang) +- doc: Fix plan translation example to use correct aggregation and column [#1362](https://github.com/apache/datafusion-ballista/pull/1362) (mattcuento) +- chore: update ballista version to 51.0.0 (from 50.0.0) [#1363](https://github.com/apache/datafusion-ballista/pull/1363) (milenkovicm) +- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento) +- chore: Add missing public API documentation/comments [#1364](https://github.com/apache/datafusion-ballista/pull/1364) (killzoner) +- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento) diff --git a/docs/source/changelog/52.0.0.md b/docs/source/changelog/52.0.0.md new file mode 100644 index 0000000000..eed5a10076 --- /dev/null +++ b/docs/source/changelog/52.0.0.md @@ -0,0 +1,92 @@ + + +# Apache DataFusion Ballista 52.0.0 Changelog + +**Performance related:** + +- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove) + +**Implemented enhancements:** + +- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner) +- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove) +- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove) +- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm) +- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod) +- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg) +- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento) +- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc) +- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove) +- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm) +- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento) +- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner) +- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm) +- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm) +- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm) + +**Fixed bugs:** + +- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm) +- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner) +- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm) +- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g) + +**Documentation updates:** + +- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove) +- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove) +- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove) +- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento) + +**Merged pull requests:** + +- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner) +- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove) +- restrict python CI to python directory [#1383](https://github.com/apache/datafusion-ballista/pull/1383) (Huy1Ng) +- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove) +- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove) +- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove) +- doc: Add Ballista extensions example to the docs. [#1382](https://github.com/apache/datafusion-ballista/pull/1382) (LouisBurke) +- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove) +- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm) +- doc: add more details for protobuf extension [#1393](https://github.com/apache/datafusion-ballista/pull/1393) (LouisBurke) +- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod) +- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove) +- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg) +- chore: update datafusion to 52 [#1394](https://github.com/apache/datafusion-ballista/pull/1394) (killzoner) +- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento) +- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm) +- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc) +- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento) +- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner) +- Improve sort-based shuffle: single spill file per partition and batch coalescing [#1431](https://github.com/apache/datafusion-ballista/pull/1431) (andygrove) +- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove) +- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm) +- chore: add confirmation before tarball is released [#1445](https://github.com/apache/datafusion-ballista/pull/1445) (milenkovicm) +- minor: add test to cover IPC arrow file read [#1450](https://github.com/apache/datafusion-ballista/pull/1450) (milenkovicm) +- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento) +- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner) +- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm) +- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm) +- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm) +- minor: improve log statements [#1482](https://github.com/apache/datafusion-ballista/pull/1482) (milenkovicm) +- chore: update datafusion to 52.2 and other deps to latest [#1483](https://github.com/apache/datafusion-ballista/pull/1483) (milenkovicm) +- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g) +- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm) diff --git a/docs/source/changelog/53.0.0.md b/docs/source/changelog/53.0.0.md new file mode 100644 index 0000000000..94a5bd1c26 --- /dev/null +++ b/docs/source/changelog/53.0.0.md @@ -0,0 +1,233 @@ + + +# Apache DataFusion Ballista 53.0.0 Changelog + +This release consists of 174 commits from 16 contributors. See credits at the end of this changelog for more information. + +**Fixed bugs:** + +- fix: remove unwrap from executor and improve task error handling [#1540](https://github.com/apache/datafusion-ballista/pull/1540) (eachsaj) +- fix: handle None task slot in update_task_info after executor lost [#1523](https://github.com/apache/datafusion-ballista/pull/1523) (milenkovicm) +- fix: [Python] serialize optimized logical plan to fix subquery support [#1586](https://github.com/apache/datafusion-ballista/pull/1586) (andygrove) +- fix: allow S3 access without explicit credentials [#1584](https://github.com/apache/datafusion-ballista/pull/1584) (andygrove) +- fix: route collect/show/to_pandas through Ballista cluster [#1585](https://github.com/apache/datafusion-ballista/pull/1585) (andygrove) +- fix: propagate session config from Python client to Ballista cluster [#1592](https://github.com/apache/datafusion-ballista/pull/1592) (andygrove) +- fix(python): ignore ballista-namespaced cluster_config keys locally [#1613](https://github.com/apache/datafusion-ballista/pull/1613) (andygrove) +- fix: `df.write_` fix as it was broken after update [#1625](https://github.com/apache/datafusion-ballista/pull/1625) (milenkovicm) +- fix(rest): remove unwrap and return 404 if executor does not exist [#1628](https://github.com/apache/datafusion-ballista/pull/1628) (milenkovicm) +- fix(metrics): avoid stage metrics inflation by tracking partition snapshots [#1652](https://github.com/apache/datafusion-ballista/pull/1652) (danielhumanmod) +- fix: compilation issue after merge [#1658](https://github.com/apache/datafusion-ballista/pull/1658) (milenkovicm) +- fix: rest api calculates stage running time correctly [#1675](https://github.com/apache/datafusion-ballista/pull/1675) (milenkovicm) +- fix: REST API does not show running jobs [#1703](https://github.com/apache/datafusion-ballista/pull/1703) (gittihub-jpg) +- fix: no executor warning, correct prometheus feature name in TUI [#1698](https://github.com/apache/datafusion-ballista/pull/1698) (killzoner) + +**Implemented enhancements:** + +- [REST]: Cancelling a completed/failed job should return "false" [#1494](https://github.com/apache/datafusion-ballista/pull/1494) (martin-g) +- feat: Add plain "status" field to the JobResponse [#1497](https://github.com/apache/datafusion-ballista/pull/1497) (martin-g) +- feat: Expose Logical and Physical plan details in the REST API [#1498](https://github.com/apache/datafusion-ballista/pull/1498) (milenkovicm) +- feat: (remote) shuffle reader cleanup [#1503](https://github.com/apache/datafusion-ballista/pull/1503) (milenkovicm) +- feat: enable scheduler rest api by default [#1506](https://github.com/apache/datafusion-ballista/pull/1506) (milenkovicm) +- feat: [REST] Return the job's start and end times [#1511](https://github.com/apache/datafusion-ballista/pull/1511) (martin-g) +- feat: remove hash policy task distribution as not used [#1526](https://github.com/apache/datafusion-ballista/pull/1526) (eachsaj) +- feat: remove full path from partition locations [#1527](https://github.com/apache/datafusion-ballista/pull/1527) (milenkovicm) +- feat: add config to ExecutorEngine::create_query_stage_exec [#1542](https://github.com/apache/datafusion-ballista/pull/1542) (milenkovicm) +- feat: Improve REST API adding task related info to job stages [#1543](https://github.com/apache/datafusion-ballista/pull/1543) (milenkovicm) +- feat: Ballista Text User Interface [#1436](https://github.com/apache/datafusion-ballista/pull/1436) (martin-g) +- feat: jupyter notebook support [#1513](https://github.com/apache/datafusion-ballista/pull/1513) (sandugood) +- feat: ExecutionEngine::create_query_stage_exec accepts partition_id [#1556](https://github.com/apache/datafusion-ballista/pull/1556) (milenkovicm) +- feat: make ballista client retry policy configurable [#1577](https://github.com/apache/datafusion-ballista/pull/1577) (eachsaj) +- feat: Executor system & process metrics reporting [#1547](https://github.com/apache/datafusion-ballista/pull/1547) (sandugood) +- feat: Scheduler config update [#1597](https://github.com/apache/datafusion-ballista/pull/1597) (sandugood) +- feat: Support `EXPLAIN ANALYZE` in Ballista [#1567](https://github.com/apache/datafusion-ballista/pull/1567) (danielhumanmod) +- feat: Add standalone shuffle writer benchmark that shuffles real Parquet input [#1600](https://github.com/apache/datafusion-ballista/pull/1600) (andygrove) +- feat: defer sort-shuffle materialization with interleave_record_batch [#1598](https://github.com/apache/datafusion-ballista/pull/1598) (andygrove) +- feat(executor): bound executor memory via --memory-pool-size [#1624](https://github.com/apache/datafusion-ballista/pull/1624) (andygrove) +- feat(bench): show TPC-H query timings in seconds and add total time [#1641](https://github.com/apache/datafusion-ballista/pull/1641) (andygrove) +- feat(aqe): Support sort-based shuffle writer in AQE [#1640](https://github.com/apache/datafusion-ballista/pull/1640) (danielhumanmod) +- feat: rest api supports plan tree rendering [#1650](https://github.com/apache/datafusion-ballista/pull/1650) (sandugood) +- feat: Cache ballista clients on executor [#1578](https://github.com/apache/datafusion-ballista/pull/1578) (milenkovicm) +- feat(aqe): Lazy stage evaluation in AQE [#1649](https://github.com/apache/datafusion-ballista/pull/1649) (milenkovicm) +- feat: move shuffle writer disk I/O off tokio worker threads [#1537](https://github.com/apache/datafusion-ballista/pull/1537) (hcrosse) +- feat(scheduler): broadcast-style hash join for small-side joins [#1647](https://github.com/apache/datafusion-ballista/pull/1647) (andygrove) +- feat: default to sort-merge join [#1651](https://github.com/apache/datafusion-ballista/pull/1651) (andygrove) +- feat: TUI shows running job information [#1717](https://github.com/apache/datafusion-ballista/pull/1717) (milenkovicm) +- feat(aqe): CoalescePartitionsRule — shuffle-partition coalescing on resolved stats [#1684](https://github.com/apache/datafusion-ballista/pull/1684) (metegenez) +- feat: TUI make task popup scrollable [#1725](https://github.com/apache/datafusion-ballista/pull/1725) (milenkovicm) +- feat(tui): Use separate areas for the table and its associated scrollbar [#1729](https://github.com/apache/datafusion-ballista/pull/1729) (martin-g) + +**Documentation updates:** + +- docs: document hash-based and sort-based shuffle implementations [#1595](https://github.com/apache/datafusion-ballista/pull/1595) (andygrove) +- docs: improve Python documentation structure [#1579](https://github.com/apache/datafusion-ballista/pull/1579) (andygrove) +- docs: add Python client build-from-source section to contributor guide [#1620](https://github.com/apache/datafusion-ballista/pull/1620) (andygrove) +- chore: remove NYC Taxi benchmark [#1644](https://github.com/apache/datafusion-ballista/pull/1644) (andygrove) +- docs: document experimental Adaptive Query Execution in tuning guide [#1645](https://github.com/apache/datafusion-ballista/pull/1645) (andygrove) +- build(bench): rework docker-compose TPC-H stack [#1646](https://github.com/apache/datafusion-ballista/pull/1646) (andygrove) +- docs: add Ballista TUI documentation [#1593](https://github.com/apache/datafusion-ballista/pull/1593) (goingforstudying-ctrl) +- [TUI] Show executor's details in a popup [#1670](https://github.com/apache/datafusion-ballista/pull/1670) (martin-g) +- [TUI] Add a config setting for rendering job stage's plan as a tree [#1704](https://github.com/apache/datafusion-ballista/pull/1704) (martin-g) +- [TUI] Add support for horizontal scrolling to the job/stage plan popups [#1711](https://github.com/apache/datafusion-ballista/pull/1711) (martin-g) +- [TUI] Add screenshots of the TUI application in README/cli.md [#1714](https://github.com/apache/datafusion-ballista/pull/1714) (martin-g) + +**Other:** + +- CI: Add CodeQL workflow for GitHub Actions security scanning [#1484](https://github.com/apache/datafusion-ballista/pull/1484) (kevinjqliu) +- ci: Harden labeler workflow, remove unnecessary checkout from pull_request_target job [#1487](https://github.com/apache/datafusion-ballista/pull/1487) (kevinjqliu) +- chore(deps): bump tokio from 1.49.0 to 1.50.0 [#1490](https://github.com/apache/datafusion-ballista/pull/1490) (dependabot[bot]) +- chore(deps): bump actions/setup-node from 6.2.0 to 6.3.0 [#1489](https://github.com/apache/datafusion-ballista/pull/1489) (dependabot[bot]) +- ci: add take and stale [#1488](https://github.com/apache/datafusion-ballista/pull/1488) (kevinjqliu) +- chore(deps): bump uuid from 1.21.0 to 1.22.0 [#1492](https://github.com/apache/datafusion-ballista/pull/1492) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.32.5 to 4.32.6 [#1491](https://github.com/apache/datafusion-ballista/pull/1491) (dependabot[bot]) +- chore(deps): bump libc from 0.2.182 to 0.2.183 [#1495](https://github.com/apache/datafusion-ballista/pull/1495) (dependabot[bot]) +- chore(deps): bump graphviz-rust from 0.9.6 to 0.9.7 [#1496](https://github.com/apache/datafusion-ballista/pull/1496) (dependabot[bot]) +- chore(deps): bump quinn-proto from 0.11.13 to 0.11.14 [#1499](https://github.com/apache/datafusion-ballista/pull/1499) (dependabot[bot]) +- chore(deps): bump quinn-proto from 0.11.13 to 0.11.14 in /python [#1500](https://github.com/apache/datafusion-ballista/pull/1500) (dependabot[bot]) +- chore(deps): bump tempfile from 3.26.0 to 3.27.0 [#1501](https://github.com/apache/datafusion-ballista/pull/1501) (dependabot[bot]) +- chore(deps): cargo update deps [#1502](https://github.com/apache/datafusion-ballista/pull/1502) (milenkovicm) +- chore(deps): bump once_cell from 1.21.3 to 1.21.4 [#1505](https://github.com/apache/datafusion-ballista/pull/1505) (dependabot[bot]) +- chore(deps): bump clap from 4.5.60 to 4.6.0 [#1504](https://github.com/apache/datafusion-ballista/pull/1504) (dependabot[bot]) +- minor: task scheduling policy config cleanup [#1507](https://github.com/apache/datafusion-ballista/pull/1507) (milenkovicm) +- minor: address task policy config comments [#1508](https://github.com/apache/datafusion-ballista/pull/1508) (milenkovicm) +- chore(deps): bump tracing-subscriber from 0.3.22 to 0.3.23 [#1510](https://github.com/apache/datafusion-ballista/pull/1510) (dependabot[bot]) +- minor: [REST] add datafusion version info [#1512](https://github.com/apache/datafusion-ballista/pull/1512) (milenkovicm) +- chore(deps): bump lz4_flex from 0.12.0 to 0.12.1 in /python [#1514](https://github.com/apache/datafusion-ballista/pull/1514) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.32.6 to 4.33.0 [#1515](https://github.com/apache/datafusion-ballista/pull/1515) (dependabot[bot]) +- chore(deps): bump rustls-webpki from 0.103.9 to 0.103.10 in /python [#1517](https://github.com/apache/datafusion-ballista/pull/1517) (dependabot[bot]) +- chore(deps): bump lz4_flex from 0.12.0 to 0.12.1 [#1518](https://github.com/apache/datafusion-ballista/pull/1518) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.33.0 to 4.34.1 [#1519](https://github.com/apache/datafusion-ballista/pull/1519) (dependabot[bot]) +- chore(deps): bump rustls-webpki from 0.103.9 to 0.103.10 [#1521](https://github.com/apache/datafusion-ballista/pull/1521) (dependabot[bot]) +- ci: pin third-party actions to Apache-approved SHAs [#1516](https://github.com/apache/datafusion-ballista/pull/1516) (kevinjqliu) +- chore(deps): bump env_logger from 0.11.9 to 0.11.10 [#1522](https://github.com/apache/datafusion-ballista/pull/1522) (dependabot[bot]) +- chore(deps): update to datafusion v.53 [#1486](https://github.com/apache/datafusion-ballista/pull/1486) (milenkovicm) +- chore(deps): bump insta from 1.46.3 to 1.47.0 [#1524](https://github.com/apache/datafusion-ballista/pull/1524) (dependabot[bot]) +- chore(deps): bump uuid from 1.22.0 to 1.23.0 [#1525](https://github.com/apache/datafusion-ballista/pull/1525) (dependabot[bot]) +- chore: update datafusion proto [#1528](https://github.com/apache/datafusion-ballista/pull/1528) (milenkovicm) +- minor: cleanup scheduler clap configuration [#1529](https://github.com/apache/datafusion-ballista/pull/1529) (milenkovicm) +- chore(deps): bump insta from 1.47.0 to 1.47.1 [#1532](https://github.com/apache/datafusion-ballista/pull/1532) (dependabot[bot]) +- chore(deps): bump ctor from 0.6.3 to 0.8.0 [#1534](https://github.com/apache/datafusion-ballista/pull/1534) (dependabot[bot]) +- chore(deps): bump md-5 from 0.10.6 to 0.11.0 [#1533](https://github.com/apache/datafusion-ballista/pull/1533) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.34.1 to 4.35.1 [#1530](https://github.com/apache/datafusion-ballista/pull/1530) (dependabot[bot]) +- chore(deps): bump insta from 1.47.1 to 1.47.2 [#1535](https://github.com/apache/datafusion-ballista/pull/1535) (dependabot[bot]) +- chore(deps): bump libc from 0.2.183 to 0.2.184 [#1538](https://github.com/apache/datafusion-ballista/pull/1538) (dependabot[bot]) +- chore(deps): bump tokio from 1.50.0 to 1.51.0 [#1544](https://github.com/apache/datafusion-ballista/pull/1544) (dependabot[bot]) +- chore(deps): bump tui-big-text from 0.8.3 to 0.8.4 [#1546](https://github.com/apache/datafusion-ballista/pull/1546) (dependabot[bot]) +- chore(deps): bump tokio from 1.51.0 to 1.51.1 [#1545](https://github.com/apache/datafusion-ballista/pull/1545) (dependabot[bot]) +- chore(deps): bump ctor from 0.8.0 to 0.9.1 [#1548](https://github.com/apache/datafusion-ballista/pull/1548) (dependabot[bot]) +- chore(deps): bump ctor from 0.9.1 to 0.10.0 [#1549](https://github.com/apache/datafusion-ballista/pull/1549) (dependabot[bot]) +- chore(deps): bump rustls from 0.23.37 to 0.23.38 [#1550](https://github.com/apache/datafusion-ballista/pull/1550) (dependabot[bot]) +- chore(deps): bump libc from 0.2.184 to 0.2.185 [#1553](https://github.com/apache/datafusion-ballista/pull/1553) (dependabot[bot]) +- chore(deps): bump rand from 0.9.2 to 0.9.4 in /python [#1552](https://github.com/apache/datafusion-ballista/pull/1552) (dependabot[bot]) +- chore(deps): bump tokio from 1.51.1 to 1.52.0 [#1555](https://github.com/apache/datafusion-ballista/pull/1555) (dependabot[bot]) +- chore(deps): bump axum from 0.8.8 to 0.8.9 [#1554](https://github.com/apache/datafusion-ballista/pull/1554) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.35.1 to 4.35.2 [#1557](https://github.com/apache/datafusion-ballista/pull/1557) (dependabot[bot]) +- chore: Fix Clippy issues with Rust 1.95.0 [#1558](https://github.com/apache/datafusion-ballista/pull/1558) (martin-g) +- chore: update deps [#1561](https://github.com/apache/datafusion-ballista/pull/1561) (milenkovicm) +- chore(deps): bump actions/setup-node from 6.3.0 to 6.4.0 [#1562](https://github.com/apache/datafusion-ballista/pull/1562) (dependabot[bot]) +- chore(deps): bump mimalloc from 0.1.48 to 0.1.49 [#1566](https://github.com/apache/datafusion-ballista/pull/1566) (dependabot[bot]) +- chore(deps): bump aws-config from 1.8.15 to 1.8.16 [#1564](https://github.com/apache/datafusion-ballista/pull/1564) (dependabot[bot]) +- chore(deps): bump rand from 0.9.4 to 0.10.1 [#1565](https://github.com/apache/datafusion-ballista/pull/1565) (dependabot[bot]) +- chore(deps): bump ctor from 0.10.0 to 0.10.1 [#1571](https://github.com/apache/datafusion-ballista/pull/1571) (dependabot[bot]) +- chore(deps): bump mimalloc from 0.1.49 to 0.1.50 [#1570](https://github.com/apache/datafusion-ballista/pull/1570) (dependabot[bot]) +- chore(deps): bump rustls from 0.23.38 to 0.23.39 [#1569](https://github.com/apache/datafusion-ballista/pull/1569) (dependabot[bot]) +- chore(deps): bump libc from 0.2.185 to 0.2.186 [#1572](https://github.com/apache/datafusion-ballista/pull/1572) (dependabot[bot]) +- [TUI] Change the key binding for job's plans [#1573](https://github.com/apache/datafusion-ballista/pull/1573) (martin-g) +- chore(deps): bump rustls-webpki from 0.103.12 to 0.103.13 [#1576](https://github.com/apache/datafusion-ballista/pull/1576) (dependabot[bot]) +- chore(deps): bump rustls-webpki from 0.103.10 to 0.103.13 in /python [#1575](https://github.com/apache/datafusion-ballista/pull/1575) (dependabot[bot]) +- minor: make config naming convention consistent [#1580](https://github.com/apache/datafusion-ballista/pull/1580) (milenkovicm) +- chore: Remove unnecessary optimizer rules due to datafusion upgrade to v53 [#1594](https://github.com/apache/datafusion-ballista/pull/1594) (sandugood) +- chore(deps): bump reqwest from 0.13.2 to 0.13.3 [#1605](https://github.com/apache/datafusion-ballista/pull/1605) (dependabot[bot]) +- ci: drop Intel macOS Python wheel build [#1612](https://github.com/apache/datafusion-ballista/pull/1612) (andygrove) +- chore(deps): bump ctor from 0.10.1 to 0.11.1 [#1618](https://github.com/apache/datafusion-ballista/pull/1618) (dependabot[bot]) +- chore(deps): bump rustls from 0.23.39 to 0.23.40 [#1619](https://github.com/apache/datafusion-ballista/pull/1619) (dependabot[bot]) +- chore: update python deps to ballista and datafusion 52 [#1590](https://github.com/apache/datafusion-ballista/pull/1590) (andygrove) +- feat(sort-shuffle): byte-copy spill files and enable block-IO transport [#1615](https://github.com/apache/datafusion-ballista/pull/1615) (andygrove) +- Use BallistaSessionContext instead of BallistaBuilder in tpch.py [#1621](https://github.com/apache/datafusion-ballista/pull/1621) (martin-g) +- feat(sort-shuffle): enable sort-based shuffle by default [#1623](https://github.com/apache/datafusion-ballista/pull/1623) (andygrove) +- perf(sort-shuffle): fix performance regression caused by datafusion upgrade [#1626](https://github.com/apache/datafusion-ballista/pull/1626) (andygrove) +- chore(deps): bump ctor from 0.11.1 to 0.12.0 [#1639](https://github.com/apache/datafusion-ballista/pull/1639) (dependabot[bot]) +- fix(sort-shuffle): bound writer memory with per-task spill threshold [#1636](https://github.com/apache/datafusion-ballista/pull/1636) (andygrove) +- chore(deps): bump graphviz-rust from 0.9.7 to 0.9.8 [#1655](https://github.com/apache/datafusion-ballista/pull/1655) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.35.2 to 4.35.3 [#1653](https://github.com/apache/datafusion-ballista/pull/1653) (dependabot[bot]) +- chore(deps): bump tokio from 1.52.1 to 1.52.2 [#1657](https://github.com/apache/datafusion-ballista/pull/1657) (dependabot[bot]) +- chore(deps): bump actions/labeler from 6.0.1 to 6.1.0 [#1659](https://github.com/apache/datafusion-ballista/pull/1659) (dependabot[bot]) +- [TUI] Show job's stages and their tasks [#1574](https://github.com/apache/datafusion-ballista/pull/1574) (martin-g) +- chore(deps): bump tonic-build from 0.14.5 to 0.14.6 [#1662](https://github.com/apache/datafusion-ballista/pull/1662) (dependabot[bot]) +- chore(deps): bump tonic from 0.14.5 to 0.14.6 [#1661](https://github.com/apache/datafusion-ballista/pull/1661) (dependabot[bot]) +- chore(deps): bump astral-tokio-tar from 0.6.0 to 0.6.1 [#1664](https://github.com/apache/datafusion-ballista/pull/1664) (dependabot[bot]) +- chore(deps): bump ctor from 0.12.0 to 1.0.2 [#1663](https://github.com/apache/datafusion-ballista/pull/1663) (dependabot[bot]) +- chore(deps): bump github/codeql-action from 4.35.3 to 4.35.4 [#1665](https://github.com/apache/datafusion-ballista/pull/1665) (dependabot[bot]) +- chore(deps): bump ctor from 1.0.2 to 1.0.3 [#1668](https://github.com/apache/datafusion-ballista/pull/1668) (dependabot[bot]) +- chore(deps): bump tonic-prost from 0.14.5 to 0.14.6 [#1667](https://github.com/apache/datafusion-ballista/pull/1667) (dependabot[bot]) +- chore(deps): bump tonic-prost-build from 0.14.5 to 0.14.6 [#1666](https://github.com/apache/datafusion-ballista/pull/1666) (dependabot[bot]) +- [TUI] Configurable tick interval [#1669](https://github.com/apache/datafusion-ballista/pull/1669) (martin-g) +- minor: add pending stage indicator for `AdaptiveDatafusionExec` [#1672](https://github.com/apache/datafusion-ballista/pull/1672) (milenkovicm) +- chore(deps): bump ctor from 1.0.3 to 1.0.4 [#1680](https://github.com/apache/datafusion-ballista/pull/1680) (dependabot[bot]) +- chore(deps): bump tokio from 1.52.2 to 1.52.3 [#1678](https://github.com/apache/datafusion-ballista/pull/1678) (dependabot[bot]) +- minor: change parameter ordering in `AdaptivePlanner::try_new_with_optimizers` [#1687](https://github.com/apache/datafusion-ballista/pull/1687) (milenkovicm) +- chore(deps): bump ctor from 1.0.4 to 1.0.5 [#1694](https://github.com/apache/datafusion-ballista/pull/1694) (dependabot[bot]) +- Fix REST API panic on job list/detail when end_time < start_time [#1693](https://github.com/apache/datafusion-ballista/pull/1693) (abhinavgautam01) +- Right align all numeric columns in the TUI tables [#1695](https://github.com/apache/datafusion-ballista/pull/1695) (martin-g) +- fix(join-selection): guard CollectLeft swap when right has multiple partitions [#1691](https://github.com/apache/datafusion-ballista/pull/1691) (andygrove) +- Minor improvements for #1675 [#1686](https://github.com/apache/datafusion-ballista/pull/1686) (martin-g) +- [CLI/TUI] Use only tracing crate for logging in CLI and TUI [#1697](https://github.com/apache/datafusion-ballista/pull/1697) (martin-g) +- chore(deps): bump urllib3 from 2.6.3 to 2.7.0 in /python [#1699](https://github.com/apache/datafusion-ballista/pull/1699) (dependabot[bot]) +- chore(deps): bump pyjwt from 2.10.1 to 2.12.0 in /python [#1700](https://github.com/apache/datafusion-ballista/pull/1700) (dependabot[bot]) +- chore(deps): bump pytest from 9.0.2 to 9.0.3 in /python [#1702](https://github.com/apache/datafusion-ballista/pull/1702) (dependabot[bot]) +- minor: change log level for few statements [#1706](https://github.com/apache/datafusion-ballista/pull/1706) (milenkovicm) +- chore(deps): bump config from 0.15.22 to 0.15.23 [#1709](https://github.com/apache/datafusion-ballista/pull/1709) (dependabot[bot]) +- Saturate scheduler job elapsed time [#1708](https://github.com/apache/datafusion-ballista/pull/1708) (MukundaKatta) +- [TUI] Executor's id is not a numeric column. It should be center aligned [#1713](https://github.com/apache/datafusion-ballista/pull/1713) (martin-g) +- Make use of Swatinem/rust-cache to make the CI workflows faster [#1705](https://github.com/apache/datafusion-ballista/pull/1705) (martin-g) +- Merge Executor's brief and extended properties [#1716](https://github.com/apache/datafusion-ballista/pull/1716) (martin-g) +- [INFRA] Set up default rulesets for default and release branches [#1715](https://github.com/apache/datafusion-ballista/pull/1715) (asf-gitbox-commits) +- chore(deps): bump github/codeql-action from 4.35.4 to 4.35.5 [#1719](https://github.com/apache/datafusion-ballista/pull/1719) (dependabot[bot]) +- chore(deps): bump ctor from 1.0.5 to 1.0.6 [#1720](https://github.com/apache/datafusion-ballista/pull/1720) (dependabot[bot]) +- chore(deps): bump dashmap from 6.1.0 to 6.2.1 [#1721](https://github.com/apache/datafusion-ballista/pull/1721) (dependabot[bot]) +- ci: add TPC-H SF10 workflow [#1688](https://github.com/apache/datafusion-ballista/pull/1688) (andygrove) +- minor: [TUI] Extract a helper method for the table/scrollbar area splitter [#1730](https://github.com/apache/datafusion-ballista/pull/1730) (martin-g) +- minor: [TUI] Sort the metrics before rendering them [#1731](https://github.com/apache/datafusion-ballista/pull/1731) (martin-g) +- chore: Do not run -cli tests twice [#1732](https://github.com/apache/datafusion-ballista/pull/1732) (martin-g) +- chore: generate changelog for ballista 53 [#1735](https://github.com/apache/datafusion-ballista/pull/1735) (milenkovicm) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 74 dependabot[bot] + 29 Marko Milenković + 25 Andy Grove + 23 Martin Grigorov + 5 alexander domenti + 4 Kevin Liu + 3 Daniel Tu + 3 Saj + 1 Abhinav Gautam + 1 Harrison Crosse + 1 Mukunda Rao Katta + 1 The Apache Software Foundation + 1 gittihub-jpg + 1 goingforstudying-ctrl + 1 jgrim + 1 mete +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/docs/source/changelog/index.md b/docs/source/changelog/index.md new file mode 100644 index 0000000000..653f006729 --- /dev/null +++ b/docs/source/changelog/index.md @@ -0,0 +1,46 @@ + + +# Changelog + +Per-release change logs for Apache DataFusion Ballista. + +```{toctree} +:maxdepth: 1 + +53.0.0 +52.0.0 +51.0.0 +50.0.0 +49.0.0 +48.0.0 +47.0.0 +46.0.0 +45.0.0 +44.0.0 +43.0.0 +0.12.0 +0.11.0 +0.10.0 +0.9.0 +0.8.0 +0.7.0 +0.6.0 +0.5.0 +``` diff --git a/docs/source/index.rst b/docs/source/index.rst index f1b3e3d59e..b14743be68 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -58,6 +58,12 @@ Table of content user-guide/spark-compatible-functions user-guide/extensions-example +.. toctree:: + :maxdepth: 1 + :caption: Changelog + + changelog/index + .. _toc.contributors: .. toctree:: From 5526c462371ee82cf46593fe9702729cdf6a0937 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 29 Jun 2026 07:38:46 -0600 Subject: [PATCH 013/107] ci: run TPC-H SF10 under AQE off + on, and build it faster (#1913) --- .github/workflows/tpch.yml | 47 ++++++++++++++++++++++++-------------- Cargo.toml | 14 ++++++++++++ 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 7dda5c1827..7e5f4b2665 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -48,7 +48,7 @@ on: jobs: tpch-sf10: - name: TPC-H SF10 (all queries) + name: TPC-H SF10 (all queries, AQE off + on) runs-on: ubuntu-latest container: image: amd64/rust @@ -71,7 +71,7 @@ jobs: - name: Build Ballista binaries run: | - cargo build --profile release-nonlto --locked \ + cargo build --profile tpch-ci --locked \ -p ballista-scheduler \ -p ballista-executor \ -p ballista-benchmarks @@ -101,12 +101,12 @@ jobs: mkdir -p "$WORK_DIR" - ./target/release-nonlto/ballista-scheduler \ + ./target/tpch-ci/ballista-scheduler \ --bind-host 127.0.0.1 \ > "$SCHEDULER_LOG" 2>&1 & SCHEDULER_PID=$! - ./target/release-nonlto/ballista-executor \ + ./target/tpch-ci/ballista-executor \ --bind-host 127.0.0.1 \ --bind-port 50051 \ --scheduler-host 127.0.0.1 \ @@ -147,19 +147,32 @@ jobs: done nc -z 127.0.0.1 50051 || { echo "executor did not start"; exit 1; } - # q16 omitted: still unsupported (matches benchmarks/run.sh). - for q in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 17 18 19 20 21 22; do - echo "::group::Query $q" - ./target/release-nonlto/tpch benchmark ballista \ - --host 127.0.0.1 --port 50050 \ - --query "$q" \ - --path "$DATA_DIR" \ - --format parquet \ - --partitions 16 \ - --iterations 1 \ - -c datafusion.optimizer.prefer_hash_join=false - echo "::endgroup::" - done + # Run the suite once per planner mode against the same data and + # cluster. AQE on/off is a per-query session setting, so the scheduler + # picks the planner per job; no need to regenerate data or restart the + # cluster. + run_suite() { + local label="$1"; shift + # q16 omitted: still unsupported (matches benchmarks/run.sh). + for q in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 17 18 19 20 21 22; do + echo "::group::[$label] Query $q" + ./target/tpch-ci/tpch benchmark ballista \ + --host 127.0.0.1 --port 50050 \ + --query "$q" \ + --path "$DATA_DIR" \ + --format parquet \ + --partitions 16 \ + --iterations 1 \ + "$@" + echo "::endgroup::" + done + } + + run_suite "AQE off" \ + -c datafusion.optimizer.prefer_hash_join=false + run_suite "AQE on" \ + -c datafusion.optimizer.prefer_hash_join=false \ + -c ballista.planner.adaptive.enabled=true - name: Upload cluster logs on failure if: failure() diff --git a/Cargo.toml b/Cargo.toml index 57346eaf05..da1776f000 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,4 +101,18 @@ strip = "debuginfo" debug = false debug-assertions = false strip = "debuginfo" +incremental = false + +# Build profile for the TPC-H integration CI job. That job only checks that the +# distributed suite runs correctly (no timing gate), so it trades some runtime +# speed for much faster compilation: opt-level 1 keeps queries fast enough while +# compiling dependencies far quicker than the opt-level 3 release profiles. Do +# not use this for performance numbers — use release-nonlto / release-lto. +[profile.tpch-ci] +inherits = "release" +opt-level = 1 +codegen-units = 256 +lto = false +debug = false +debug-assertions = false incremental = false \ No newline at end of file From 068096b1b9d719b47d6b0fea1e5a296f9298081c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 30 Jun 2026 06:54:14 -0600 Subject: [PATCH 014/107] chore(deps): upgrade to DataFusion 54 (#1906) --- Cargo.lock | 404 ++++++++---------- Cargo.toml | 18 +- ballista-cli/Cargo.toml | 2 +- ballista/client/tests/context_basic.rs | 2 +- ballista/core/src/diagram.rs | 40 +- .../core/src/execution_plans/chaos_exec.rs | 7 +- .../distributed_explain_analyze.rs | 7 - .../src/execution_plans/distributed_query.rs | 6 - .../src/execution_plans/shuffle_reader.rs | 13 +- .../src/execution_plans/shuffle_writer.rs | 47 +- .../execution_plans/sort_shuffle/writer.rs | 10 +- .../src/execution_plans/unresolved_shuffle.rs | 5 - ballista/core/src/extension.rs | 31 ++ ballista/core/src/planner.rs | 2 - ballista/core/src/registry.rs | 48 +++ ballista/core/src/serde/mod.rs | 81 +++- .../core/src/serde/scheduler/from_proto.rs | 10 +- ballista/executor/src/collect.rs | 8 +- ballista/executor/src/execution_engine.rs | 133 +++++- ballista/executor/src/execution_loop.rs | 3 + ballista/executor/src/executor.rs | 5 - ballista/executor/src/executor_server.rs | 9 + .../src/physical_optimizer/join_selection.rs | 109 +++-- ballista/scheduler/src/planner.rs | 173 ++------ ballista/scheduler/src/state/aqe/adapter.rs | 7 +- .../src/state/aqe/execution_plan/adaptive.rs | 5 - .../state/aqe/execution_plan/dynamic_join.rs | 10 +- .../src/state/aqe/execution_plan/exchange.rs | 13 +- .../state/aqe/optimizer_rule/chaos_exec.rs | 3 +- .../aqe/optimizer_rule/coalesce_partitions.rs | 9 +- .../optimizer_rule/distributed_exchange.rs | 127 ++---- .../aqe/optimizer_rule/join_selection.rs | 31 +- .../aqe/optimizer_rule/propagate_empty.rs | 44 +- ballista/scheduler/src/state/aqe/planner.rs | 18 +- .../src/state/aqe/test/alter_stages.rs | 60 ++- .../src/state/aqe/test/plan_to_stages.rs | 5 +- .../src/state/distributed_explain.rs | 2 +- .../scheduler/src/state/execution_graph.rs | 16 +- .../src/state/execution_graph_dot.rs | 77 ++-- .../scheduler/src/state/execution_stage.rs | 4 +- ballista/scheduler/src/test_utils.rs | 7 +- 41 files changed, 767 insertions(+), 844 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 342ad458aa..369a967188 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,35 +132,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest 0.10.7", - "liblzma", - "log", - "miniz_oxide 0.8.9", - "num-bigint", - "quad-rand", - "rand 0.9.4", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum 0.27.2", - "strum_macros 0.27.2", - "thiserror 2.0.18", - "uuid", - "zstd", -] - [[package]] name = "approx" version = "0.5.1" @@ -260,6 +231,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-avro" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap 2.14.0", + "liblzma", + "rand 0.9.4", + "serde", + "serde_json", + "snap", + "strum_macros", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" version = "58.3.0" @@ -583,7 +578,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.2", - "sha1", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -1297,7 +1292,6 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] [[package]] @@ -1457,31 +1451,6 @@ dependencies = [ "time", ] -[[package]] -name = "bon" -version = "3.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b" -dependencies = [ - "darling 0.23.0", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "brotli" version = "8.0.3" @@ -1921,6 +1890,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -2211,14 +2195,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "bzip2", "chrono", "datafusion-catalog", @@ -2249,14 +2232,13 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "liblzma", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -2267,9 +2249,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" dependencies = [ "arrow", "async-trait", @@ -2292,9 +2274,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" dependencies = [ "arrow", "async-trait", @@ -2315,9 +2297,9 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22c001ad1ac11cda09dab69b151eef5b1a992e23bc524ab0d1e63e5dea327" +checksum = "2a9533845ab888508718c2ad540cff205d1111525a9952def8b0e584b6453784" dependencies = [ "arrow", "async-trait", @@ -2343,17 +2325,17 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" dependencies = [ - "ahash", - "apache-avro", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "hex", "indexmap 2.14.0", "itertools 0.14.0", @@ -2361,18 +2343,18 @@ dependencies = [ "log", "object_store", "parquet", - "paste", "recursive", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" dependencies = [ "futures", "log", @@ -2381,9 +2363,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-compression", @@ -2407,6 +2389,7 @@ dependencies = [ "liblzma", "log", "object_store", + "parking_lot", "rand 0.9.4", "tokio", "tokio-util", @@ -2416,9 +2399,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" dependencies = [ "arrow", "arrow-ipc", @@ -2440,29 +2423,28 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" +checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", "datafusion-common", "datafusion-datasource", - "datafusion-physical-expr-common", + "datafusion-physical-expr-adapter", "datafusion-physical-plan", "datafusion-session", "futures", - "num-traits", "object_store", ] [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" dependencies = [ "arrow", "async-trait", @@ -2483,9 +2465,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" dependencies = [ "arrow", "async-trait", @@ -2500,16 +2482,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", @@ -2519,6 +2500,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -2537,20 +2519,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -2567,11 +2548,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2582,7 +2564,6 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "recursive", "serde_json", "sqlparser", @@ -2590,22 +2571,21 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" dependencies = [ "arrow", "arrow-buffer", @@ -2620,26 +2600,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", "rand 0.9.4", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -2649,19 +2628,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2670,9 +2648,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" dependencies = [ "arrow", "arrow-ord", @@ -2686,34 +2664,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" dependencies = [ "arrow", "datafusion-common", @@ -2724,14 +2702,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2739,9 +2716,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" dependencies = [ "datafusion-doc", "quote", @@ -2750,9 +2727,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" dependencies = [ "arrow", "chrono", @@ -2770,11 +2747,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2782,11 +2758,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "recursive", "tokio", @@ -2794,9 +2769,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" dependencies = [ "arrow", "datafusion-common", @@ -2809,26 +2784,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" dependencies = [ "arrow", "datafusion-common", @@ -2845,12 +2820,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2865,7 +2841,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2877,9 +2853,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" dependencies = [ "arrow", "chrono", @@ -2900,14 +2876,13 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost", - "rand 0.9.4", ] [[package]] name = "datafusion-proto-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" dependencies = [ "arrow", "datafusion-common", @@ -2916,9 +2891,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" dependencies = [ "arrow", "datafusion-common", @@ -2927,15 +2902,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" dependencies = [ "async-trait", "datafusion-common", @@ -2947,9 +2921,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" +checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" dependencies = [ "arrow", "bigdecimal", @@ -2962,21 +2936,24 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-functions-nested", "log", + "num-traits", "percent-encoding", "rand 0.9.4", "serde_json", - "sha1", - "sha2 0.10.9", + "sha1 0.11.0", + "sha2 0.11.0", + "twox-hash", "url", ] [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" dependencies = [ "arrow", "bigdecimal", @@ -2993,9 +2970,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2" dependencies = [ "async-recursion", "async-trait", @@ -3216,9 +3193,9 @@ dependencies = [ [[package]] name = "endian-type" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" [[package]] name = "env_filter" @@ -3306,17 +3283,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" -[[package]] -name = "fd-lock" -version = "4.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" -dependencies = [ - "cfg-if", - "rustix 1.1.4", - "windows-sys 0.59.0", -] - [[package]] name = "ferroid" version = "2.0.0" @@ -4705,9 +4671,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.30.1" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -4765,7 +4731,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -5581,12 +5546,6 @@ dependencies = [ "pulldown-cmark", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quick-xml" version = "0.39.4" @@ -5682,9 +5641,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "radix_trie" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" dependencies = [ "endian-type", "nibble_vec", @@ -5783,7 +5742,7 @@ dependencies = [ "lru 0.18.0", "palette", "serde", - "strum 0.28.0", + "strum", "thiserror 2.0.18", "unicode-segmentation", "unicode-truncate", @@ -5859,7 +5818,7 @@ dependencies = [ "line-clipping", "ratatui-core", "serde", - "strum 0.28.0", + "strum", "time", "unicode-segmentation", "unicode-width 0.2.2", @@ -6269,24 +6228,23 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline" -version = "17.0.2" +version = "18.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" +checksum = "53f6a737db68eb1a8ccff86b584b2fc13eca6a7bb6f78ebc7c529547e3ab9684" dependencies = [ "bitflags 2.13.0", "cfg-if", "clipboard-win", - "fd-lock", "home", "libc", "log", "memchr", - "nix 0.30.1", + "nix 0.31.3", "radix_trie", "unicode-segmentation", "unicode-width 0.2.2", "utf8parse", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6416,16 +6374,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -6463,6 +6411,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6572,6 +6521,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -6713,9 +6673,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -6811,31 +6771,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" - [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", + "strum_macros", ] [[package]] @@ -6852,11 +6794,12 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck 0.5.0", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -7605,6 +7548,9 @@ name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.4", +] [[package]] name = "typenum" diff --git a/Cargo.toml b/Cargo.toml index da1776f000..5719b9b16c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,20 +26,20 @@ edition = "2024" rust-version = "1.88.0" [workspace.dependencies] -arrow = { version = "58", features = ["ipc_compression"] } -arrow-flight = { version = "58", features = ["flight-sql-experimental"] } +arrow = { version = "58.3", features = ["ipc_compression"] } +arrow-flight = { version = "58.3", features = ["flight-sql-experimental"] } clap = { version = "4.5", features = ["derive", "cargo"] } -datafusion = "53" -datafusion-cli = "53" -datafusion-proto = "53" -datafusion-proto-common = "53" -datafusion-spark = "53" -datafusion-substrait = "53" +datafusion = "54" +datafusion-cli = "54" +datafusion-proto = "54" +datafusion-proto-common = "54" +datafusion-spark = "54" +datafusion-substrait = "54" insta = "1.47" itertools = "0.15" -object_store = "0.13" +object_store = "0.13.2" prost = "0.14" prost-types = "0.14" rstest = { version = "0.26" } diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml index 6796f49eeb..683a3c2453 100644 --- a/ballista-cli/Cargo.toml +++ b/ballista-cli/Cargo.toml @@ -36,7 +36,7 @@ crate-type = ["cdylib", "rlib"] ballista = { path = "../ballista/client", version = "53.0.0", features = ["standalone"], optional = true } datafusion = { workspace = true, optional = true } datafusion-cli = { workspace = true, optional = true } -rustyline = { version = "17.0.1", optional = true } +rustyline = { version = "18.0.0", optional = true } mimalloc = { workspace = true, optional = true } # TUI/web shared deps diff --git a/ballista/client/tests/context_basic.rs b/ballista/client/tests/context_basic.rs index 5e9c29bb38..395f570313 100644 --- a/ballista/client/tests/context_basic.rs +++ b/ballista/client/tests/context_basic.rs @@ -449,7 +449,7 @@ mod basic { "+-------------------------------------------------------------------+", "| approx_percentile_cont_with_weight(test.id,Int64(2),Float64(0.5)) |", "+-------------------------------------------------------------------+", - "| 3 |", + "| 3.5 |", "+-------------------------------------------------------------------+", ]; assert_result_eq(expected, &res); diff --git a/ballista/core/src/diagram.rs b/ballista/core/src/diagram.rs index 5498c1161c..79604e2a13 100644 --- a/ballista/core/src/diagram.rs +++ b/ballista/core/src/diagram.rs @@ -84,43 +84,27 @@ fn build_exec_plan_diagram( id: &mut AtomicUsize, draw_entity: bool, ) -> Result { - let operator_str = if plan.as_any().downcast_ref::().is_some() { + let operator_str = if plan.downcast_ref::().is_some() { "AggregateExec" - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "SortExec" - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "ProjectionExec" - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "HashJoinExec" - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "DataSourceExec" - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "FilterExec" - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "ShuffleWriterExec" - } else if plan - .as_any() - .downcast_ref::() - .is_some() - { + } else if plan.downcast_ref::().is_some() { "SortShuffleWriterExec" - } else if plan - .as_any() - .downcast_ref::() - .is_some() - { + } else if plan.downcast_ref::().is_some() { "UnresolvedShuffleExec" - } else if plan - .as_any() - .downcast_ref::() - .is_some() - { + } else if plan.downcast_ref::().is_some() { "CoalesceBatchesExec" - } else if plan - .as_any() - .downcast_ref::() - .is_some() - { + } else if plan.downcast_ref::().is_some() { "CoalescePartitionsExec" } else { warn!("Unknown: {plan:?}"); @@ -137,7 +121,7 @@ fn build_exec_plan_diagram( )?; } for child in plan.children() { - if let Some(shuffle) = child.as_any().downcast_ref::() { + if let Some(shuffle) = child.downcast_ref::() { if !draw_entity { writeln!( w, diff --git a/ballista/core/src/execution_plans/chaos_exec.rs b/ballista/core/src/execution_plans/chaos_exec.rs index fda436ec58..569f4da724 100644 --- a/ballista/core/src/execution_plans/chaos_exec.rs +++ b/ballista/core/src/execution_plans/chaos_exec.rs @@ -42,7 +42,6 @@ use datafusion::physical_plan::{ use futures::StreamExt; use rand::rngs::StdRng; use rand::{RngExt, SeedableRng}; -use std::any::Any; use std::sync::Arc; /// Physical execution plan node that randomly injects failures for chaos/robustness testing. @@ -124,10 +123,6 @@ impl ExecutionPlan for ChaosExec { "ChaosExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { self.input.properties() } @@ -218,7 +213,7 @@ impl ExecutionPlan for ChaosExec { self.input.metrics() } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.input.partition_statistics(partition) } diff --git a/ballista/core/src/execution_plans/distributed_explain_analyze.rs b/ballista/core/src/execution_plans/distributed_explain_analyze.rs index af02b0a22c..3d7ba2c867 100644 --- a/ballista/core/src/execution_plans/distributed_explain_analyze.rs +++ b/ballista/core/src/execution_plans/distributed_explain_analyze.rs @@ -37,7 +37,6 @@ use datafusion::physical_plan::{ }; use datafusion_proto::logical_plan::AsLogicalPlan; use futures::StreamExt; -use std::any::Any; use std::convert::TryInto; use std::marker::PhantomData; use std::sync::Arc; @@ -108,10 +107,6 @@ impl ExecutionPlan for DistributedExplainAnalyzeExec "DistributedExplainAnalyzeExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -133,7 +128,6 @@ impl ExecutionPlan for DistributedExplainAnalyzeExec let query_exec = children.pop().unwrap(); if query_exec - .as_any() .downcast_ref::>() .is_some() { @@ -173,7 +167,6 @@ impl ExecutionPlan for DistributedExplainAnalyzeExec } let job_id = query_exec - .as_any() .downcast_ref::>() .ok_or_else(|| { DataFusionError::Internal( diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs index 0181678104..a7d79ef824 100644 --- a/ballista/core/src/execution_plans/distributed_query.rs +++ b/ballista/core/src/execution_plans/distributed_query.rs @@ -49,7 +49,6 @@ use datafusion_proto::logical_plan::{ use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use log::{debug, error, info}; use parking_lot::Mutex; -use std::any::Any; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -177,10 +176,6 @@ impl ExecutionPlan for DistributedQueryExec { "DistributedQueryExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.plan.schema().as_arrow().clone().into() } @@ -842,7 +837,6 @@ mod test { let new_exec = exec.clone().with_new_children(vec![]).unwrap(); let new_exec = new_exec - .as_any() .downcast_ref::>() .unwrap(); diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index 556ccc8090..a9955ac5f5 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -47,7 +47,6 @@ use itertools::Itertools; use log::{debug, error, trace}; use rand::prelude::SliceRandom; use rand::rng; -use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; use std::fs::File; @@ -311,10 +310,6 @@ impl ExecutionPlan for ShuffleReaderExec { "ShuffleReaderExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } @@ -426,7 +421,7 @@ impl ExecutionPlan for ShuffleReaderExec { Some(self.metrics.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { if self.broadcast { if let Some(idx) = partition && idx != 0 @@ -445,7 +440,7 @@ impl ExecutionPlan for ShuffleReaderExec { "broadcast shuffle reader at stage {} returned aggregated statistics: {:?}", self.stage_id, stats ); - return Ok(stats); + return Ok(Arc::new(stats)); } if let Some(idx) = partition { let partition_count = self.properties().partitioning.partition_count(); @@ -474,7 +469,7 @@ impl ExecutionPlan for ShuffleReaderExec { "shuffle reader at stage: {} and partition {} returned statistics: {:?}", self.stage_id, idx, stat_for_partition ); - stat_for_partition + stat_for_partition.map(Arc::new) } else { let stats_for_partitions = stats_for_partitions( self.schema.fields().len(), @@ -487,7 +482,7 @@ impl ExecutionPlan for ShuffleReaderExec { "shuffle reader at stage: {} returned statistics for all partitions: {:?}", self.stage_id, stats_for_partitions ); - Ok(stats_for_partitions) + Ok(Arc::new(stats_for_partitions)) } } } diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index a75925d6df..3398be2275 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -24,7 +24,6 @@ use datafusion::arrow::ipc::CompressionType; use datafusion::arrow::ipc::writer::IpcWriteOptions; use datafusion::arrow::ipc::writer::StreamWriter; -use std::any::Any; use std::fmt::Debug; use std::fs; use std::fs::File; @@ -286,7 +285,7 @@ impl ShuffleWriterExec { exprs, num_output_partitions, repart_time, - ); + )?; while let Some(input_batch) = rx.blocking_recv() { partitioner.partition( @@ -438,10 +437,6 @@ impl ExecutionPlan for ShuffleWriterExec { "ShuffleWriterExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.plan.schema() } @@ -561,7 +556,7 @@ impl ExecutionPlan for ShuffleWriterExec { Some(self.metrics.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.plan.partition_statistics(partition) } } @@ -635,23 +630,22 @@ mod tests { assert_eq!(1, batches.len()); let batch = &batches[0]; assert_eq!(3, batch.num_columns()); - assert_eq!(2, batch.num_rows()); + // One metadata row per non-empty output partition; how many that is + // depends on the hash distribution, so only bound it. + let num_partitions = batch.num_rows(); + assert!((1..=2).contains(&num_partitions)); let path = batch.columns()[1] .as_any() .downcast_ref::() .unwrap(); - - let file0 = path.value(0); - assert!( - file0.ends_with("/jobOne/1/0/data-0.arrow") - || file0.ends_with("\\jobOne\\1\\0\\data-0.arrow") - ); - let file1 = path.value(1); - - assert!( - file1.ends_with("/jobOne/1/1/data-0.arrow") - || file1.ends_with("\\jobOne\\1\\1\\data-0.arrow") - ); + for i in 0..num_partitions { + let f = path.value(i); + assert!( + (0..2).any(|p| f.ends_with(&format!("/jobOne/1/{p}/data-0.arrow")) + || f.ends_with(&format!("\\jobOne\\1\\{p}\\data-0.arrow"))), + "unexpected shuffle file path: {f}" + ); + } let stats = batch.columns()[2] .as_any() @@ -664,8 +658,9 @@ mod tests { .as_any() .downcast_ref::() .unwrap(); - assert_eq!(4, num_rows.value(0)); - assert_eq!(4, num_rows.value(1)); + // Total rows are conserved across partitions regardless of the hash split. + let total: u64 = (0..num_partitions).map(|i| num_rows.value(i)).sum(); + assert_eq!(8, total); Ok(()) } @@ -693,7 +688,8 @@ mod tests { assert_eq!(1, batches.len()); let batch = &batches[0]; assert_eq!(3, batch.num_columns()); - assert_eq!(2, batch.num_rows()); + let num_partitions = batch.num_rows(); + assert!((1..=2).contains(&num_partitions)); let stats = batch.columns()[2] .as_any() .downcast_ref::() @@ -704,8 +700,9 @@ mod tests { .as_any() .downcast_ref::() .unwrap(); - assert_eq!(2, num_rows.value(0)); - assert_eq!(2, num_rows.value(1)); + // Total rows are conserved across partitions regardless of the hash split. + let total: u64 = (0..num_partitions).map(|i| num_rows.value(i)).sum(); + assert_eq!(4, total); Ok(()) } diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs b/ballista/core/src/execution_plans/sort_shuffle/writer.rs index 81b201680f..632b5aca07 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs @@ -21,7 +21,6 @@ //! per input partition, along with an index file mapping partition IDs to //! byte offsets. -use std::any::Any; use std::fs::File; use std::future::Future; use std::io::{BufWriter, Seek, Write}; @@ -539,10 +538,6 @@ impl ExecutionPlan for SortShuffleWriterExec { "SortShuffleWriterExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.plan.schema() } @@ -663,7 +658,7 @@ impl ExecutionPlan for SortShuffleWriterExec { Some(self.metrics.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.plan.partition_statistics(partition) } } @@ -1164,7 +1159,8 @@ mod tests { // Reference: DataFusion's BatchPartitioner::new_hash_partitioner let mut ref_partitioner = - BatchPartitioner::new_hash_partitioner(exprs.clone(), 4, Time::default()); + BatchPartitioner::new_hash_partitioner(exprs.clone(), 4, Time::default()) + .unwrap(); let mut ref_assignments = [usize::MAX; 10]; ref_partitioner .partition(batch.clone(), |partition, sub_batch| { diff --git a/ballista/core/src/execution_plans/unresolved_shuffle.rs b/ballista/core/src/execution_plans/unresolved_shuffle.rs index 3f3567b6a9..64bfd8fec9 100644 --- a/ballista/core/src/execution_plans/unresolved_shuffle.rs +++ b/ballista/core/src/execution_plans/unresolved_shuffle.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::any::Any; use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; @@ -192,10 +191,6 @@ impl ExecutionPlan for UnresolvedShuffleExec { "UnresolvedShuffleExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs index d53307339c..dd5ec26926 100644 --- a/ballista/core/src/extension.rs +++ b/ballista/core/src/extension.rs @@ -789,6 +789,22 @@ impl SessionConfigHelperExt for SessionConfig { // // See https://github.com/apache/datafusion-ballista/issues/1648 .set_bool("datafusion.optimizer.prefer_hash_join", false) + // + // DataFusion 54 plans uncorrelated scalar subqueries as a physical + // `ScalarSubqueryExec` wrapping a `ScalarSubqueryExpr` that reads an + // in-process shared results container. That container cannot cross + // process or stage boundaries, and `datafusion-proto` can only + // deserialize the expr inside its surrounding exec, so when Ballista + // splits a plan into stages the expr is serialized without its exec + // and the executor fails to decode it. Disabling this option makes + // the optimizer rewrite uncorrelated scalar subqueries to joins, + // which Ballista distributes correctly. + // + // See https://github.com/apache/datafusion-ballista/issues/1909 + .set_bool( + "datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery", + false, + ) } } @@ -1072,6 +1088,21 @@ mod test { ); } + // Uncorrelated scalar subqueries must be rewritten to joins rather than + // planned as a physical `ScalarSubqueryExec`, whose `ScalarSubqueryExpr` + // cannot be deserialized once Ballista splits the plan into stages. See + // #1909. + #[test] + fn should_disable_physical_uncorrelated_scalar_subquery() { + let config = SessionConfig::new().upgrade_for_ballista(); + assert!( + !config + .options() + .optimizer + .enable_physical_uncorrelated_scalar_subquery + ); + } + #[test] fn should_convert_to_key_value_pairs() { // key value pairs should contain datafusion and ballista values diff --git a/ballista/core/src/planner.rs b/ballista/core/src/planner.rs index e1ed75998c..ecf41733b8 100644 --- a/ballista/core/src/planner.rs +++ b/ballista/core/src/planner.rs @@ -307,12 +307,10 @@ mod test { assert!(matches!(analyze_df.logical_plan(), LogicalPlan::Analyze(_))); let explain = plan - .as_any() .downcast_ref::>() .unwrap(); assert!( explain.children()[0] - .as_any() .downcast_ref::>() .is_some() ); diff --git a/ballista/core/src/registry.rs b/ballista/core/src/registry.rs index 1d45e7b58a..8ec1d97539 100644 --- a/ballista/core/src/registry.rs +++ b/ballista/core/src/registry.rs @@ -19,7 +19,9 @@ use datafusion::common::DataFusionError; use datafusion::execution::{FunctionRegistry, SessionState}; use datafusion::functions::all_default_functions; use datafusion::functions_aggregate::all_default_aggregate_functions; +use datafusion::functions_nested::all_default_higher_order_functions; use datafusion::functions_window::all_default_window_functions; +use datafusion::logical_expr::HigherOrderUDF; use datafusion::logical_expr::planner::ExprPlanner; use datafusion::logical_expr::{AggregateUDF, ScalarUDF, WindowUDF}; use std::collections::{HashMap, HashSet}; @@ -41,6 +43,8 @@ pub struct BallistaFunctionRegistry { pub aggregate_functions: HashMap>, /// Window user-defined functions. pub window_functions: HashMap>, + /// Higher-order (lambda) user-defined functions. + pub higher_order_functions: HashMap>, } impl Default for BallistaFunctionRegistry { @@ -62,6 +66,12 @@ impl Default for BallistaFunctionRegistry { .map(|f| (f.name().to_string(), f)) .collect(); + let higher_order_functions: HashMap> = + all_default_higher_order_functions() + .into_iter() + .map(|f| (f.name().to_string(), f)) + .collect(); + #[cfg(feature = "spark-compat")] let (scalar_functions, aggregate_functions, window_functions) = { let mut scalar_functions = scalar_functions; @@ -85,6 +95,7 @@ impl Default for BallistaFunctionRegistry { scalar_functions, aggregate_functions, window_functions, + higher_order_functions, } } } @@ -106,6 +117,21 @@ impl FunctionRegistry for BallistaFunctionRegistry { self.window_functions.keys().cloned().collect() } + fn higher_order_function_names(&self) -> HashSet { + self.higher_order_functions.keys().cloned().collect() + } + + fn higher_order_function( + &self, + name: &str, + ) -> datafusion::common::Result> { + self.higher_order_functions.get(name).cloned().ok_or_else(|| { + DataFusionError::Internal(format!( + "There is no higher-order function named \"{name}\" in the TaskContext" + )) + }) + } + fn udf(&self, name: &str) -> datafusion::common::Result> { let result = self.scalar_functions.get(name); @@ -142,11 +168,13 @@ impl From<&SessionState> for BallistaFunctionRegistry { let scalar_functions = state.scalar_functions().clone(); let aggregate_functions = state.aggregate_functions().clone(); let window_functions = state.window_functions().clone(); + let higher_order_functions = state.higher_order_functions().clone(); Self { scalar_functions, aggregate_functions, window_functions, + higher_order_functions, } } } @@ -164,6 +192,26 @@ mod tests { assert!(registry.udfs().contains("abs")); } + #[test] + fn test_higher_order_functions_available() { + let registry = BallistaFunctionRegistry::default(); + + // DataFusion ships built-in higher-order functions (e.g. array_filter); + // the registry the executor runs with must carry them or those queries + // fail to plan on the executor. + let names = registry.higher_order_function_names(); + assert!( + !names.is_empty(), + "registry must expose DataFusion's built-in higher-order functions" + ); + for name in &names { + assert!( + registry.higher_order_function(name).is_ok(), + "named higher-order function {name} must be retrievable" + ); + } + } + #[test] #[cfg(not(feature = "spark-compat"))] fn test_spark_functions_unavailable_without_feature() { diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index 46e3430c38..0206afe5e8 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -36,6 +36,7 @@ use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; use datafusion_proto::physical_plan::to_proto::serialize_partitioning; use datafusion_proto::physical_plan::{ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, + PhysicalPlanDecodeContext, }; use datafusion_proto::protobuf::proto_error; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; @@ -381,15 +382,15 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { ) })?; let converter = DefaultPhysicalProtoConverter {}; + let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self); match ballista_plan { PhysicalPlanType::ShuffleWriter(shuffle_writer) => { let input = inputs[0].clone(); let shuffle_output_partitioning = parse_protobuf_hash_partitioning( shuffle_writer.output_partitioning.as_ref(), - ctx, + &decode_ctx, input.schema().as_ref(), - self.default_codec.as_ref(), &converter, )?; @@ -406,9 +407,8 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { let shuffle_output_partitioning = parse_protobuf_hash_partitioning( sort_shuffle_writer.output_partitioning.as_ref(), - ctx, + &decode_ctx, input.schema().as_ref(), - self.default_codec.as_ref(), &converter, )?; @@ -460,9 +460,8 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { .collect::, DataFusionError>>()?; let partitioning = parse_protobuf_partitioning( shuffle_reader.partitioning.as_ref(), - ctx, + &decode_ctx, schema.as_ref(), - self.default_codec.as_ref(), &converter, )?; let partitioning = partitioning @@ -503,9 +502,8 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { Arc::new(convert_required!(unresolved_shuffle.schema)?); let partitioning = parse_protobuf_partitioning( unresolved_shuffle.partitioning.as_ref(), - ctx, + &decode_ctx, schema.as_ref(), - self.default_codec.as_ref(), &converter, )?; let partitioning = partitioning @@ -557,7 +555,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { node: Arc, buf: &mut Vec, ) -> Result<(), DataFusionError> { - if let Some(exec) = node.as_any().downcast_ref::() { + if let Some(exec) = node.downcast_ref::() { // note that we use shuffle_output_partitioning() rather than output_partitioning() // to get the true output partitioning let output_partitioning = match exec.shuffle_output_partitioning() { @@ -596,7 +594,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { })?; Ok(()) - } else if let Some(exec) = node.as_any().downcast_ref::() { + } else if let Some(exec) = node.downcast_ref::() { let output_partitioning = match exec.shuffle_output_partitioning() { Partitioning::Hash(exprs, partition_count) => { Some(datafusion_proto::protobuf::PhysicalHashRepartition { @@ -639,7 +637,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { })?; Ok(()) - } else if let Some(exec) = node.as_any().downcast_ref::() { + } else if let Some(exec) = node.downcast_ref::() { let stage_id = exec.stage_id as u32; let mut partition = vec![]; for location in &exec.partition { @@ -682,7 +680,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { })?; Ok(()) - } else if let Some(exec) = node.as_any().downcast_ref::() { + } else if let Some(exec) = node.downcast_ref::() { let converter = DefaultPhysicalProtoConverter {}; let partitioning = serialize_partitioning( &exec.properties().partitioning, @@ -708,7 +706,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { })?; Ok(()) - } else if let Some(exec) = node.as_any().downcast_ref::() { + } else if let Some(exec) = node.downcast_ref::() { let proto = protobuf::BallistaPhysicalPlanNode { physical_plan_type: Some(PhysicalPlanType::ChaosExec( protobuf::ChaosExecNode { @@ -812,6 +810,55 @@ mod test { ])) } + // Regression coverage for #1838 and the removed `make_filter_projection_serde_safe` + // workaround: a `FilterExec` that projects to zero columns must survive physical + // plan serialization with its empty projection intact. datafusion-proto 53.1.0 + // could not distinguish `Some(vec![])` (empty projection) from `None` (full + // projection) and decoded the former back as `None`, shifting column indices. + // DataFusion 54 preserves the distinction, so no Ballista-side rewrite is needed. + #[tokio::test] + async fn filter_exec_empty_projection_survives_physical_serde() { + use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::expressions::lit; + use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder}; + use datafusion_proto::physical_plan::{ + AsExecutionPlan, DefaultPhysicalExtensionCodec, + }; + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let input: Arc = Arc::new(EmptyExec::new(schema)); + let filter = FilterExecBuilder::new(lit(true), input) + .apply_projection(Some(vec![])) + .unwrap() + .build() + .unwrap(); + let plan: Arc = Arc::new(filter); + assert_eq!( + plan.schema().fields().len(), + 0, + "precondition: filter projects to zero columns" + ); + + let codec = DefaultPhysicalExtensionCodec {}; + let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec).unwrap(); + let ctx = SessionContext::new().task_ctx(); + let decoded = proto.try_into_physical_plan(&ctx, &codec).unwrap(); + + assert_eq!( + decoded.schema().fields().len(), + 0, + "decoded FilterExec must still project zero columns" + ); + let filter = decoded + .downcast_ref::() + .expect("decoded plan must be a FilterExec"); + assert_eq!( + filter.projection().as_ref().map(|p| p.is_empty()), + Some(true), + "empty projection must round-trip as Some(vec![]), not None" + ); + } + #[tokio::test] async fn test_unresolved_shuffle_exec_roundtrip() { let schema = create_test_schema(); @@ -834,7 +881,6 @@ mod test { let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected UnresolvedShuffleExec"); @@ -871,7 +917,6 @@ mod test { let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -914,7 +959,6 @@ mod test { let ctx = SessionContext::new().task_ctx(); let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -968,7 +1012,6 @@ mod test { let ctx = SessionContext::new().task_ctx(); let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -1014,7 +1057,6 @@ mod test { let ctx = SessionContext::new().task_ctx(); let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected UnresolvedShuffleExec"); @@ -1065,7 +1107,6 @@ mod test { let ctx = SessionContext::new().task_ctx(); let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected ShuffleReaderExec"); @@ -1163,7 +1204,6 @@ mod test { let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected UnresolvedShuffleExec"); @@ -1190,7 +1230,6 @@ mod test { let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap(); let decoded_exec = decoded_plan - .as_any() .downcast_ref::() .expect("Expected ShuffleReaderExec"); diff --git a/ballista/core/src/serde/scheduler/from_proto.rs b/ballista/core/src/serde/scheduler/from_proto.rs index ba8b9b33a0..e6675194a8 100644 --- a/ballista/core/src/serde/scheduler/from_proto.rs +++ b/ballista/core/src/serde/scheduler/from_proto.rs @@ -19,7 +19,7 @@ use chrono::{TimeZone, Utc}; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::execution::TaskContext; -use datafusion::logical_expr::{AggregateUDF, ScalarUDF, WindowUDF}; +use datafusion::logical_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF}; use datafusion::physical_plan::metrics::{ Count, Gauge, MetricValue, MetricsSet, PruningMetrics, RatioMetrics, Time, Timestamp, }; @@ -338,6 +338,7 @@ impl Into for protobuf::ExecutorData { /// /// This function deserializes the execution plan from the protobuf representation /// and constructs a complete task definition with the provided runtime configuration. +#[allow(clippy::too_many_arguments)] pub fn get_task_definition( task: protobuf::TaskDefinition, produce_runtime: RuntimeProducer, @@ -345,6 +346,7 @@ pub fn get_task_definition>, aggregate_functions: HashMap>, window_functions: HashMap>, + higher_order_functions: HashMap>, codec: BallistaCodec, ) -> Result { let session_config = session_config.update_from_key_value_pair(&task.props); @@ -354,6 +356,7 @@ pub fn get_task_definition>, aggregate_functions: HashMap>, window_functions: HashMap>, + higher_order_functions: HashMap>, codec: BallistaCodec, ) -> Result, BallistaError> { let session_config = session_config.update_from_key_value_pair(&multi_task.props); @@ -418,6 +424,7 @@ pub fn get_task_definition_vec< scalar_functions: scalar_functions.clone(), aggregate_functions: aggregate_functions.clone(), window_functions: window_functions.clone(), + higher_order_functions: higher_order_functions.clone(), }); let ctx = TaskContext::new( @@ -425,6 +432,7 @@ pub fn get_task_definition_vec< uuid::Uuid::new_v4().to_string(), session_config.clone(), scalar_functions, + higher_order_functions, aggregate_functions, window_functions, runtime.clone(), diff --git a/ballista/executor/src/collect.rs b/ballista/executor/src/collect.rs index 40fae6cc58..31d2bc28ca 100644 --- a/ballista/executor/src/collect.rs +++ b/ballista/executor/src/collect.rs @@ -18,9 +18,9 @@ //! The CollectExec operator retrieves results from the cluster and returns them as a single //! vector of [`RecordBatch`](datafusion::arrow::record_batch::RecordBatch). +use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use std::{any::Any, pin::Pin}; use datafusion::arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion::error::DataFusionError; @@ -79,10 +79,6 @@ impl ExecutionPlan for CollectExec { "CollectExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.plan.schema() } @@ -125,7 +121,7 @@ impl ExecutionPlan for CollectExec { })) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.plan.partition_statistics(partition) } } diff --git a/ballista/executor/src/execution_engine.rs b/ballista/executor/src/execution_engine.rs index 6ce96fc8be..e579621c9e 100644 --- a/ballista/executor/src/execution_engine.rs +++ b/ballista/executor/src/execution_engine.rs @@ -27,11 +27,18 @@ use ballista_core::execution_plans::{ShuffleReaderExec, ShuffleWriterExec}; use ballista_core::serde::protobuf::ShuffleWritePartition; use ballista_core::{JobId, utils}; use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::datasource::physical_plan::{ + FileGroup, FileScanConfig, FileScanConfigBuilder, +}; +use datafusion::datasource::source::DataSourceExec; use datafusion::error::{DataFusionError, Result}; use datafusion::execution::context::TaskContext; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::metrics::MetricsSet; use datafusion::prelude::SessionConfig; +use log::warn; +use std::any::Any; use std::fmt::{Debug, Display}; use std::sync::Arc; @@ -101,19 +108,76 @@ impl DefaultExecutionEngine { } } +/// Restrict a `DataSourceExec` to the file group for `partition_id`. +/// +/// DataFusion 54's `DataSourceExec` hands file groups to partition streams from +/// a shared work-queue that is only divided across partitions when all +/// partitions of one plan instance are polled concurrently. Ballista runs one +/// partition per task on its own plan instance, so a task that polls a single +/// partition in isolation would otherwise drain the whole queue and scan the +/// entire table. Keeping only this task's file group (other slots emptied, +/// partition count preserved) makes the lone `execute(partition_id)` read just +/// that group. +/// +/// Returns `None` for any node that is not a file-backed `DataSourceExec`, and +/// for a `partition_id` outside the source's file groups (e.g. when an operator +/// between the scan and the stage output changed the partition count). +/// +/// Only `FileScanConfig` distributes work from the shared queue. Other data +/// sources (e.g. `MemorySourceConfig`) isolate partitions in their own +/// `open(partition)` and need no restriction, so they are left unchanged. An +/// unrecognized source type is warned about, since a future source that +/// distributes work across partitions would over-read here without handling. +fn restrict_scan_to_partition( + plan: &Arc, + partition_id: usize, +) -> Option> { + let exec = plan.downcast_ref::()?; + let source: &dyn Any = exec.data_source().as_ref(); + let Some(config) = source.downcast_ref::() else { + if source.downcast_ref::().is_none() { + warn!( + "restrict_scan_to_partition: unrecognized DataSourceExec source type \ + left unrestricted; if it distributes work across partitions from a \ + shared queue, a single-partition task could over-read" + ); + } + return None; + }; + if partition_id >= config.file_groups.len() { + return None; + } + let file_groups: Vec = config + .file_groups + .iter() + .enumerate() + .map(|(i, group)| { + if i == partition_id { + group.clone() + } else { + FileGroup::new(vec![]) + } + }) + .collect(); + let config = FileScanConfigBuilder::from(config.clone()) + .with_file_groups(file_groups) + .build(); + Some(DataSourceExec::from_data_source(config)) +} + impl ExecutionEngine for DefaultExecutionEngine { fn create_query_stage_exec( &self, job_id: JobId, stage_id: usize, - _partition_id: usize, + partition_id: usize, plan: Arc, work_dir: &str, _config: &SessionConfig, ) -> Result> { let plan = plan .transform(|p| { - if let Some(reader) = p.as_any().downcast_ref::() { + if let Some(reader) = p.downcast_ref::() { match &self.client_pool { Some(client_pool) => Ok(Transformed::yes(Arc::new( reader @@ -124,6 +188,10 @@ impl ExecutionEngine for DefaultExecutionEngine { reader.with_work_dir(work_dir.to_string()), ))), } + } else if let Some(rewritten) = + restrict_scan_to_partition(&p, partition_id) + { + Ok(Transformed::yes(rewritten)) } else { Ok(Transformed::no(p)) } @@ -132,7 +200,7 @@ impl ExecutionEngine for DefaultExecutionEngine { // the query plan created by the scheduler always starts with a shuffle writer // (either ShuffleWriterExec or SortShuffleWriterExec) - if let Some(shuffle_writer) = plan.as_any().downcast_ref::() { + if let Some(shuffle_writer) = plan.downcast_ref::() { // recreate the shuffle writer with the correct working directory let exec = ShuffleWriterExec::try_new( job_id, @@ -145,7 +213,7 @@ impl ExecutionEngine for DefaultExecutionEngine { ShuffleWriterVariant::Hash(exec), ))) } else if let Some(sort_shuffle_writer) = - plan.as_any().downcast_ref::() + plan.downcast_ref::() { // recreate the sort shuffle writer with the correct working directory let exec = SortShuffleWriterExec::try_new( @@ -259,3 +327,60 @@ impl QueryStageExecutor for DefaultQueryStageExec { } } } + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::datasource::listing::PartitionedFile; + use datafusion::datasource::physical_plan::ParquetSource; + use datafusion::execution::object_store::ObjectStoreUrl; + use datafusion::physical_plan::empty::EmptyExec; + + /// Build a `DataSourceExec` over `n` file groups, one file each. + fn scan_with_file_groups(n: usize) -> Arc { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + let source = Arc::new(ParquetSource::new(schema)); + let mut builder = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source); + for i in 0..n { + builder = + builder.with_file_group(FileGroup::new(vec![PartitionedFile::new( + format!("file{i}.parquet"), + 100, + )])); + } + DataSourceExec::from_data_source(builder.build()) + } + + /// Number of files in each file group of a `DataSourceExec`. + fn group_file_counts(plan: &Arc) -> Vec { + let exec = plan.downcast_ref::().unwrap(); + let source: &dyn Any = exec.data_source().as_ref(); + let config = source.downcast_ref::().unwrap(); + config.file_groups.iter().map(|g| g.len()).collect() + } + + #[test] + fn restrict_scan_keeps_only_its_own_group() { + let plan = scan_with_file_groups(4); + // partition 2 keeps only group 2; the partition count is preserved. + let restricted = restrict_scan_to_partition(&plan, 2).expect("scan rewritten"); + assert_eq!(group_file_counts(&restricted), vec![0, 0, 1, 0]); + } + + #[test] + fn restrict_scan_partition_out_of_range_is_left_untouched() { + let plan = scan_with_file_groups(3); + // an operator between the scan and the stage output may change the + // partition count; in that case we must not rewrite the scan. + assert!(restrict_scan_to_partition(&plan, 3).is_none()); + } + + #[test] + fn restrict_scan_ignores_non_file_scans() { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + let plan: Arc = Arc::new(EmptyExec::new(schema)); + assert!(restrict_scan_to_partition(&plan, 0).is_none()); + } +} diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index f64fae3014..a8b08efd7e 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -256,6 +256,8 @@ async fn run_received_task &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::new(Schema::empty()) } diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs index 6ef9fa5e4e..3900a0f384 100644 --- a/ballista/executor/src/executor_server.rs +++ b/ballista/executor/src/executor_server.rs @@ -398,6 +398,7 @@ impl ExecutorServer ExecutorGrpc self.executor.function_registry.scalar_functions.clone(), self.executor.function_registry.aggregate_functions.clone(), self.executor.function_registry.window_functions.clone(), + self.executor + .function_registry + .higher_order_functions + .clone(), self.codec.clone(), ) .map_err(|e| Status::invalid_argument(format!("{e}")))?, @@ -853,6 +858,10 @@ impl ExecutorGrpc self.executor.function_registry.scalar_functions.clone(), self.executor.function_registry.aggregate_functions.clone(), self.executor.function_registry.window_functions.clone(), + self.executor + .function_registry + .higher_order_functions + .clone(), self.codec.clone(), ) .map_err(|e| Status::invalid_argument(format!("{e}")))?; diff --git a/ballista/scheduler/src/physical_optimizer/join_selection.rs b/ballista/scheduler/src/physical_optimizer/join_selection.rs index b50b0f3c21..7410ec0270 100644 --- a/ballista/scheduler/src/physical_optimizer/join_selection.rs +++ b/ballista/scheduler/src/physical_optimizer/join_selection.rs @@ -323,66 +323,65 @@ fn statistical_join_selection_subrule( collect_threshold_byte_size: usize, collect_threshold_num_rows: usize, ) -> Result>> { - let transformed = - if let Some(hash_join) = plan.as_any().downcast_ref::() { - match hash_join.partition_mode() { - PartitionMode::Auto => try_collect_left( - hash_join, - false, - collect_threshold_byte_size, - collect_threshold_num_rows, - )? + let transformed = if let Some(hash_join) = plan.downcast_ref::() { + match hash_join.partition_mode() { + PartitionMode::Auto => try_collect_left( + hash_join, + false, + collect_threshold_byte_size, + collect_threshold_num_rows, + )? + .map_or_else( + || partitioned_hash_join(hash_join).map(Some), + |v| Ok(Some(v)), + )?, + PartitionMode::CollectLeft => try_collect_left(hash_join, true, 0, 0)? .map_or_else( || partitioned_hash_join(hash_join).map(Some), |v| Ok(Some(v)), )?, - PartitionMode::CollectLeft => try_collect_left(hash_join, true, 0, 0)? - .map_or_else( - || partitioned_hash_join(hash_join).map(Some), - |v| Ok(Some(v)), - )?, - PartitionMode::Partitioned => { - let left = hash_join.left(); - let right = hash_join.right(); - if hash_join.join_type().supports_swap() - && should_swap_join_order(&**left, &**right)? - { - hash_join - .swap_inputs(PartitionMode::Partitioned) - .map(Some)? - } else { - None - } + PartitionMode::Partitioned => { + let left = hash_join.left(); + let right = hash_join.right(); + if hash_join.join_type().supports_swap() + && should_swap_join_order(&**left, &**right)? + { + hash_join + .swap_inputs(PartitionMode::Partitioned) + .map(Some)? + } else { + None } } - } else if let Some(cross_join) = plan.as_any().downcast_ref::() { - let left = cross_join.left(); - let right = cross_join.right(); - if right.properties().output_partitioning().partition_count() > 1 { - None - } else if should_swap_join_order(&**left, &**right)? { - cross_join.swap_inputs().map(Some)? - } else { - None - } - } else if let Some(nl_join) = plan.as_any().downcast_ref::() { - let left = nl_join.left(); - let right = nl_join.right(); - // next few lines are different from original datafusion rule - // partition count of right side has to be equal one to be - // able to swap inputs - if right.properties().output_partitioning().partition_count() > 1 { - None - } else if nl_join.join_type().supports_swap() - && should_swap_join_order(&**left, &**right)? - { - nl_join.swap_inputs().map(Some)? - } else { - None - } + } + } else if let Some(cross_join) = plan.downcast_ref::() { + let left = cross_join.left(); + let right = cross_join.right(); + if right.properties().output_partitioning().partition_count() > 1 { + None + } else if should_swap_join_order(&**left, &**right)? { + cross_join.swap_inputs().map(Some)? } else { None - }; + } + } else if let Some(nl_join) = plan.downcast_ref::() { + let left = nl_join.left(); + let right = nl_join.right(); + // next few lines are different from original datafusion rule + // partition count of right side has to be equal one to be + // able to swap inputs + if right.properties().output_partitioning().partition_count() > 1 { + None + } else if nl_join.join_type().supports_swap() + && should_swap_join_order(&**left, &**right)? + { + nl_join.swap_inputs().map(Some)? + } else { + None + } + } else { + None + }; Ok(if let Some(transformed) = transformed { Transformed::yes(transformed) @@ -416,7 +415,7 @@ fn hash_join_convert_symmetric_subrule( config_options: &ConfigOptions, ) -> Result> { // Check if the current plan node is a HashJoinExec. - if let Some(hash_join) = input.as_any().downcast_ref::() { + if let Some(hash_join) = input.downcast_ref::() { let left_unbounded = hash_join.left.boundedness().is_unbounded(); let left_incremental = matches!( hash_join.left.pipeline_behavior(), @@ -556,7 +555,7 @@ pub fn hash_join_swap_subrule( mut input: Arc, _config_options: &ConfigOptions, ) -> Result> { - if let Some(hash_join) = input.as_any().downcast_ref::() + if let Some(hash_join) = input.downcast_ref::() && hash_join.left.boundedness().is_unbounded() && !hash_join.right.boundedness().is_unbounded() && matches!( @@ -830,7 +829,7 @@ mod test { // `swap_inputs` for Inner wraps the join in a ProjectionExec to // restore the output column order. Walk the tree to find the join. fn find_hash_join(plan: &Arc) -> Option<&HashJoinExec> { - if let Some(hj) = plan.as_any().downcast_ref::() { + if let Some(hj) = plan.downcast_ref::() { return Some(hj); } for child in plan.children() { diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index ed396719f4..95fcc6c9b0 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -33,15 +33,11 @@ use ballista_core::{ serde::scheduler::PartitionLocation, }; use datafusion::arrow::datatypes::DataType; -use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::config::ConfigOptions; -use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_optimizer::enforce_sorting::EnforceSorting; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder}; use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; -use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::{ @@ -111,10 +107,8 @@ impl DistributedPlanner for DefaultDistributedPlanner { config: &ConfigOptions, ) -> Result>> { debug!("Planning query stages for job: [{job_id}]"); - // Workaround until DF 54 migration : - let serde_safe_plan = make_filter_projection_serde_safe(execution_plan)?; let (new_plan, mut stages) = - self.plan_query_stages_internal(job_id, serde_safe_plan, config)?; + self.plan_query_stages_internal(job_id, execution_plan, config)?; stages.push(create_shuffle_writer_with_config( job_id, self.next_stage_id(), @@ -146,15 +140,13 @@ impl DefaultDistributedPlanner { // Broadcast-join lowering: HashJoinExec(CollectLeft) gets its own // controlled recursion so the build side is written as a broadcast stage. - if let Some(hash_join) = execution_plan.as_any().downcast_ref::() + if let Some(hash_join) = execution_plan.downcast_ref::() && *hash_join.partition_mode() == PartitionMode::CollectLeft { // Build subtree: peel CoalescePartitionsExec if present, then // recurse to lower its internal stages. let mut build = hash_join.left().clone(); - if let Some(coalesce) = - build.as_any().downcast_ref::() - { + if let Some(coalesce) = build.downcast_ref::() { build = coalesce.children()[0].clone(); } let (build, mut stages) = @@ -198,10 +190,7 @@ impl DefaultDistributedPlanner { stages.append(&mut child_stages); } - if let Some(_coalesce) = execution_plan - .as_any() - .downcast_ref::() - { + if let Some(_coalesce) = execution_plan.downcast_ref::() { let input = children[0].clone(); let input = self.optimizer_enforce_sorting.optimize(input, config)?; let shuffle_writer = create_shuffle_writer_with_config( @@ -218,10 +207,9 @@ impl DefaultDistributedPlanner { with_new_children_if_necessary(execution_plan, vec![unresolved_shuffle])?, stages, )) - } else if let Some(_sort_preserving_merge) = execution_plan - .as_any() - .downcast_ref::( - ) { + } else if let Some(_sort_preserving_merge) = + execution_plan.downcast_ref::() + { let shuffle_writer = create_shuffle_writer_with_config( job_id, self.next_stage_id(), @@ -235,9 +223,7 @@ impl DefaultDistributedPlanner { with_new_children_if_necessary(execution_plan, vec![unresolved_shuffle])?, stages, )) - } else if let Some(repart) = - execution_plan.as_any().downcast_ref::() - { + } else if let Some(repart) = execution_plan.downcast_ref::() { match repart.properties().output_partitioning() { Partitioning::Hash(_, _) => { let input = children[0].clone(); @@ -295,7 +281,7 @@ impl DefaultDistributedPlanner { debug!("broadcast check: threshold is 0, broadcast disabled"); return Ok(plan); } - let Some(hash_join) = plan.as_any().downcast_ref::() else { + let Some(hash_join) = plan.downcast_ref::() else { return Ok(plan); }; debug!( @@ -398,7 +384,6 @@ impl DefaultDistributedPlanner { }; let promoted_join = promoted - .as_any() .downcast_ref::() .expect("promoted plan must still be a HashJoinExec"); let new_left: Arc = if promoted_join @@ -420,40 +405,6 @@ impl DefaultDistributedPlanner { } } -/// Workaround until DF 54 migration: -/// datafusion-proto 53.1.0 encodes both `Some([])` (zero cols) and `None` (all cols) -/// as an empty list, decoding back to `None` and shifting column indices (#1838). -/// Rewrite `FilterExec(Some([]))` → empty `ProjectionExec` wrapped `FilterExec(None)` -fn make_filter_projection_serde_safe( - plan: Arc, -) -> datafusion::error::Result> { - Ok(plan - .transform_up(|node| { - if let Some(filter) = node.as_any().downcast_ref::() { - let empty_projection = filter - .projection() - .as_ref() - .map(|p| p.is_empty()) - .unwrap_or(false); - if empty_projection { - let orig_filter_exec = FilterExecBuilder::from(filter) - .apply_projection(None)? - .build()?; - let zero_expr: Vec<(Arc, String)> = vec![]; - let proj = ProjectionExec::try_new( - zero_expr, - Arc::new(orig_filter_exec) as Arc, - )?; - return Ok( - Transformed::yes(Arc::new(proj) as Arc), - ); - } - } - Ok(Transformed::no(node)) - })? - .data) -} - fn create_unresolved_shuffle( shuffle_writer: &dyn ShuffleWriter, ) -> Arc { @@ -470,9 +421,7 @@ fn create_unresolved_shuffle( pub fn find_unresolved_shuffles( plan: &Arc, ) -> Result> { - if let Some(unresolved_shuffle) = - plan.as_any().downcast_ref::() - { + if let Some(unresolved_shuffle) = plan.downcast_ref::() { Ok(vec![unresolved_shuffle.clone()]) } else { Ok(plan @@ -495,9 +444,7 @@ pub fn remove_unresolved_shuffles( ) -> Result> { let mut new_children: Vec> = vec![]; for child in stage.children() { - if let Some(unresolved_shuffle) = - child.as_any().downcast_ref::() - { + if let Some(unresolved_shuffle) = child.downcast_ref::() { let p = partition_locations .get(&unresolved_shuffle.stage_id) .ok_or_else(|| { @@ -558,7 +505,7 @@ pub fn rollback_resolved_shuffles( ) -> Result> { let mut new_children: Vec> = vec![]; for child in stage.children() { - if let Some(shuffle_reader) = child.as_any().downcast_ref::() { + if let Some(shuffle_reader) = child.downcast_ref::() { let stage_id = shuffle_reader.stage_id; let unresolved = if shuffle_reader.broadcast { Arc::new(UnresolvedShuffleExec::new_broadcast( @@ -636,15 +583,13 @@ mod test { use ballista_core::execution_plans::{SortShuffleWriterExec, UnresolvedShuffleExec}; use ballista_core::serde::BallistaCodec; use datafusion::arrow::compute::SortOptions; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::execution::TaskContext; use datafusion::physical_expr::expressions::Column; use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode}; - use datafusion::physical_plan::empty::EmptyExec; - use datafusion::physical_plan::expressions::lit; + use datafusion::physical_plan::filter::FilterExec; - use datafusion::physical_plan::filter::FilterExecBuilder; + use datafusion::physical_plan::joins::HashJoinExec; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::sorts::sort::SortExec; @@ -660,68 +605,16 @@ mod test { macro_rules! downcast_exec { ($exec: expr, $ty: ty) => { - $exec.as_any().downcast_ref::<$ty>().expect(&format!( - "Downcast to {} failed. Got {:?}", - stringify!($ty), - $exec - )) + ($exec.as_ref() as &dyn ExecutionPlan) + .downcast_ref::<$ty>() + .expect(&format!( + "Downcast to {} failed. Got {:?}", + stringify!($ty), + $exec + )) }; } - /// Regression test for issue #1838: a `FilterExec` projecting to zero columns - /// is not serialization-safe in datafusion-proto 53.1.0 (the empty projection - /// decodes back as `None`, shifting downstream column indices). The planner - /// must rewrite it into a serde-safe equivalent that still emits zero columns. - #[test] - fn empty_projection_filter_is_rewritten_serde_safe() -> Result<(), BallistaError> { - // 1-column input feeding a FilterExec that projects to ZERO columns - // (mirrors TPC-DS Q9's `FROM reason WHERE r_reason_sk = 1`). - let schema = Arc::new(Schema::new(vec![Field::new( - "r_reason_sk", - DataType::Int32, - false, - )])); - let input: Arc = Arc::new(EmptyExec::new(schema)); - let filter = FilterExecBuilder::new(lit(true), input) - .apply_projection(Some(vec![]))? - .build()?; - let plan: Arc = Arc::new(filter); - assert_eq!( - plan.schema().fields().len(), - 0, - "precondition: filter projects to zero columns" - ); - - let serde_safe_plan = super::make_filter_projection_serde_safe(plan)?; - - // schema is preserved - assert_eq!( - serde_safe_plan.schema().fields().len(), - 0, - "rewrite must preserve the zero-column output schema" - ); - - // FilterExec does not retain an empty projection - let mut has_empty_filter_projection = false; - serde_safe_plan.apply(|node| { - if let Some(f) = node.as_any().downcast_ref::() - && f.projection() - .as_ref() - .map(|p| p.is_empty()) - .unwrap_or(false) - { - has_empty_filter_projection = true; - } - Ok(TreeNodeRecursion::Continue) - })?; - assert!( - !has_empty_filter_projection, - "FilterExec with empty projection must be rewritten to be serde-safe" - ); - - Ok(()) - } - #[tokio::test] async fn distributed_aggregate_plan() -> Result<(), BallistaError> { let ctx = datafusion_test_context("testdata").await?; @@ -1017,11 +910,10 @@ order by let mut walker: Vec> = vec![stage.clone() as Arc]; while let Some(node) = walker.pop() { - if let Some(hj) = node.as_any().downcast_ref::() { + if let Some(hj) = node.downcast_ref::() { assert_eq!(*hj.partition_mode(), PartitionMode::CollectLeft); let left = hj.children()[0].clone(); let unresolved = left - .as_any() .downcast_ref::() .expect("left input should be UnresolvedShuffleExec"); assert!(unresolved.broadcast, "left input should be broadcast"); @@ -1062,15 +954,13 @@ order by let mut walker: Vec> = vec![stage.clone() as Arc]; while let Some(node) = walker.pop() { - if let Some(unresolved) = - node.as_any().downcast_ref::() - { + if let Some(unresolved) = node.downcast_ref::() { assert!( !unresolved.broadcast, "no broadcast reader expected with threshold=0" ); } - if let Some(hj) = node.as_any().downcast_ref::() { + if let Some(hj) = node.downcast_ref::() { assert_ne!( *hj.partition_mode(), PartitionMode::CollectLeft, @@ -1113,8 +1003,7 @@ order by let mut walker: Vec> = vec![stage.clone() as Arc]; while let Some(node) = walker.pop() { - if let Some(unresolved) = - node.as_any().downcast_ref::() + if let Some(unresolved) = node.downcast_ref::() && unresolved.broadcast { max_upstream = max_upstream.max(unresolved.upstream_partition_count); @@ -1184,7 +1073,6 @@ order by let resolved_child = resolved.children()[0].clone(); let reader = resolved_child - .as_any() .downcast_ref::() .expect("expected resolved ShuffleReaderExec"); assert!(reader.broadcast); @@ -1214,7 +1102,6 @@ order by let rolled_back = crate::planner::rollback_resolved_shuffles(parent)?; let child = rolled_back.children()[0].clone(); let unresolved = child - .as_any() .downcast_ref::() .expect("expected rolled-back UnresolvedShuffleExec"); assert!(unresolved.broadcast); @@ -1290,7 +1177,7 @@ order by assert_eq!(2, partitioning.partition_count()); let partition_col = match partitioning { Partitioning::Hash(exprs, 2) => match exprs.as_slice() { - [col] => col.as_any().downcast_ref::(), + [col] => col.downcast_ref::(), _ => None, }, _ => None, @@ -1304,7 +1191,7 @@ order by let window = downcast_exec!(filter.children()[0], BoundedWindowAggExec); let partition_by = window.partition_keys(); let partition_by = match partition_by[..] { - [ref col] => col.as_any().downcast_ref::(), + [ref col] => col.downcast_ref::(), _ => None, }; assert_eq!(Some(&Column::new("l_shipmode", 1)), partition_by); @@ -1321,7 +1208,7 @@ order by ); assert_eq!( Some(&Column::new("l_shipmode", 1)), - expr1.expr.as_any().downcast_ref() + expr1.expr.downcast_ref() ); assert_eq!( SortOptions { @@ -1332,7 +1219,7 @@ order by ); assert_eq!( Some(&Column::new("l_shipdate", 0)), - expr2.expr.as_any().downcast_ref() + expr2.expr.downcast_ref() ); } _ => panic!("invalid sort {sort:?}"), diff --git a/ballista/scheduler/src/state/aqe/adapter.rs b/ballista/scheduler/src/state/aqe/adapter.rs index f751d81b97..2040982341 100644 --- a/ballista/scheduler/src/state/aqe/adapter.rs +++ b/ballista/scheduler/src/state/aqe/adapter.rs @@ -45,7 +45,7 @@ impl BallistaAdapter { &mut self, plan: Arc, ) -> datafusion::error::Result>> { - if let Some(exchange) = plan.as_any().downcast_ref::() { + if let Some(exchange) = plan.downcast_ref::() { let schema = exchange.schema().clone(); let partitions = exchange.shuffle_partitions().ok_or_else(|| { DataFusionError::Execution( @@ -119,7 +119,7 @@ impl BallistaAdapter { job_id: &JobId, config: &ConfigOptions, ) -> datafusion::error::Result { - if let Some(root) = plan.as_any().downcast_ref::() { + if let Some(root) = plan.downcast_ref::() { let mut adapter = BallistaAdapter::default(); let plan = root .input() @@ -146,8 +146,7 @@ impl BallistaAdapter { plan: writer, inputs: adapter.inputs, }) - } else if let Some(root) = plan.as_any().downcast_ref::() - { + } else if let Some(root) = plan.downcast_ref::() { let mut adapter = BallistaAdapter::default(); let plan = root .input() diff --git a/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs b/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs index 1a1c3962c4..9bc7451ae7 100644 --- a/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs +++ b/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs @@ -22,7 +22,6 @@ use datafusion::{ physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}, }; use parking_lot::Mutex; -use std::any::Any; use std::fmt::Formatter; use std::sync::atomic::AtomicBool; use std::sync::{Arc, atomic::AtomicI64}; @@ -126,10 +125,6 @@ impl ExecutionPlan for AdaptiveDatafusionExec { "AdaptiveDatafusionExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { self.input.properties() } diff --git a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs index 3c7853c26c..2bac00644b 100644 --- a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs +++ b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs @@ -65,10 +65,6 @@ impl ExecutionPlan for DynamicJoinSelectionExec { "DynamicJoinSelectionExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -383,16 +379,14 @@ impl DynamicJoinSelectionExec { /// its children. pub(crate) fn upstream_resolved(&self) -> bool { fn has_join_or_unresolved_exchange(plan: &Arc) -> bool { - if let Some(exchange) = plan.as_any().downcast_ref::() { + if let Some(exchange) = plan.downcast_ref::() { // this should be fine, once we find first resolved // exchange it should not have any unresolved shuffles later // nor dynamic joins return !exchange.shuffle_created(); }; - plan.as_any() - .downcast_ref::() - .is_some() + plan.downcast_ref::().is_some() || plan .children() .iter() diff --git a/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs b/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs index 28f1011e45..63cc181502 100644 --- a/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs +++ b/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs @@ -29,7 +29,6 @@ use datafusion::{ }; use log::trace; use parking_lot::Mutex; -use std::any::Any; use std::ops::Deref; use std::sync::{Arc, atomic::AtomicI64}; @@ -326,10 +325,6 @@ impl ExecutionPlan for ExchangeExec { "ExchangeExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -381,7 +376,7 @@ impl ExecutionPlan for ExchangeExec { )) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { let schema = self.input.schema(); match self.shuffle_partitions.lock().deref() { // @@ -406,7 +401,7 @@ impl ExecutionPlan for ExchangeExec { "shuffle reader at stage: {:?} and partition {} returned statistics: {:?}", self.stage_id, idx, stat_for_partition ); - stat_for_partition + stat_for_partition.map(Arc::new) } else { let stats_for_partitions = stats_for_partitions( schema.fields().len(), @@ -419,10 +414,10 @@ impl ExecutionPlan for ExchangeExec { "shuffle reader at stage: {:?} returned statistics for all partitions: {:?}", self.stage_id, stats_for_partitions ); - Ok(stats_for_partitions) + Ok(Arc::new(stats_for_partitions)) } } - None => Ok(Statistics::new_unknown(&schema)), + None => Ok(Arc::new(Statistics::new_unknown(&schema))), } } } diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs index a2b8f7b2e1..bef85f3fc4 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs @@ -177,7 +177,7 @@ mod tests { let mut count = 0usize; result .apply(|node| { - if node.as_any().downcast_ref::().is_some() { + if node.downcast_ref::().is_some() { count += 1; } Ok(TreeNodeRecursion::Continue) @@ -193,7 +193,6 @@ mod tests { let plan = leaf_exec(); let result = rule.optimize(plan, &chaos_config(0.42, 42)).unwrap(); let chaos = result - .as_any() .downcast_ref::() .expect("single-node plan always wraps root"); assert!( diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs index 4686fde0fd..5947be4c3d 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs @@ -155,7 +155,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule { ); // Get the subtree below the root. Two root kinds, same outcome. - let input = if let Some(ex) = plan.as_any().downcast_ref::() { + let input = if let Some(ex) = plan.downcast_ref::() { debug!( "[coalesce-rule] root=ExchangeExec plan_id={} stage_id={:?} stage_resolved={}", ex.plan_id, @@ -163,7 +163,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule { ex.shuffle_partitions().is_some(), ); ex.input().clone() - } else if let Some(adp) = plan.as_any().downcast_ref::() { + } else if let Some(adp) = plan.downcast_ref::() { debug!( "[coalesce-rule] root=AdaptiveDatafusionExec stage_id={:?}", adp.stage_id(), @@ -182,7 +182,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule { // *this* stage's group, they belong to whatever stage wrote them. let mut leaves: Vec> = Vec::new(); input.apply(|node| { - if node.as_any().is::() { + if node.is::() { leaves.push(node.clone()); Ok(TreeNodeRecursion::Jump) } else { @@ -192,8 +192,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule { // Helper: downcast each Arc back to &ExchangeExec. fn as_exchange(arc: &Arc) -> &ExchangeExec { - arc.as_any() - .downcast_ref::() + arc.downcast_ref::() .expect("filtered to ExchangeExec above") } diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs index 7b60e1e43d..a07312d14e 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs @@ -45,12 +45,9 @@ impl DistributedExchangeRule { &self, execution_plan: Arc, ) -> datafusion::error::Result>> { - if let Some(coalesce) = execution_plan - .as_any() - .downcast_ref::() - { + if let Some(coalesce) = execution_plan.downcast_ref::() { let input = coalesce.input(); - if input.as_any().downcast_ref::().is_none() + if input.downcast_ref::().is_none() && !matches!(nearest_exchange_status(input), ExchangeStatus::Unresolved) { let exchange_exec = ExchangeExec::new( @@ -63,12 +60,11 @@ impl DistributedExchangeRule { execution_plan.with_new_children(vec![Arc::new(exchange_exec)])?, )); } - } else if let Some(sort_preserving_merge) = execution_plan - .as_any() - .downcast_ref::( - ) { + } else if let Some(sort_preserving_merge) = + execution_plan.downcast_ref::() + { let input = sort_preserving_merge.input(); - if input.as_any().downcast_ref::().is_none() + if input.downcast_ref::().is_none() && !matches!(nearest_exchange_status(input), ExchangeStatus::Unresolved) { let exchange_exec = ExchangeExec::new( @@ -81,8 +77,7 @@ impl DistributedExchangeRule { execution_plan.with_new_children(vec![Arc::new(exchange_exec)])?, )); } - } else if let Some(repartition) = - execution_plan.as_any().downcast_ref::() + } else if let Some(repartition) = execution_plan.downcast_ref::() && let execution_plan::Partitioning::Hash(_, _) = repartition.partitioning() { let input = repartition.input(); @@ -112,7 +107,6 @@ impl PhysicalOptimizerRule for DistributedExchangeRule { if result .data - .as_any() .downcast_ref::() .is_some() { @@ -143,7 +137,7 @@ impl PhysicalOptimizerRule for DistributedExchangeRule { /// (short-circuits), `Resolved` if every branch that has an exchange has a resolved /// one, and `None` if no exchange is found anywhere. fn nearest_exchange_status(plan: &Arc) -> ExchangeStatus { - if let Some(exchange) = plan.as_any().downcast_ref::() { + if let Some(exchange) = plan.downcast_ref::() { if exchange.shuffle_created() && !exchange.inactive_stage { ExchangeStatus::Resolved } else { @@ -241,25 +235,18 @@ mod tests { let result = rule.optimize(coalesce, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let coalesce_out = adaptive .input() - .as_any() .downcast_ref::() .unwrap(); let child = coalesce_out.children()[0]; assert!( - child.as_any().downcast_ref::().is_some(), + child.downcast_ref::().is_some(), "direct child should remain ExchangeExec" ); assert!( - child.children()[0] - .as_any() - .downcast_ref::() - .is_none(), + child.children()[0].downcast_ref::().is_none(), "ExchangeExec should not wrap another ExchangeExec" ); } @@ -277,18 +264,13 @@ mod tests { let result = rule.optimize(outer, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let outer_coalesce = adaptive .input() - .as_any() .downcast_ref::() .unwrap(); assert!( outer_coalesce.children()[0] - .as_any() .downcast_ref::() .is_none(), "should not inject ExchangeExec when unresolved exchange is in subtree" @@ -307,18 +289,13 @@ mod tests { let result = rule.optimize(outer, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let outer_coalesce = adaptive .input() - .as_any() .downcast_ref::() .unwrap(); assert!( outer_coalesce.children()[0] - .as_any() .downcast_ref::() .is_some(), "should inject ExchangeExec when subtree only has resolved exchanges" @@ -334,20 +311,13 @@ mod tests { let result = rule.optimize(input, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let spm = adaptive .input() - .as_any() .downcast_ref::() .expect("child should be SortPreservingMergeExec"); assert!( - spm.children()[0] - .as_any() - .downcast_ref::() - .is_some(), + spm.children()[0].downcast_ref::().is_some(), "SortPreservingMergeExec should have ExchangeExec injected as its child" ); } @@ -359,23 +329,14 @@ mod tests { let result = rule.optimize(input, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let spm = adaptive .input() - .as_any() .downcast_ref::() .unwrap(); let child = spm.children()[0]; - assert!(child.as_any().downcast_ref::().is_some()); - assert!( - child.children()[0] - .as_any() - .downcast_ref::() - .is_none() - ); + assert!(child.downcast_ref::().is_some()); + assert!(child.children()[0].downcast_ref::().is_none()); } #[test] @@ -388,20 +349,13 @@ mod tests { let result = rule.optimize(input, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let spm = adaptive .input() - .as_any() .downcast_ref::() .unwrap(); assert!( - spm.children()[0] - .as_any() - .downcast_ref::() - .is_none(), + spm.children()[0].downcast_ref::().is_none(), "should not inject ExchangeExec when unresolved exchange is in subtree" ); } @@ -419,13 +373,9 @@ mod tests { let result = rule.optimize(repartition, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); let exchange = adaptive .input() - .as_any() .downcast_ref::() .expect("Hash RepartitionExec should be replaced with ExchangeExec"); assert!( @@ -444,16 +394,9 @@ mod tests { let result = rule.optimize(repartition, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); assert!( - adaptive - .input() - .as_any() - .downcast_ref::() - .is_some(), + adaptive.input().downcast_ref::().is_some(), "RoundRobin repartition should be kept as-is (not replaced)" ); } @@ -472,16 +415,9 @@ mod tests { let result = rule.optimize(repartition, &config()).unwrap(); - let adaptive = result - .as_any() - .downcast_ref::() - .unwrap(); + let adaptive = result.downcast_ref::().unwrap(); assert!( - adaptive - .input() - .as_any() - .downcast_ref::() - .is_some(), + adaptive.input().downcast_ref::().is_some(), "Hash repartition should be kept when input has an unresolved exchange" ); } @@ -493,10 +429,7 @@ mod tests { let rule = DistributedExchangeRule::default(); let result = rule.optimize(leaf_exec(), &config()).unwrap(); assert!( - result - .as_any() - .downcast_ref::() - .is_some(), + result.downcast_ref::().is_some(), "optimize should always wrap the result in AdaptiveDatafusionExec" ); } @@ -510,13 +443,11 @@ mod tests { let result = rule.optimize(adaptive, &config()).unwrap(); let outer = result - .as_any() .downcast_ref::() .expect("result should be AdaptiveDatafusionExec"); assert!( outer .input() - .as_any() .downcast_ref::() .is_none(), "existing AdaptiveDatafusionExec should not be wrapped in another one" @@ -609,15 +540,12 @@ mod tests { ) .unwrap(); let exchange1 = result1 - .as_any() .downcast_ref::() .unwrap() .input() - .as_any() .downcast_ref::() .unwrap() .children()[0] - .as_any() .downcast_ref::() .unwrap(); assert_eq!( @@ -632,15 +560,12 @@ mod tests { ) .unwrap(); let exchange2 = result2 - .as_any() .downcast_ref::() .unwrap() .input() - .as_any() .downcast_ref::() .unwrap() .children()[0] - .as_any() .downcast_ref::() .unwrap(); assert_eq!( diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs index 848b0ca0ec..53f44dae61 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs @@ -61,13 +61,11 @@ impl PhysicalOptimizerRule for DelayJoinSelectionRule { .unwrap_or_default(); if bc.adaptive_join_enabled() { let result = plan.transform_up(|node| { - if let Some(hj) = node.as_any().downcast_ref::() { + if let Some(hj) = node.downcast_ref::() { let dynamic = DynamicJoinSelectionExec::from_hash_join(hj, self.plan_id())?; Ok(Transformed::yes(dynamic as Arc)) - } else if let Some(smj) = - node.as_any().downcast_ref::() - { + } else if let Some(smj) = node.downcast_ref::() { let dynamic = DynamicJoinSelectionExec::from_sort_join(smj, self.plan_id())?; Ok(Transformed::yes(dynamic as Arc)) @@ -146,7 +144,7 @@ impl PhysicalOptimizerRule for SelectJoinRule { if bc.adaptive_join_enabled() { let result = plan.transform_up(|node| { if let Some(dynamic_join) = - node.as_any().downcast_ref::() + node.downcast_ref::() { if dynamic_join.upstream_resolved() { match dynamic_join.to_actual_join(config)? { @@ -161,7 +159,6 @@ impl PhysicalOptimizerRule for SelectJoinRule { let right = hash_join_exec.right.clone(); let right = right - .as_any() .downcast_ref::() .ok_or(DataFusionError::Execution( "ExchangeExec is expected at this point (right)" @@ -174,19 +171,17 @@ impl PhysicalOptimizerRule for SelectJoinRule { .with_new_children(vec![left, right])?; let join = join - .as_any() - .downcast_ref::() - .ok_or(DataFusionError::Execution( - "HashJoinExec is expected at this point (right)" - .to_owned(), - ))?; + .downcast_ref::() + .ok_or(DataFusionError::Execution( + "HashJoinExec is expected at this point (right)" + .to_owned(), + ))?; let join = join.swap_inputs(PartitionMode::CollectLeft)?; Ok(Transformed::yes(join)) } else { let left = hash_join_exec .left - .as_any() .downcast_ref::() .ok_or(DataFusionError::Execution( "ExchangeExec is expected at this point (left)" @@ -216,7 +211,7 @@ impl PhysicalOptimizerRule for SelectJoinRule { let exec = plan.transform_up(|p| { if let Some(hash_join) = - p.as_any().downcast_ref::() + p.downcast_ref::() && matches!( hash_join.partition_mode(), PartitionMode::CollectLeft @@ -224,10 +219,9 @@ impl PhysicalOptimizerRule for SelectJoinRule { { if let Some(data_source) = hash_join .left - .as_any() - .downcast_ref::() - && let Ok(Some(left)) = - data_source.repartitioned(1, config) + .downcast_ref::( + ) && let Ok(Some(left)) = + data_source.repartitioned(1, config) { let right = hash_join.right.clone(); let p = @@ -242,7 +236,6 @@ impl PhysicalOptimizerRule for SelectJoinRule { { let left = if let Some(exchange) = hash_join .left - .as_any() .downcast_ref::() { Arc::new( diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs index 86002c316f..87d52a5bd8 100644 --- a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs +++ b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs @@ -41,7 +41,7 @@ use std::sync::Arc; macro_rules! is_empty_exec { ($e:expr) => { - $e.as_any().downcast_ref::().is_some() + $e.downcast_ref::().is_some() }; } @@ -75,45 +75,45 @@ impl PropagateEmptyExecRule { fn transform( plan: Arc, ) -> datafusion::error::Result>> { - if let Some(filter) = plan.as_any().downcast_ref::() + if let Some(filter) = plan.downcast_ref::() && is_empty_exec!(filter.input()) { Ok(Transformed::yes(filter.input().clone())) - } else if let Some(coalesce) = plan.as_any().downcast_ref::() + } else if let Some(coalesce) = plan.downcast_ref::() && is_empty_exec!(coalesce.input()) { Ok(Transformed::yes(coalesce.input().clone())) - } else if let Some(exchange) = plan.as_any().downcast_ref::() + } else if let Some(exchange) = plan.downcast_ref::() && is_empty_exec!(exchange.input()) { Ok(Transformed::yes(exchange.input().clone())) - } else if let Some(projection) = plan.as_any().downcast_ref::() + } else if let Some(projection) = plan.downcast_ref::() && is_empty_exec!(projection.input()) { empty_exec!(projection) - } else if let Some(limit) = plan.as_any().downcast_ref::() + } else if let Some(limit) = plan.downcast_ref::() && is_empty_exec!(limit.input()) { Ok(Transformed::yes(limit.input().clone())) - } else if let Some(limit) = plan.as_any().downcast_ref::() + } else if let Some(limit) = plan.downcast_ref::() && is_empty_exec!(limit.input()) { Ok(Transformed::yes(limit.input().clone())) - } else if let Some(aggregation) = plan.as_any().downcast_ref::() + } else if let Some(aggregation) = plan.downcast_ref::() && is_empty_exec!(aggregation.input()) { empty_exec!(aggregation) - } else if let Some(repartition) = plan.as_any().downcast_ref::() + } else if let Some(repartition) = plan.downcast_ref::() && is_empty_exec!(repartition.input()) { empty_exec!(repartition) } else if let Some(coalesce_partition) = - plan.as_any().downcast_ref::() + plan.downcast_ref::() && is_empty_exec!(coalesce_partition.input()) { empty_exec!(coalesce_partition) } else if let Some(sort_preserving_merge) = - plan.as_any().downcast_ref::() + plan.downcast_ref::() && is_empty_exec!(sort_preserving_merge.input()) { empty_exec!(sort_preserving_merge) @@ -180,7 +180,7 @@ impl PropagateEmptyExecRule { } _ => Ok(Transformed::no(plan)), } - } else if let Some(exchange) = plan.as_any().downcast_ref::() { + } else if let Some(exchange) = plan.downcast_ref::() { let stats = exchange.partition_statistics(None)?; match stats.num_rows { Precision::Exact(0) => empty_exec!(plan), @@ -213,7 +213,7 @@ impl PhysicalOptimizerRule for PropagateEmptyExecRule { /// Extracting the physical join information from the incoming [ExecutionPlan] pub fn as_join(plan: &Arc) -> Option> { - let any = plan.as_any(); + let any = plan.as_ref(); if let Some(join) = any.downcast_ref::() { return Some(JoinInfo { @@ -422,7 +422,7 @@ mod tests { /// Assert the result is an EmptyExec. fn assert_empty(result: &Arc) { assert!( - result.as_any().downcast_ref::().is_some(), + result.downcast_ref::().is_some(), "expected EmptyExec, got {:?}", result.name() ); @@ -431,7 +431,7 @@ mod tests { /// Assert the result is a ProjectionExec (null-padded surviving side). fn assert_projection(result: &Arc) { assert!( - result.as_any().downcast_ref::().is_some(), + result.downcast_ref::().is_some(), "expected ProjectionExec, got {:?}", result.name() ); @@ -441,8 +441,8 @@ mod tests { /// i.e. the join was left untouched. fn assert_untouched(result: &Arc) { assert!( - result.as_any().downcast_ref::().is_none() - && result.as_any().downcast_ref::().is_none(), + result.downcast_ref::().is_none() + && result.downcast_ref::().is_none(), "expected join to be untouched, got {:?}", result.name() ); @@ -732,7 +732,7 @@ mod tests { let plan = hash_join(Arc::clone(&left), empty_stats_exec(), JoinType::LeftAnti); let result = transform(plan); // Should be the left child itself, not wrapped in anything - assert!(result.as_any().downcast_ref::().is_some()); + assert!(result.downcast_ref::().is_some()); } #[test] @@ -761,7 +761,7 @@ mod tests { let right = non_empty_stats_exec(); let plan = hash_join(empty_stats_exec(), Arc::clone(&right), JoinType::RightAnti); let result = transform(plan); - assert!(result.as_any().downcast_ref::().is_some()); + assert!(result.downcast_ref::().is_some()); } #[test] @@ -857,7 +857,7 @@ mod tests { JoinType::LeftAnti, ); let result = transform(plan); - assert!(result.as_any().downcast_ref::().is_some()); + assert!(result.downcast_ref::().is_some()); } #[test] @@ -868,7 +868,7 @@ mod tests { JoinType::RightAnti, ); let result = transform(plan); - assert!(result.as_any().downcast_ref::().is_some()); + assert!(result.downcast_ref::().is_some()); } // ── unknown stats — never optimised ───────────────────────────────────── @@ -945,7 +945,7 @@ mod tests { .optimize(plan.clone(), &ConfigOptions::default()) .unwrap(); - let empty_exec = result.as_any().downcast_ref::(); + let empty_exec = result.downcast_ref::(); assert!(empty_exec.is_some(), "expected EmptyExec"); assert_eq!( empty_exec diff --git a/ballista/scheduler/src/state/aqe/planner.rs b/ballista/scheduler/src/state/aqe/planner.rs index 81afd9d5df..844bcc32b1 100644 --- a/ballista/scheduler/src/state/aqe/planner.rs +++ b/ballista/scheduler/src/state/aqe/planner.rs @@ -216,8 +216,8 @@ impl AdaptivePlanner { .as_ref() .map(|stage| { ( - stage.as_any().downcast_ref::(), - stage.as_any().downcast_ref::(), + stage.downcast_ref::(), + stage.downcast_ref::(), ) }) { Some((Some(stage), None)) => { @@ -266,7 +266,6 @@ impl AdaptivePlanner { ), )?; let is_broadcast = stage - .as_any() .downcast_ref::() .map(|e| e.broadcast) .unwrap_or(false); @@ -395,7 +394,7 @@ impl AdaptivePlanner { if !runnable_stages.is_empty() { let mut runnable = Vec::new(); for exec in runnable_stages.into_iter() { - match exec.as_any().downcast_ref::() { + match exec.downcast_ref::() { Some(exchange) if exchange.inactive_stage => continue, Some(exchange) if exchange.stage_id().is_none() => { exchange.set_stage_id(self.stage_id_generator); @@ -418,9 +417,7 @@ impl AdaptivePlanner { } Ok(Some(runnable)) - } else if let Some(root) = - self.plan.as_any().downcast_ref::() - { + } else if let Some(root) = self.plan.downcast_ref::() { // shuffle writer has finished // there is no more runnable stages if root.shuffle_created() { @@ -462,8 +459,7 @@ impl AdaptivePlanner { runnable_stages .into_iter() .map(|exec| { - exec.as_any() - .downcast_ref::() + exec.downcast_ref::() .ok_or_else(|| { datafusion::common::DataFusionError::Plan( "ExchangeExec expected".into(), @@ -569,7 +565,7 @@ impl AdaptivePlanner { node: &Arc, runnable_stages: &mut Vec>, ) -> bool { - if let Some(exchange) = node.as_any().downcast_ref::() + if let Some(exchange) = node.downcast_ref::() && exchange.shuffle_created() { // we found exchange which has partitions resolved or this stage is not @@ -577,7 +573,7 @@ impl AdaptivePlanner { // all runnable children has been run false - } else if let Some(exchange) = node.as_any().downcast_ref::() + } else if let Some(exchange) = node.downcast_ref::() && !exchange.shuffle_created() { // we found exchange which has not been resolved (run) diff --git a/ballista/scheduler/src/state/aqe/test/alter_stages.rs b/ballista/scheduler/src/state/aqe/test/alter_stages.rs index fcf48e9cfa..7757cddd79 100644 --- a/ballista/scheduler/src/state/aqe/test/alter_stages.rs +++ b/ballista/scheduler/src/state/aqe/test/alter_stages.rs @@ -37,7 +37,6 @@ use datafusion::physical_plan::test::exec::StatisticsExec; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties, }; -use std::any::Any; use std::collections::HashSet; use std::fmt::Formatter; use std::sync::Arc; @@ -211,45 +210,42 @@ async fn should_support_join_re_ordering() -> datafusion::error::Result<()> { // join ordering changes as build side is bigger than probe side // after exchange statistic updated. - assert_plan!(planner.current_plan(), @ r" + assert_plan!(planner.current_plan(), @ " AdaptiveDatafusionExec: is_final=false, plan_id=2, stage_id=pending, stage_resolved=false - ProjectionExec: expr=[big_col@1 as big_col, big_col@0 as big_col] - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)] - ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true - CooperativeExec - MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] - ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true - CooperativeExec - MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)], projection=[big_col@1, big_col@0] + ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true + CooperativeExec + MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] + ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true + CooperativeExec + MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] "); let stages = planner.runnable_stages()?.unwrap(); assert_eq!(1, stages.len()); - assert_plan!(planner.current_plan(), @ r" + assert_plan!(planner.current_plan(), @ " AdaptiveDatafusionExec: is_final=true, plan_id=2, stage_id=2, stage_resolved=false - ProjectionExec: expr=[big_col@1 as big_col, big_col@0 as big_col] - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)] - ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true - CooperativeExec - MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] - ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true - CooperativeExec - MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)], projection=[big_col@1, big_col@0] + ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true + CooperativeExec + MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] + ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true + CooperativeExec + MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] "); planner.finalise_stage_internal(2, small_statistics_exchange())?; - assert_plan!(planner.current_plan(), @ r" + assert_plan!(planner.current_plan(), @ " AdaptiveDatafusionExec: is_final=true, plan_id=2, stage_id=2, stage_resolved=true - ProjectionExec: expr=[big_col@1 as big_col, big_col@0 as big_col] - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)] - ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true - CooperativeExec - MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] - ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true - CooperativeExec - MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)], projection=[big_col@1, big_col@0] + ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true + CooperativeExec + MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] + ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true + CooperativeExec + MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]] "); Ok(()) @@ -649,10 +645,6 @@ impl ExecutionPlan for MockPartitionedScan { "MockPartitionedScan" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.plan_properties } @@ -679,7 +671,7 @@ impl ExecutionPlan for MockPartitionedScan { fn partition_statistics( &self, _partition: Option, - ) -> datafusion::common::Result { - Ok(self.statistics.clone()) + ) -> datafusion::common::Result> { + Ok(Arc::new(self.statistics.clone())) } } diff --git a/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs b/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs index 528a912fb8..b115f273c5 100644 --- a/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs +++ b/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs @@ -25,6 +25,7 @@ use crate::state::aqe::test::{ use ballista_core::execution_plans::SortShuffleWriterExec; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::common::ColumnStatistics; +use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::Statistics; use datafusion::physical_plan::test::exec::StatisticsExec; use std::collections::HashSet; @@ -475,7 +476,7 @@ async fn should_use_sort_shuffle_when_enabled() -> datafusion::error::Result<()> let plan = stages.first().unwrap().plan.as_ref(); assert!( - plan.as_any() + (plan as &dyn ExecutionPlan) .downcast_ref::() .is_some(), "expected SortShuffleWriterExec when sort shuffle is enabled, got plan: {plan:?}" @@ -505,7 +506,7 @@ async fn should_use_sort_shuffle_by_default() -> datafusion::error::Result<()> { let plan = stages.first().unwrap().plan.as_ref(); assert!( - plan.as_any() + (plan as &dyn ExecutionPlan) .downcast_ref::() .is_some(), "expected SortShuffleWriterExec by default, got plan: {plan:?}" diff --git a/ballista/scheduler/src/state/distributed_explain.rs b/ballista/scheduler/src/state/distributed_explain.rs index 4292950329..c4408be468 100644 --- a/ballista/scheduler/src/state/distributed_explain.rs +++ b/ballista/scheduler/src/state/distributed_explain.rs @@ -195,7 +195,7 @@ pub(crate) async fn handle_explain_plan( plan: Arc, ) -> ballista_core::error::Result> { if let LogicalPlan::Explain(explain_plan) = &logical_plan - && let Some(explain) = plan.as_any().downcast_ref::() + && let Some(explain) = plan.downcast_ref::() { let inner_plan = explain_plan.plan.clone(); let plans = explain.stringified_plans(); diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 72fd05f8f9..51228a40b6 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -1633,14 +1633,12 @@ impl ExecutionPlanVisitor for ExecutionStageBuilder { plan: &dyn ExecutionPlan, ) -> std::result::Result { // Handle both ShuffleWriterExec and SortShuffleWriterExec - if let Some(shuffle_write) = plan.as_any().downcast_ref::() { + if let Some(shuffle_write) = plan.downcast_ref::() { self.current_stage_id = shuffle_write.stage_id(); - } else if let Some(shuffle_write) = - plan.as_any().downcast_ref::() - { + } else if let Some(shuffle_write) = plan.downcast_ref::() { self.current_stage_id = shuffle_write.stage_id(); } else if let Some(unresolved_shuffle) = - plan.as_any().downcast_ref::() + plan.downcast_ref::() { if let Some(output_links) = self.output_links.get_mut(&unresolved_shuffle.stage_id) @@ -1710,18 +1708,14 @@ impl TaskDescription { /// Returns the number of output partitions this task will produce. pub fn get_output_partition_number(&self) -> usize { // Try ShuffleWriterExec first - if let Some(shuffle_writer) = - self.plan.as_any().downcast_ref::() - { + if let Some(shuffle_writer) = self.plan.downcast_ref::() { return shuffle_writer .shuffle_output_partitioning() .map(|partitioning| partitioning.partition_count()) .unwrap_or(1); } // Try SortShuffleWriterExec - if let Some(shuffle_writer) = - self.plan.as_any().downcast_ref::() - { + if let Some(shuffle_writer) = self.plan.downcast_ref::() { return shuffle_writer .shuffle_output_partitioning() .partition_count(); diff --git a/ballista/scheduler/src/state/execution_graph_dot.rs b/ballista/scheduler/src/state/execution_graph_dot.rs index f43772dd0f..eaef4dc5e9 100644 --- a/ballista/scheduler/src/state/execution_graph_dot.rs +++ b/ballista/scheduler/src/state/execution_graph_dot.rs @@ -148,7 +148,7 @@ fn write_plan_recursive( let node_name = format!("{prefix}_{i}"); let display_name = get_operator_name(plan); - if let Some(reader) = plan.as_any().downcast_ref::() { + if let Some(reader) = plan.downcast_ref::() { for part in &reader.partition { for loc in part { state @@ -156,7 +156,7 @@ fn write_plan_recursive( .insert(node_name.clone(), loc.partition_id.stage_id); } } - } else if let Some(reader) = plan.as_any().downcast_ref::() { + } else if let Some(reader) = plan.downcast_ref::() { state.readers.insert(node_name.clone(), reader.stage_id); } @@ -230,9 +230,9 @@ fn sanitize(str: &str, max_len: Option) -> String { } #[allow(deprecated)] fn get_operator_name(plan: &dyn ExecutionPlan) -> String { - if let Some(exec) = plan.as_any().downcast_ref::() { + if let Some(exec) = plan.downcast_ref::() { format!("Filter: {}", exec.predicate()) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { let expr = exec .expr() .iter() @@ -241,7 +241,7 @@ fn get_operator_name(plan: &dyn ExecutionPlan) -> String { .collect::>() .join(", "); format!("Projection: {}", sanitize_dot_label(&expr)) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { let sort_expr = exec .expr() .iter() @@ -257,7 +257,7 @@ fn get_operator_name(plan: &dyn ExecutionPlan) -> String { .collect::>() .join(", "); format!("Sort: {}", sanitize_dot_label(&sort_expr)) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { let group_exprs_with_alias = exec.group_expr().expr(); let group_expr = group_exprs_with_alias .iter() @@ -277,19 +277,19 @@ aggr=[{}]", sanitize_dot_label(&group_expr), sanitize_dot_label(&aggr_expr) ) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!("CoalesceBatches [batchSize={}]", exec.target_batch_size()) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!( "CoalescePartitions [{}]", format_partitioning(exec.properties().output_partitioning().clone()) ) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!( "RepartitionExec [{}]", format_partitioning(exec.properties().output_partitioning().clone()) ) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { let join_expr = exec .on() .iter() @@ -308,49 +308,46 @@ filter_expr={}", sanitize_dot_label(&join_expr), sanitize_dot_label(&filter_expr) ) - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "CrossJoin".to_string() - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { "Union".to_string() - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!("UnresolvedShuffleExec [stage_id={}]", exec.stage_id) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!("ShuffleReader [{} partitions]", exec.partition.len()) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!( "ShuffleWriter [{} partitions]", exec.input_partition_count() ) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!( "SortShuffleWriter [{} partitions]", exec.input_partition_count() ) - } else if let Some(exec) = plan.as_any().downcast_ref::() { - let config = if let Some(config) = - exec.data_source().as_any().downcast_ref::() - { - get_file_scan(config) - } else if let Some(_config) = exec - .data_source() - .as_any() - .downcast_ref::() - { - "Memory".to_string() - } else { - "Unknown".to_string() - }; + } else if let Some(exec) = plan.downcast_ref::() { + let config = + if let Some(config) = exec.data_source().downcast_ref::() { + get_file_scan(config) + } else if let Some(_config) = + exec.data_source().downcast_ref::() + { + "Memory".to_string() + } else { + "Unknown".to_string() + }; let parts = exec.properties().output_partitioning().partition_count(); format!("DataSourceExec: ({config}) [{parts} partitions]") - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!( "GlobalLimit(skip={}, fetch={:?})", exec.skip(), exec.fetch() ) - } else if let Some(exec) = plan.as_any().downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { format!("LocalLimit({})", exec.fetch()) } else { debug!("Unknown physical operator when producing DOT graph: {plan:?}"); @@ -589,6 +586,14 @@ filter_expr="] .options_mut() .optimizer .enable_round_robin_repartition = false; + config + .options_mut() + .optimizer + .hash_join_single_partition_threshold = 0; + config + .options_mut() + .optimizer + .hash_join_single_partition_threshold_rows = 0; let ctx = SessionContext::new_with_config(config); let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::UInt32, false), @@ -627,6 +632,14 @@ filter_expr="] .options_mut() .optimizer .enable_round_robin_repartition = false; + config + .options_mut() + .optimizer + .hash_join_single_partition_threshold = 0; + config + .options_mut() + .optimizer + .hash_join_single_partition_threshold_rows = 0; let ctx = SessionContext::new_with_config(config); let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, false)])); diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index a9695d3b8b..e9e8f1b07f 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -1005,11 +1005,11 @@ impl Debug for FailedStage { /// will be different. Here, we should use the input partition count. fn get_stage_partitions(plan: Arc) -> usize { // Try ShuffleWriterExec first - if let Some(shuffle_writer) = plan.as_any().downcast_ref::() { + if let Some(shuffle_writer) = plan.downcast_ref::() { return shuffle_writer.input_partition_count(); } // Try SortShuffleWriterExec - if let Some(shuffle_writer) = plan.as_any().downcast_ref::() { + if let Some(shuffle_writer) = plan.downcast_ref::() { return shuffle_writer.input_partition_count(); } // Fallback to output partitioning diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs index 810983ef2b..4f7b29651c 100644 --- a/ballista/scheduler/src/test_utils.rs +++ b/ballista/scheduler/src/test_utils.rs @@ -19,7 +19,6 @@ use ballista_core::error::{BallistaError, Result}; use ballista_core::extension::SessionConfigExt; use ballista_core::{JobId, JobStatusSubscriber}; use datafusion::catalog::Session; -use std::any::Any; use std::collections::HashMap; use std::future::Future; use std::sync::Arc; @@ -80,10 +79,6 @@ pub struct ExplodingTableProvider; #[async_trait::async_trait] impl TableProvider for ExplodingTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::new(Schema::empty()) } @@ -1016,7 +1011,7 @@ pub async fn test_coalesce_plan(partition: usize) -> StaticExecutionGraph { /// Creates a test execution graph with a join operation. pub async fn test_join_plan(partition: usize) -> StaticExecutionGraph { - let mut config = SessionConfig::new().with_target_partitions(partition); + let mut config = SessionConfig::new_with_ballista().with_target_partitions(partition); config .options_mut() .optimizer From 7cc71f3140f767ae692ef623b3e4c4f3e8585865 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 1 Jul 2026 08:05:24 -0600 Subject: [PATCH 015/107] ci: verify TPC-H query results against single-process DataFusion (#1917) --- .github/workflows/tpch.yml | 2 + benchmarks/src/bin/tpch.rs | 416 ++++++++++++++++++++++++++++++------- 2 files changed, 345 insertions(+), 73 deletions(-) diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 7e5f4b2665..38dd179c24 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -169,8 +169,10 @@ jobs: } run_suite "AQE off" \ + --verify \ -c datafusion.optimizer.prefer_hash_join=false run_suite "AQE on" \ + --verify \ -c datafusion.optimizer.prefer_hash_join=false \ -c ballista.planner.adaptive.enabled=true diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index 398c9aa45c..282eb241f2 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -125,6 +125,11 @@ struct BallistaBenchmarkOpt { /// -c datafusion.execution.target_partitions=16 #[structopt(short = "c", long = "config", number_of_values = 1)] config_overrides: Vec, + + /// Verify each Ballista result against single-process DataFusion over the + /// same data. + #[structopt(long = "verify")] + verify: bool, } #[derive(Debug, StructOpt, Clone)] @@ -296,19 +301,19 @@ async fn benchmark_datafusion(opt: DataFusionBenchmarkOpt) -> Result Result Result Result<()> { .map(|q| vec![q]) .unwrap_or_else(|| (1..=22).collect()); + let oracle_ctx = if opt.verify { + let cfg = SessionConfig::new() + .with_target_partitions(opt.partitions) + .with_batch_size(opt.batch_size); + let ctx = SessionContext::new_with_config(cfg); + register_datafusion_tables( + &ctx, + opt.path.to_str().unwrap(), + opt.file_format.as_str(), + opt.partitions, + ) + .await?; + Some(ctx) + } else { + None + }; + let mut benchmark_run = BenchmarkRun::new(); let mut total_elapsed = 0.0; @@ -456,9 +479,10 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> { println!("Running benchmark with query {}:\n {:?}", query, sqls); } let mut batches = vec![]; + let answer_idx = answer_statement_index(&sqls); for i in 0..opt.iterations { let start = Instant::now(); - for sql in &sqls { + for (idx, sql) in sqls.iter().enumerate() { let df = ctx .sql(sql) .await @@ -468,11 +492,14 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> { if opt.debug { println!("=== Optimized logical plan ===\n{plan:?}\n"); } - batches = df + let collected = df .collect() .await .map_err(|e| DataFusionError::Plan(format!("{e:?}"))) .unwrap(); + if idx == answer_idx { + batches = collected; + } } let elapsed = start.elapsed().as_secs_f64(); secs.push(elapsed); @@ -495,10 +522,21 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> { if let Some(expected_results_path) = opt.expected_results.as_ref() { let expected = get_expected_results(query, expected_results_path).await?; - assert_expected_results(&expected, &batches) + compare_results(&expected, &batches)?; } } + if let Some(oracle_ctx) = &oracle_ctx { + let expected = + execute_query_capturing_answer(oracle_ctx, &sqls, opt.debug).await?; + compare_results(&expected, &batches).map_err(|e| { + DataFusionError::Execution(format!( + "query {query} verification failed: {e}" + )) + })?; + println!("Query {query} verified against DataFusion: OK"); + } + if opt.iterations > 1 { let avg = secs.iter().sum::() / secs.len() as f64; println!("Query {} avg time: {:.3} s", query, avg); @@ -642,6 +680,46 @@ fn get_query_sql_by_path(query: usize, mut sql_path: String) -> Result { } } +/// Registers all TPC-H tables on a single-process DataFusion context (the +/// correctness oracle for `--verify`). +async fn register_datafusion_tables( + ctx: &SessionContext, + path: &str, + file_format: &str, + partitions: usize, +) -> Result<()> { + for table in TABLES { + let table_provider = { + let mut session_state = ctx.state(); + get_table(&mut session_state, path, table, file_format, partitions).await? + }; + ctx.register_table(*table, table_provider)?; + } + Ok(()) +} + +/// Executes all statements of a query in order and returns the batches of the +/// answer statement (see `answer_statement_index`). +async fn execute_query_capturing_answer( + ctx: &SessionContext, + statements: &[String], + debug: bool, +) -> Result> { + let answer_idx = answer_statement_index(statements); + let mut answer = vec![]; + for (idx, sql) in statements.iter().enumerate() { + if debug { + println!("Executing: {sql}"); + } + let df = ctx.sql(sql).await?; + let collected = df.collect().await?; + if idx == answer_idx { + answer = collected; + } + } + Ok(answer) +} + async fn register_tables( path: &str, file_format: &str, @@ -716,6 +794,20 @@ fn find_path(path: &str, table: &str, ext: &str) -> Result { } } +/// Index of the statement whose result is the query answer: the last `SELECT` +/// or `WITH` statement, or the last statement if none qualifies. TPC-H query +/// files may wrap the answer in setup/teardown statements (e.g. q15 creates and +/// drops a view); only the query statement's result is the answer. +fn answer_statement_index(statements: &[String]) -> usize { + statements + .iter() + .rposition(|s| { + let head = s.trim_start().to_ascii_lowercase(); + head.starts_with("select") || head.starts_with("with") + }) + .unwrap_or_else(|| statements.len().saturating_sub(1)) +} + /// Get the SQL statements from the specified query file fn get_query_sql(query: usize) -> Result> { if query > 0 && query < 23 { @@ -1116,27 +1208,137 @@ impl BenchmarkRun { } } -/// Compare actual results against expected results at scale factor 1 -fn assert_expected_results(expected: &[RecordBatch], actual: &[RecordBatch]) { - // assert schema equality without comparing nullable values - assert_eq!( - nullable_schema(expected[0].schema()), - nullable_schema(actual[0].schema()) - ); +/// Maximum relative-or-absolute difference tolerated between floating-point +/// cells. Distributed (Ballista) and single-process (DataFusion) execution +/// aggregate in different orders, and float `sum` is non-associative, so ratio +/// queries can differ in their last digits. +const FLOAT_TOLERANCE: f64 = 1e-6; + +enum Cell { + Null, + Float(f64), + Text(String), +} - // convert both datasets to Vec> for simple comparison - let expected_vec = result_vec(expected); - let actual_vec = result_vec(actual); +impl std::fmt::Display for Cell { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Cell::Null => f.write_str("NULL"), + Cell::Float(x) => write!(f, "{x}"), + Cell::Text(s) => f.write_str(s), + } + } +} - // basic result comparison - assert_eq!(expected_vec.len(), actual_vec.len()); +fn cell_at(column: &ArrayRef, row: usize) -> Cell { + if column.is_null(row) { + return Cell::Null; + } + match column.data_type() { + DataType::Float64 => Cell::Float( + column + .as_any() + .downcast_ref::() + .unwrap() + .value(row), + ), + DataType::Float32 => Cell::Float( + column + .as_any() + .downcast_ref::() + .unwrap() + .value(row) as f64, + ), + _ => Cell::Text(col_str(column, row)), + } +} + +fn rows_as_cells(batches: &[RecordBatch]) -> Vec> { + let mut rows = vec![]; + for batch in batches { + for row in 0..batch.num_rows() { + rows.push(batch.columns().iter().map(|c| cell_at(c, row)).collect()); + } + } + rows +} + +fn floats_close(a: f64, b: f64) -> bool { + if a == b { + return true; + } + (a - b).abs() <= FLOAT_TOLERANCE * a.abs().max(b.abs()).max(1.0) +} - // compare each row. this works as all TPC-H queries have deterministically ordered results - for i in 0..actual_vec.len() { - assert_eq!(expected_vec[i], actual_vec[i]); +fn cells_equal(a: &Cell, b: &Cell) -> bool { + match (a, b) { + (Cell::Null, Cell::Null) => true, + (Cell::Float(x), Cell::Float(y)) => floats_close(*x, *y), + (Cell::Text(x), Cell::Text(y)) => x == y, + _ => false, } } +/// Canonicalizes Arrow string/binary representation variants so that physical +/// differences (e.g. `Utf8` vs `Utf8View`) are not treated as result +/// differences: single-process DataFusion may infer Parquet strings as +/// `Utf8View` while Ballista produces `Utf8`, but both stringify identically. +fn canonical_type(dt: &DataType) -> DataType { + match dt { + DataType::Utf8View | DataType::LargeUtf8 => DataType::Utf8, + DataType::BinaryView | DataType::LargeBinary => DataType::Binary, + _ => dt.clone(), + } +} + +/// Schema reduced to (name, canonical type) pairs, ignoring nullability and +/// string/binary representation, for result comparison. +fn comparable_schema(schema: &Schema) -> Vec<(String, DataType)> { + schema + .fields() + .iter() + .map(|f| (f.name().clone(), canonical_type(f.data_type()))) + .collect() +} + +/// Compares two result sets, tolerating tiny floating-point differences. +/// Returns an error describing the first mismatch instead of panicking, so the +/// benchmark binary exits non-zero with a useful message. +fn compare_results(expected: &[RecordBatch], actual: &[RecordBatch]) -> Result<()> { + let expected_rows = rows_as_cells(expected); + let actual_rows = rows_as_cells(actual); + + if expected_rows.len() != actual_rows.len() { + return Err(DataFusionError::Execution(format!( + "result mismatch: expected {} rows, got {} rows", + expected_rows.len(), + actual_rows.len() + ))); + } + + if let (Some(e), Some(a)) = (expected.first(), actual.first()) { + let e_schema = comparable_schema(&e.schema()); + let a_schema = comparable_schema(&a.schema()); + if e_schema != a_schema { + return Err(DataFusionError::Execution(format!( + "schema mismatch:\n expected: {e_schema:?}\n actual: {a_schema:?}" + ))); + } + } + + for (i, (erow, arow)) in expected_rows.iter().zip(actual_rows.iter()).enumerate() { + for (j, (ecell, acell)) in erow.iter().zip(arow.iter()).enumerate() { + if !cells_equal(ecell, acell) { + return Err(DataFusionError::Execution(format!( + "result mismatch at row {i}, column {j}: expected `{ecell}`, got `{acell}`" + ))); + } + } + } + + Ok(()) +} + /// Get the expected answer for a specific query at scale factor 1 async fn get_expected_results(n: usize, path: &str) -> Result> { let ctx = SessionContext::new(); @@ -1179,37 +1381,6 @@ async fn get_expected_results(n: usize, path: &str) -> Result> df.collect().await } -// convert the schema to the same but with all columns set to nullable=true. -// this allows direct schema comparison ignoring nullable. -fn nullable_schema(schema: Arc) -> Schema { - Schema::new( - schema - .fields() - .iter() - .map(|field| { - Field::new(Field::name(field), Field::data_type(field).to_owned(), true) - }) - .collect::>(), - ) -} - -/// Converts the results into a 2d array of strings, `result[row][column]` -/// Special cases nulls to NULL for testing -fn result_vec(results: &[RecordBatch]) -> Vec> { - let mut result = vec![]; - for batch in results { - for row_index in 0..batch.num_rows() { - let row_vec = batch - .columns() - .iter() - .map(|column| col_str(column, row_index)) - .collect(); - result.push(row_vec); - } - } - result -} - fn get_answer_schema(n: usize) -> Schema { match n { 1 => Schema::new(vec![ @@ -1757,13 +1928,112 @@ mod tests { let expected_schema = get_answer_schema(n); let normalized = normalize_for_verification(actual, expected_schema).await?; - assert_expected_results(&expected, &normalized) + compare_results(&expected, &normalized)?; } else { println!("TPCH_DATA environment variable not set, skipping test"); } Ok(()) } + + fn f64_batch(values: Vec) -> RecordBatch { + let schema = + Arc::new(Schema::new(vec![Field::new("x", DataType::Float64, false)])); + RecordBatch::try_new(schema, vec![Arc::new(Float64Array::from(values))]).unwrap() + } + + fn i64_batch(values: Vec) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![Field::new("n", DataType::Int64, false)])); + RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(values))]).unwrap() + } + + #[test] + fn compare_results_equal_ok() { + assert!( + super::compare_results(&[i64_batch(vec![1, 2])], &[i64_batch(vec![1, 2])]) + .is_ok() + ); + } + + #[test] + fn compare_results_row_count_mismatch() { + let err = super::compare_results(&[i64_batch(vec![1])], &[i64_batch(vec![1, 2])]) + .unwrap_err() + .to_string(); + assert!(err.contains("expected 1 rows, got 2 rows"), "{err}"); + } + + #[test] + fn compare_results_value_mismatch() { + let err = super::compare_results(&[i64_batch(vec![1])], &[i64_batch(vec![2])]) + .unwrap_err() + .to_string(); + assert!(err.contains("row 0, column 0"), "{err}"); + } + + #[test] + fn compare_results_floats_within_tolerance_ok() { + // differ only beyond the tolerance's significant digits + assert!( + super::compare_results( + &[f64_batch(vec![123.4567890])], + &[f64_batch(vec![123.4567891])] + ) + .is_ok() + ); + } + + #[test] + fn compare_results_floats_outside_tolerance_err() { + assert!( + super::compare_results(&[f64_batch(vec![100.0])], &[f64_batch(vec![100.5])]) + .is_err() + ); + } + + #[test] + fn answer_statement_index_picks_select_not_drop() { + let stmts = vec![ + "create view revenue0 as select 1".to_string(), + "select * from revenue0 order by a".to_string(), + "drop view revenue0".to_string(), + ]; + assert_eq!(super::answer_statement_index(&stmts), 1); + } + + #[test] + fn answer_statement_index_single_select() { + let stmts = vec!["select 1".to_string()]; + assert_eq!(super::answer_statement_index(&stmts), 0); + } + + #[test] + fn answer_statement_index_with_cte() { + let stmts = vec!["WITH t AS (select 1) select * from t".to_string()]; + assert_eq!(super::answer_statement_index(&stmts), 0); + } + + #[test] + fn compare_results_ignores_utf8view_vs_utf8() { + let view_schema = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Utf8View, + false, + )])); + let utf8_schema = + Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); + let a = RecordBatch::try_new( + view_schema, + vec![Arc::new(StringViewArray::from(vec!["x", "y"]))], + ) + .unwrap(); + let b = RecordBatch::try_new( + utf8_schema, + vec![Arc::new(StringArray::from(vec!["x", "y"]))], + ) + .unwrap(); + assert!(super::compare_results(&[a], &[b]).is_ok()); + } } #[cfg(test)] From 4659f1e3f72a258a35ad1635d72e9adac09a9609 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 1 Jul 2026 08:42:12 -0600 Subject: [PATCH 016/107] feat: broadcast small build sides via non-zero CollectLeft thresholds (#1900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(core): broadcast small build sides via non-zero CollectLeft thresholds Set hash_join_single_partition_threshold to 10MB and hash_join_single_partition_threshold_rows to 1M so the AQE join resolver collects small build sides into a broadcast (CollectLeft) hash join instead of repartitioning them. Update the client settings test to assert the new default values. * fix(scheduler): restrict CollectLeft broadcast to safe join types The adaptive join resolver chose CollectLeft purely on size thresholds. A CollectLeft join broadcasts the build (left) side to every probe task, where each task sees only its slice of the probe side, so join types that emit rows on behalf of the build side (Left, Full, LeftSemi, LeftAnti, LeftMark) would have every task emit them independently and produce duplicate or spurious rows. Restrict CollectLeft to join types whose output is driven entirely by the probe side (Inner, Right, RightSemi, RightAnti, RightMark), evaluated on the post-swap join type using the same swap decision as the resolver, and keep unsafe join types repartitioned. Add resolver tests covering left/left-anti (repartitioned) and right (still collected) joins. * fix(scheduler): guard static planner broadcast and add join-type test matrix The static DefaultDistributedPlanner promoted small-side hash joins to CollectLeft on size alone, with no join-type check, so a LEFT/FULL/semi/anti/ mark join whose build side fit under broadcast_join_threshold_bytes was broadcast and produced duplicate or spurious rows — the same hazard already guarded on the adaptive path. Move the broadcast-safety predicate into physical_optimizer::join_selection so both planners share one definition, and apply it in maybe_promote_to_broadcast using the post-swap join type. Add a test matrix covering both planners, both build-side orientations, every join type, prefer_hash_join on/off, and the post-swap decision, plus a unit test that locks down the broadcast-safe join-type set. * test(core): update upgrade defaults test for non-zero CollectLeft threshold #1902 added tests asserting the Ballista default hash_join_single_partition_threshold is 0; this PR raises that default to 10 MB (and the row threshold to 1M), so should_apply_defaults_when_upgrading_plain_config now asserts the new values. Give should_preserve_user_overrides_on_upgrade an override distinct from the default so it still proves the user value survives. * fix(scheduler): use dyn ExecutionPlan::downcast_ref in broadcast tests DataFusion 54 removed ExecutionPlan::as_any in favor of the inherent downcast_ref on dyn ExecutionPlan, which auto-derefs through Arc. The new broadcast-safety tests still used as_any().downcast_ref(), which no longer compiles. Drop the as_any() hop to match the rest of the file. * fix: demote DataFusion-promoted unsafe CollectLeft joins to partitioned DataFusion's JoinSelection runs during create_physical_plan and, with the non-zero hash_join_single_partition_threshold default this PR sets, can stamp CollectLeft on a join without restricting by join type. The distributed planner's broadcast-safety guard only covered the Partitioned -> CollectLeft promotion path, so a pre-promoted unsafe join (e.g. a LEFT join with the small side already on the left) reached the broadcast-lowering branch and replicated the outer side across every probe task, producing duplicate/spurious rows. In maybe_promote_to_broadcast, demote any CollectLeft hash join whose join type is not broadcast-safe back to a Partitioned (shuffle) join, hash-partitioning both inputs on the join keys. This is a correctness guard, so it runs regardless of the Ballista broadcast threshold. Add a test that mirrors the production session config so DataFusion's own JoinSelection performs the promotion. * fix: disable DataFusion dynamic filter pushdown in session defaults DataFusion 54 populates dynamic filters at runtime from an upstream operator (a hash join build side, a TopK heap, a partial aggregate) and reads them in a downstream scan within the same plan. Ballista splits a plan into stages at shuffle and broadcast boundaries that execute as independent tasks, so when the producing operator and the consuming scan land in different stages the filter is never populated across the boundary and the consuming scan blocks forever. This deadlocks every multi-join TPC-H query under the adaptive (AQE) planner, where broadcast (CollectLeft) hash joins create the cross-stage producer/ consumer split — the build stages complete but the probe stages hang. The static planner avoids it only because it plans sort-merge joins, which carry no join dynamic filter. Pin datafusion.optimizer.enable_dynamic_filter_pushdown to false in the Ballista session defaults, alongside the other DataFusion options Ballista already disables because they rely on in-process state that cannot cross stage boundaries. Tracked by #1375. --- ballista/client/tests/context_checks.rs | 23 +- ballista/core/src/extension.rs | 41 ++- .../src/physical_optimizer/join_selection.rs | 61 ++++ ballista/scheduler/src/planner.rs | 321 +++++++++++++++++- .../state/aqe/execution_plan/dynamic_join.rs | 131 ++++++- .../src/state/aqe/test/join_selection.rs | 32 ++ 6 files changed, 574 insertions(+), 35 deletions(-) diff --git a/ballista/client/tests/context_checks.rs b/ballista/client/tests/context_checks.rs index 2638942886..75d17f37ff 100644 --- a/ballista/client/tests/context_checks.rs +++ b/ballista/client/tests/context_checks.rs @@ -520,17 +520,14 @@ mod supported { Ok(()) } - // As mentioned in https://github.com/apache/datafusion-ballista/issues/1055 - // "Left/full outer join incorrect for CollectLeft / broadcast" - // - // In order to make correct results (decreasing performance) CollectLeft - // has been disabled until fixed - + // Ballista raises these DataFusion thresholds above zero so the adaptive + // join resolver can collect a small build side into a broadcast + // (CollectLeft) hash join instead of repartitioning it. #[rstest] #[case::standalone(standalone_context())] #[case::remote(remote_context())] #[tokio::test] - async fn should_disable_collect_left( + async fn should_set_collect_left_thresholds( #[future(awt)] #[case] ctx: SessionContext, @@ -542,12 +539,12 @@ mod supported { .await?; let expected = [ - "+----------------------------------------------------------------+-------+", - "| name | value |", - "+----------------------------------------------------------------+-------+", - "| datafusion.optimizer.hash_join_single_partition_threshold | 0 |", - "| datafusion.optimizer.hash_join_single_partition_threshold_rows | 0 |", - "+----------------------------------------------------------------+-------+", + "+----------------------------------------------------------------+----------+", + "| name | value |", + "+----------------------------------------------------------------+----------+", + "| datafusion.optimizer.hash_join_single_partition_threshold | 10485760 |", + "| datafusion.optimizer.hash_join_single_partition_threshold_rows | 1000000 |", + "+----------------------------------------------------------------+----------+", ]; assert_batches_eq!(expected, &result); diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs index dd5ec26926..aaa68c6735 100644 --- a/ballista/core/src/extension.rs +++ b/ballista/core/src/extension.rs @@ -767,18 +767,15 @@ impl SessionConfigHelperExt for SessionConfig { // same like previous comment .set_bool("datafusion.sql_parser.map_string_types_to_utf8view", false) // - // As mentioned in https://github.com/apache/datafusion-ballista/issues/1055 - // "Left/full outer join incorrect for CollectLeft / broadcast" - // - // In order to make correct results (decreasing performance) CollectLeft - // has been disabled until fixed + // A build side smaller than these thresholds is collected into a + // CollectLeft (broadcast) hash join rather than being repartitioned. .set_u64( "datafusion.optimizer.hash_join_single_partition_threshold", - 0, + 10 * 1024 * 1024, ) .set_u64( "datafusion.optimizer.hash_join_single_partition_threshold_rows", - 0, + 1_000_000, ) // // DataFusion's hash join has no spill support, so each parallel @@ -805,6 +802,19 @@ impl SessionConfigHelperExt for SessionConfig { "datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery", false, ) + // + // DataFusion's dynamic filters are populated at runtime by an + // upstream operator (a hash join build side, a TopK heap, a partial + // aggregate) and read by a downstream scan within the same plan. + // Ballista splits a plan into stages at shuffle and broadcast + // boundaries that run as independent tasks, so when the producing + // operator and the consuming scan land in different stages the + // filter is never populated across the boundary and the scan blocks + // forever. Disable dynamic filter pushdown until Ballista can carry + // dynamic filters across stage boundaries. + // + // See https://github.com/apache/datafusion-ballista/issues/1375 + .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", false) } } @@ -1046,7 +1056,9 @@ mod test { // re-applying the defaults would discard them. See #1901. #[test] fn should_preserve_user_overrides_on_upgrade() { - // Ballista defaults these to prefer_hash_join=false and threshold=0. + // Ballista defaults these to prefer_hash_join=false and the threshold to + // 10 MB. The overrides below differ from those defaults so the assertions + // prove the user's values survived `upgrade_for_ballista`. let mut config = SessionConfig::new_with_ballista(); config .options_mut() @@ -1056,7 +1068,7 @@ mod test { .options_mut() .set( "datafusion.optimizer.hash_join_single_partition_threshold", - "10485760", + "5242880", ) .unwrap(); @@ -1068,7 +1080,7 @@ mod test { .options() .optimizer .hash_join_single_partition_threshold, - 10485760 + 5242880 ); } @@ -1084,7 +1096,14 @@ mod test { .options() .optimizer .hash_join_single_partition_threshold, - 0 + 10 * 1024 * 1024 + ); + assert_eq!( + config + .options() + .optimizer + .hash_join_single_partition_threshold_rows, + 1_000_000 ); } diff --git a/ballista/scheduler/src/physical_optimizer/join_selection.rs b/ballista/scheduler/src/physical_optimizer/join_selection.rs index 7410ec0270..20d4649283 100644 --- a/ballista/scheduler/src/physical_optimizer/join_selection.rs +++ b/ballista/scheduler/src/physical_optimizer/join_selection.rs @@ -111,6 +111,31 @@ fn supports_collect_by_thresholds( } } +/// Whether a `CollectLeft` (broadcast) hash join is correct for `join_type` in +/// Ballista's distributed execution. +/// +/// `CollectLeft` replicates the build (left) side to every probe task, and each +/// task processes only its own slice of the probe (right) side. Output rows that +/// are determined by a single probe-side row are therefore produced exactly +/// once. Join types that emit rows on behalf of the build side — unmatched outer +/// rows, or semi/anti/mark rows that depend on a build row's global match status +/// — would instead be emitted independently by every task, producing duplicate +/// or spurious rows. Only join types whose output is driven entirely by the +/// probe side are safe to broadcast. +/// +/// `join_type` must be the join type *after* any build-side swap, since the +/// build side is always the left input of the resulting `HashJoinExec`. +pub(crate) fn collect_left_broadcast_safe(join_type: JoinType) -> bool { + matches!( + join_type, + JoinType::Inner + | JoinType::Right + | JoinType::RightSemi + | JoinType::RightAnti + | JoinType::RightMark + ) +} + /// Predicate that checks whether the given join type supports input swapping. #[deprecated(since = "45.0.0", note = "use JoinType::supports_swap instead")] #[allow(dead_code)] @@ -617,6 +642,7 @@ fn apply_subrules( mod test { use std::sync::Arc; + use super::collect_left_broadcast_safe; use datafusion::{ arrow::datatypes::{DataType, Field, Schema}, common::{ColumnStatistics, JoinSide, JoinType, Statistics, stats::Precision}, @@ -629,6 +655,41 @@ mod test { test::exec::StatisticsExec, }, }; + + // A CollectLeft broadcast replicates the build (left) side to every probe + // task. Only join types whose output is driven entirely by the probe (right) + // side may be broadcast; any type that emits rows on behalf of the build side + // would duplicate them across tasks. This locks down the full set so a new or + // reclassified `JoinType` can't silently become broadcastable. + #[test] + fn collect_left_broadcast_safe_matches_probe_driven_join_types() { + let safe = [ + JoinType::Inner, + JoinType::Right, + JoinType::RightSemi, + JoinType::RightAnti, + JoinType::RightMark, + ]; + let unsafe_types = [ + JoinType::Left, + JoinType::Full, + JoinType::LeftSemi, + JoinType::LeftAnti, + JoinType::LeftMark, + ]; + for join_type in safe { + assert!( + collect_left_broadcast_safe(join_type), + "{join_type:?} should be safe to broadcast" + ); + } + for join_type in unsafe_types { + assert!( + !collect_left_broadcast_safe(join_type), + "{join_type:?} must not be broadcast" + ); + } + } // // join selection should not change order of joins for // nested loop join as we're not able to insert new CoalescePartitions diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 95fcc6c9b0..6538cb45bb 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -45,7 +45,9 @@ use datafusion::physical_plan::{ }; use log::debug; -use crate::physical_optimizer::join_selection::should_swap_join_order; +use crate::physical_optimizer::join_selection::{ + collect_left_broadcast_safe, should_swap_join_order, +}; type PartialQueryStageResult = (Arc, Vec>); @@ -270,6 +272,29 @@ impl DefaultDistributedPlanner { plan: Arc, config: &ConfigOptions, ) -> Result> { + let Some(hash_join) = plan.downcast_ref::() else { + return Ok(plan); + }; + + // DataFusion's own `JoinSelection` may have already stamped + // `CollectLeft` on this join (it does so for any build side under + // `datafusion.optimizer.hash_join_single_partition_threshold`, without + // restricting by join type). A `CollectLeft` join replicates the build + // side to every probe task, which is only correct for probe-driven join + // types. If the join type is not broadcast-safe, demote it back to a + // partitioned (shuffle) join. This is a correctness guard, so it runs + // regardless of the Ballista broadcast threshold below. + if *hash_join.partition_mode() == PartitionMode::CollectLeft { + if collect_left_broadcast_safe(*hash_join.join_type()) { + return Ok(plan); + } + debug!( + "broadcast check: demoting DataFusion-promoted CollectLeft join with unsafe join_type={:?} to Partitioned", + hash_join.join_type(), + ); + return Self::demote_collect_left_to_partitioned(hash_join, config); + } + let threshold_bytes = config .extensions .get::() @@ -281,9 +306,6 @@ impl DefaultDistributedPlanner { debug!("broadcast check: threshold is 0, broadcast disabled"); return Ok(plan); } - let Some(hash_join) = plan.downcast_ref::() else { - return Ok(plan); - }; debug!( "broadcast check: evaluating HashJoinExec mode={:?} join_type={:?} threshold={threshold_bytes}", hash_join.partition_mode(), @@ -361,6 +383,22 @@ impl DefaultDistributedPlanner { right_under }; + // A CollectLeft join broadcasts the build (left) side to every probe + // task, which is only correct for join types that never emit rows on + // behalf of the build side. The build side is the left input of the + // resulting join, so check the join type after the swap is applied. + let promoted_join_type = if swap { + hash_join.join_type().swap() + } else { + *hash_join.join_type() + }; + if !collect_left_broadcast_safe(promoted_join_type) { + debug!( + "broadcast check: join type {promoted_join_type:?} is not broadcast-safe, skipping promotion" + ); + return Ok(plan); + } + debug!( "broadcast check: promoting to CollectLeft (left_under={left_under}, right_under={right_under}, swap={swap})" ); @@ -403,6 +441,36 @@ impl DefaultDistributedPlanner { vec![new_left, new_right], )?) } + + /// Rewrites a `HashJoinExec(CollectLeft)` as an equivalent + /// `HashJoinExec(Partitioned)`, hash-partitioning both inputs on the join + /// keys so the join is executed as a shuffle join rather than by + /// broadcasting the build side. Used to undo an unsafe `CollectLeft` + /// promotion that DataFusion's `JoinSelection` applied before the + /// distributed planner ran. + fn demote_collect_left_to_partitioned( + hash_join: &HashJoinExec, + config: &ConfigOptions, + ) -> Result> { + let partitions = config.execution.target_partitions; + let left_keys: Vec<_> = + hash_join.on().iter().map(|(l, _)| Arc::clone(l)).collect(); + let right_keys: Vec<_> = + hash_join.on().iter().map(|(_, r)| Arc::clone(r)).collect(); + let new_left: Arc = Arc::new(RepartitionExec::try_new( + hash_join.left().clone(), + Partitioning::Hash(left_keys, partitions), + )?); + let new_right: Arc = Arc::new(RepartitionExec::try_new( + hash_join.right().clone(), + Partitioning::Hash(right_keys, partitions), + )?); + Ok(hash_join + .builder() + .with_partition_mode(PartitionMode::Partitioned) + .with_new_children(vec![new_left, new_right])? + .build_exec()?) + } } fn create_unresolved_shuffle( @@ -888,7 +956,7 @@ order by async fn distributed_broadcast_join_plan() -> Result<(), BallistaError> { use datafusion::physical_plan::joins::PartitionMode; - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1)?; + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -936,7 +1004,7 @@ order by -> Result<(), BallistaError> { use datafusion::physical_plan::joins::PartitionMode; - let (ctx, options) = make_broadcast_test_ctx(0, 1)?; + let (ctx, options) = make_broadcast_test_ctx(0, 1, true)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -974,6 +1042,243 @@ order by Ok(()) } + // Across every expressible join type and both build-side orientations, any + // CollectLeft join the static planner promotes must be broadcast-safe: a + // CollectLeft join replicates the build (left) side to every probe task, so + // its (post-swap) join type must be driven by the probe side. `small` is + // under the threshold and `big` is not. + #[tokio::test] + async fn distributed_broadcast_only_promotes_safe_join_types() + -> Result<(), BallistaError> { + use crate::physical_optimizer::join_selection::collect_left_broadcast_safe; + use datafusion::physical_plan::joins::PartitionMode; + + let sqls = [ + "select * from big join small on big.k = small.k", + "select * from small join big on small.k = big.k", + "select * from big left join small on big.k = small.k", + "select * from small left join big on small.k = big.k", + "select * from big right join small on big.k = small.k", + "select * from small right join big on small.k = big.k", + "select * from big full join small on big.k = small.k", + "select * from small full join big on small.k = big.k", + ]; + + for sql in sqls { + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?; + let plan = ctx.sql(sql).await?.into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = planner.plan_query_stages( + &job_uuid.to_string().into(), + plan, + &options, + )?; + + for stage in &stages { + let mut walker: Vec> = + vec![stage.clone() as Arc]; + while let Some(node) = walker.pop() { + if let Some(hj) = node.downcast_ref::() + && *hj.partition_mode() == PartitionMode::CollectLeft + { + assert!( + collect_left_broadcast_safe(*hj.join_type()), + "unsafe CollectLeft join_type={:?} promoted for `{sql}`", + hj.join_type() + ); + } + walker.extend(node.children().iter().map(|c| (*c).clone())); + } + } + } + + Ok(()) + } + + // When DataFusion's threshold is non-zero, DataFusion's own `JoinSelection` + // runs during `create_physical_plan` and can stamp `CollectLeft` on a LEFT + // join whose small side is already on the left (no swap, no join-type + // check). The distributed planner must demote such an unsafe `CollectLeft` + // join back to a partitioned join rather than broadcasting the outer side. + #[tokio::test] + async fn df_promoted_left_join_must_not_broadcast() -> Result<(), BallistaError> { + use ballista_core::extension::SessionConfigExt; + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::common::JoinType; + use datafusion::physical_plan::joins::PartitionMode; + use datafusion::prelude::SessionConfig; + + let big_schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let small_schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("w", DataType::Int32, false), + ])); + let big_batch = RecordBatch::try_new( + big_schema, + vec![ + Arc::new(Int32Array::from((0..10_000).collect::>())), + Arc::new(Int32Array::from((0..10_000).collect::>())), + ], + ) + .map_err(|e| BallistaError::General(e.to_string()))?; + let small_batch = RecordBatch::try_new( + small_schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(Int32Array::from(vec![10, 20])), + ], + ) + .map_err(|e| BallistaError::General(e.to_string()))?; + + // Match the PR's production session config: DF and Ballista thresholds + // both at 10 MB so DF's `JoinSelection` can promote on its own. + let session_config = SessionConfig::new() + .with_target_partitions(2) + .set_bool("datafusion.optimizer.prefer_hash_join", true) + .set_usize( + "datafusion.optimizer.hash_join_single_partition_threshold", + 10 * 1024 * 1024, + ) + .with_ballista_broadcast_join_threshold_bytes(10 * 1024 * 1024); + let ctx = datafusion::prelude::SessionContext::new_with_config(session_config); + ctx.register_batch("big", big_batch)?; + ctx.register_batch("small", small_batch)?; + let options = ctx.state().config().options().clone(); + + let plan = ctx + .sql("select * from small left join big on small.k = big.k") + .await? + .into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = + planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?; + + for stage in &stages { + let mut walker: Vec> = + vec![stage.clone() as Arc]; + while let Some(node) = walker.pop() { + if let Some(hj) = node.downcast_ref::() { + assert!( + !(matches!(hj.join_type(), JoinType::Left) + && *hj.partition_mode() == PartitionMode::CollectLeft), + "LEFT join must not be CollectLeft (would broadcast the \ + outer side across probe tasks)" + ); + } + if let Some(unresolved) = node.downcast_ref::() { + assert!( + !unresolved.broadcast, + "no broadcast reader expected for an unsafe LEFT join" + ); + } + walker.extend(node.children().iter().map(|c| (*c).clone())); + } + } + Ok(()) + } + + // A LEFT join with the small side on the left builds (broadcasts) the left + // side and emits a null-padded row for every unmatched left row; each probe + // task would emit those independently. The guard must keep it repartitioned + // even though the small side is under the broadcast threshold. + #[tokio::test] + async fn distributed_left_join_small_build_not_broadcast() -> Result<(), BallistaError> + { + use datafusion::physical_plan::joins::PartitionMode; + + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?; + + let df = ctx + .sql("select * from small left join big on small.k = big.k") + .await?; + let plan = df.into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = + planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?; + + for stage in &stages { + let mut walker: Vec> = + vec![stage.clone() as Arc]; + while let Some(node) = walker.pop() { + if let Some(hj) = node.downcast_ref::() { + assert_ne!( + *hj.partition_mode(), + PartitionMode::CollectLeft, + "LEFT join with broadcast build side must not be CollectLeft" + ); + } + if let Some(unresolved) = node.downcast_ref::() { + assert!( + !unresolved.broadcast, + "no broadcast reader expected for unsafe LEFT join" + ); + } + walker.extend(node.children().iter().map(|c| (*c).clone())); + } + } + + Ok(()) + } + + // The static planner only promotes `HashJoinExec`. With prefer_hash_join + // disabled the join plans as a `SortMergeJoinExec`, so no broadcast happens + // even though the build side is under the threshold. + #[tokio::test] + async fn distributed_no_broadcast_when_sort_merge_join() -> Result<(), BallistaError> + { + use datafusion::physical_plan::joins::PartitionMode; + + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false)?; + + let df = ctx + .sql("select count(*) from big join small on big.k = small.k") + .await?; + let plan = df.into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = + planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?; + + for stage in &stages { + let mut walker: Vec> = + vec![stage.clone() as Arc]; + while let Some(node) = walker.pop() { + if let Some(hj) = node.downcast_ref::() { + assert_ne!( + *hj.partition_mode(), + PartitionMode::CollectLeft, + "no CollectLeft expected when joins are sort-merge" + ); + } + if let Some(unresolved) = node.downcast_ref::() { + assert!( + !unresolved.broadcast, + "no broadcast reader expected for sort-merge join" + ); + } + walker.extend(node.children().iter().map(|c| (*c).clone())); + } + } + + Ok(()) + } + #[tokio::test] async fn distributed_broadcast_join_plan_multi_partition_build() -> Result<(), BallistaError> { @@ -981,7 +1286,7 @@ order by // broadcast build stage has 3 input partitions and writes 3 shuffle // files. The broadcast UnresolvedShuffleExec must report // upstream_partition_count = 3. - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024 * 1024, 3)?; + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024 * 1024, 3, true)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -1273,6 +1578,7 @@ order by fn make_broadcast_test_ctx( threshold_bytes: usize, small_partitions: usize, + prefer_hash_join: bool, ) -> Result< ( datafusion::prelude::SessionContext, @@ -1320,6 +1626,7 @@ order by let session_config = SessionConfig::new() .with_target_partitions(2) + .set_bool("datafusion.optimizer.prefer_hash_join", prefer_hash_join) .set_usize( "datafusion.optimizer.hash_join_single_partition_threshold", 0, diff --git a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs index 2bac00644b..c7d22ae8aa 100644 --- a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs +++ b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs @@ -32,7 +32,9 @@ use datafusion::{ use log::debug; use std::sync::Arc; +use crate::physical_optimizer::join_selection::collect_left_broadcast_safe; use crate::state::aqe::execution_plan::ExchangeExec; +use crate::state::aqe::optimizer_rule::join_selection::SelectJoinRule; /// has children of this join been /// repartitioned @@ -217,7 +219,7 @@ impl DynamicJoinSelectionExec { let threshold_collect_left_join_rows = config.optimizer.hash_join_single_partition_threshold_rows; - let partition_mode = if Self::supports_collect_by_thresholds( + let under_threshold = Self::supports_collect_by_thresholds( self.left.as_ref(), threshold_collect_left_join_bytes, threshold_collect_left_join_rows, @@ -225,12 +227,31 @@ impl DynamicJoinSelectionExec { self.right.as_ref(), threshold_collect_left_join_bytes, threshold_collect_left_join_rows, - ) { - PartitionMode::CollectLeft + ); + + // The resolver collects the smaller side onto the build (left) input, + // swapping the join type when the right side is the smaller one (the + // `swap_inputs` calls in `SelectJoinRule`). A `CollectLeft` join then + // broadcasts that build side to every probe task, which is only correct + // for join types that never emit rows on behalf of the build side. Use + // the same swap decision as the resolver to determine the resulting join + // type and skip the broadcast when it would be unsafe. + let build_side_join_type = if SelectJoinRule::supports_swap_join_order( + self.left.as_ref(), + self.right.as_ref(), + )? { + self.join_type.swap() } else { - PartitionMode::Partitioned + self.join_type }; + let partition_mode = + if under_threshold && collect_left_broadcast_safe(build_side_join_type) { + PartitionMode::CollectLeft + } else { + PartitionMode::Partitioned + }; + let stats_left = self.left.partition_statistics(None)?; let stats_right = self.right.partition_statistics(None)?; @@ -424,12 +445,114 @@ mod tests { use super::*; use crate::state::aqe::execution_plan::ExchangeExec; use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::common::{ColumnStatistics, Statistics, stats::Precision}; use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::expressions::Column; + use datafusion::physical_plan::test::exec::StatisticsExec; fn test_schema() -> Arc { Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)])) } + fn stats_exec(num_rows: usize) -> Arc { + Arc::new(StatisticsExec::new( + Statistics { + num_rows: Precision::Inexact(num_rows), + total_byte_size: Precision::Inexact(num_rows * 16), + column_statistics: vec![ColumnStatistics::new_unknown()], + }, + Schema::new(vec![Field::new("k", DataType::Int32, false)]), + )) + } + + /// Run the resolver's join-strategy decision for `join_type` with `left_rows` + /// and `right_rows` on the two sides, both small enough to be collectable. + fn actual_join_for( + join_type: JoinType, + left_rows: usize, + right_rows: usize, + prefer_hash_join: bool, + ) -> JoinSelectionAction { + let left = stats_exec(left_rows); + let right = stats_exec(right_rows); + let on: JoinOn = + vec![(Arc::new(Column::new("k", 0)), Arc::new(Column::new("k", 0)))]; + let dj = DynamicJoinSelectionExec { + properties: Arc::clone(left.properties()), + left, + right, + on, + filter: None, + join_type, + projection: None, + null_equality: NullEquality::NullEqualsNothing, + selection_state: JoinInputState::Unknown, + null_aware: false, + plan_id: 0, + }; + let mut config = ConfigOptions::new(); + config.optimizer.prefer_hash_join = prefer_hash_join; + config.optimizer.hash_join_single_partition_threshold = 10 * 1024 * 1024; + config.optimizer.hash_join_single_partition_threshold_rows = 1_000_000; + dj.to_actual_join(&config).unwrap() + } + + fn is_collected(action: &JoinSelectionAction) -> bool { + matches!( + action, + JoinSelectionAction::CollectLeft(_) | JoinSelectionAction::LateCollectLeft(_) + ) + } + + // With equal-sized sides (no swap) the resolver must collect a small build + // side into a CollectLeft broadcast only for join types whose output is + // driven by the probe side; everything that emits build-side rows must be + // repartitioned instead. `prefer_hash_join` only selects the repartitioned + // fallback (hash vs sort-merge), so the broadcast-safety decision must be + // identical under both settings. + #[test] + fn to_actual_join_collects_only_broadcast_safe_join_types() { + for prefer_hash_join in [true, false] { + for join_type in [ + JoinType::Inner, + JoinType::Left, + JoinType::Right, + JoinType::Full, + JoinType::LeftSemi, + JoinType::RightSemi, + JoinType::LeftAnti, + JoinType::RightAnti, + JoinType::LeftMark, + JoinType::RightMark, + ] { + let action = actual_join_for(join_type, 100, 100, prefer_hash_join); + assert_eq!( + is_collected(&action), + collect_left_broadcast_safe(join_type), + "join_type {join_type:?} (prefer_hash_join={prefer_hash_join}): collected={}, expected safe={}", + is_collected(&action), + collect_left_broadcast_safe(join_type), + ); + } + } + } + + // Safety is evaluated on the join type *after* the build-side swap. A LEFT + // join with the smaller side on the right swaps to a (safe) RIGHT join and is + // collected; a RIGHT join with the smaller side on the right swaps to an + // (unsafe) LEFT join and must be repartitioned. + #[test] + fn to_actual_join_uses_post_swap_join_type() { + assert!( + is_collected(&actual_join_for(JoinType::Left, 1000, 10, true)), + "LEFT with small right swaps to a safe RIGHT join and should collect" + ); + assert!( + !is_collected(&actual_join_for(JoinType::Right, 1000, 10, true)), + "RIGHT with small right swaps to an unsafe LEFT join and must repartition" + ); + } + /// Constructs a minimal `DynamicJoinSelectionExec` around the given children. /// Properties are borrowed from the left child — correct enough for unit tests /// that only exercise `children_resolved`. diff --git a/ballista/scheduler/src/state/aqe/test/join_selection.rs b/ballista/scheduler/src/state/aqe/test/join_selection.rs index 7dba88e81f..b652ca7d30 100644 --- a/ballista/scheduler/src/state/aqe/test/join_selection.rs +++ b/ballista/scheduler/src/state/aqe/test/join_selection.rs @@ -141,6 +141,38 @@ async fn test_hash_join_two_tables_coalesce() -> datafusion::common::Result<()> Ok(()) } +// A `CollectLeft` join broadcasts (replicates) the build/left side to every +// probe task, each of which sees only its slice of the probe/right side. For a +// LEFT join that emits a null-padded row for every unmatched left row, each +// task would emit those rows independently, producing duplicate/spurious rows +// (apache/datafusion-ballista#1055). The resolver must therefore keep an +// unsafe-to-broadcast join type repartitioned instead of collecting it. +#[tokio::test] +async fn test_left_join_not_collected_left() -> datafusion::common::Result<()> { + let ctx = make_ctx(true); + register_2tables(&ctx); + + let lp = ctx + .sql("SELECT t1.id, t2.val FROM t1 LEFT JOIN t2 ON t1.id = t2.id") + .await + .unwrap() + .into_optimized_plan() + .unwrap(); + + let planner = AdaptivePlanner::try_new(&ctx, &lp, "test_job".to_owned()).await?; + let plan = planner.current_plan(); + assert_plan!(plan, @ r" + AdaptiveDatafusionExec: is_final=false, plan_id=3, stage_id=pending, stage_resolved=false + ProjectionExec: expr=[id@0 as id, val@2 as val] + DynamicJoinSelectionExec: plan_id=0, join_type=Left, on=[(id@0, id@0)] repartitioned=true + ExchangeExec: partitioning=Hash([id@0], 4), plan_id=1, stage_id=pending, stage_resolved=false + DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1] + ExchangeExec: partitioning=Hash([id@0], 4), plan_id=2, stage_id=pending, stage_resolved=false + DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1] + "); + Ok(()) +} + #[tokio::test] async fn test_hash_join_two_tables_repartition() -> datafusion::common::Result<()> { let ctx = make_ctx_without_collect_left(true); From 24f0e4a9651db083cc071c57be6eb17271f91319 Mon Sep 17 00:00:00 2001 From: Jogesh A Dinavahi Date: Wed, 1 Jul 2026 19:12:49 -0500 Subject: [PATCH 017/107] bump pyballista to df54 (#1919) --- python/Cargo.lock | 834 +++++++++++++++++++----------------------- python/Cargo.toml | 6 +- python/pyproject.toml | 2 +- python/uv.lock | 33 +- 4 files changed, 396 insertions(+), 479 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index dc1c986dc8..f4e556ce28 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -2,54 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "abi_stable" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" -dependencies = [ - "abi_stable_derive", - "abi_stable_shared", - "const_panic", - "core_extensions", - "crossbeam-channel", - "generational-arena", - "libloading", - "lock_api", - "parking_lot", - "paste", - "repr_offset", - "rustc_version", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "abi_stable_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" -dependencies = [ - "abi_stable_shared", - "as_derive_utils", - "core_extensions", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", - "typed-arena", -] - -[[package]] -name = "abi_stable_shared" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" -dependencies = [ - "core_extensions", -] - [[package]] name = "adler2" version = "2.0.1" @@ -115,35 +67,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest 0.10.7", - "liblzma", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.4", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum", - "strum_macros", - "thiserror", - "uuid", - "zstd", -] - [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -229,6 +152,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-avro" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap", + "liblzma", + "rand 0.9.4", + "serde", + "serde_json", + "snap", + "strum_macros", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" version = "58.3.0" @@ -440,18 +387,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "as_derive_utils" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" -dependencies = [ - "core_extensions", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "async-compression" version = "0.4.42" @@ -469,9 +404,6 @@ name = "async-ffi" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" -dependencies = [ - "abi_stable", -] [[package]] name = "async-trait" @@ -481,7 +413,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -582,7 +514,7 @@ dependencies = [ "datafusion-proto", "datafusion-proto-common", "futures", - "itertools", + "itertools 0.15.0", "log", "md-5 0.11.0", "parking_lot", @@ -642,7 +574,7 @@ dependencies = [ "futures", "http", "insta", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -653,7 +585,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic", - "tower-http", + "tower-http 0.7.0", "uuid", ] @@ -674,7 +606,6 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] [[package]] @@ -703,7 +634,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", + "cpufeatures", ] [[package]] @@ -724,31 +655,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "bon" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "brotli" version = "8.0.2" @@ -834,7 +740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "rand_core 0.10.1", ] @@ -928,15 +834,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "const_panic" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" -dependencies = [ - "typewit", -] - [[package]] name = "constant_time_eq" version = "0.4.2" @@ -960,37 +857,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "core_extensions" -version = "1.5.4" +name = "cpufeatures" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ - "core_extensions_proc_macros", + "libc", ] [[package]] -name = "core_extensions_proc_macros" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" - -[[package]] -name = "cpufeatures" -version = "0.2.17" +name = "crc" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ - "libc", + "crc-catalog", ] [[package]] -name = "cpufeatures" -version = "0.3.0" +name = "crc-catalog" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" @@ -1001,15 +889,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1072,40 +951,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "darling" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" -dependencies = [ - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", -] - -[[package]] -name = "darling_macro" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.117", -] - [[package]] name = "dashmap" version = "6.2.1" @@ -1122,14 +967,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "bzip2", "chrono", "datafusion-catalog", @@ -1160,14 +1004,13 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "itertools", + "indexmap", + "itertools 0.14.0", "liblzma", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -1178,9 +1021,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" dependencies = [ "arrow", "async-trait", @@ -1194,7 +1037,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -1203,9 +1046,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" dependencies = [ "arrow", "async-trait", @@ -1219,42 +1062,42 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", ] [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" dependencies = [ - "ahash", - "apache-avro", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "libc", "log", "object_store", "parquet", - "paste", "recursive", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" dependencies = [ "futures", "log", @@ -1263,9 +1106,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-compression", @@ -1285,10 +1128,11 @@ dependencies = [ "flate2", "futures", "glob", - "itertools", + "itertools 0.14.0", "liblzma", "log", "object_store", + "parking_lot", "rand 0.9.4", "tokio", "tokio-util", @@ -1298,9 +1142,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" dependencies = [ "arrow", "arrow-ipc", @@ -1315,36 +1159,35 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.14.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-avro" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" +checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", "datafusion-common", "datafusion-datasource", - "datafusion-physical-expr-common", + "datafusion-physical-expr-adapter", "datafusion-physical-plan", "datafusion-session", "futures", - "num-traits", "object_store", ] [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" dependencies = [ "arrow", "async-trait", @@ -1365,9 +1208,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" dependencies = [ "arrow", "async-trait", @@ -1382,16 +1225,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", @@ -1401,6 +1243,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -1409,7 +1252,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -1419,20 +1262,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -1448,11 +1290,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -1462,8 +1305,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", - "itertools", - "paste", + "itertools 0.14.0", "recursive", "serde_json", "sqlparser", @@ -1471,28 +1313,27 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" dependencies = [ "arrow", "datafusion-common", "indexmap", - "itertools", - "paste", + "itertools 0.14.0", ] [[package]] name = "datafusion-ffi" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c" +checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152" dependencies = [ - "abi_stable", "arrow", "arrow-schema", "async-ffi", "async-trait", + "chrono", "datafusion-catalog", "datafusion-common", "datafusion-datasource", @@ -1501,22 +1342,25 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-proto", "datafusion-proto-common", "datafusion-session", "futures", + "libloading", "log", "prost", "semver", + "stabby", "tokio", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" dependencies = [ "arrow", "arrow-buffer", @@ -1531,26 +1375,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", - "itertools", + "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", "rand 0.9.4", "regex", "sha2", - "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -1560,19 +1403,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -1581,9 +1423,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" dependencies = [ "arrow", "arrow-ord", @@ -1597,34 +1439,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", - "itertools", + "hashbrown 0.17.1", + "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" dependencies = [ "arrow", "datafusion-common", @@ -1635,14 +1477,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1650,20 +1491,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.117", + "syn", ] [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" dependencies = [ "arrow", "chrono", @@ -1672,7 +1513,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap", - "itertools", + "itertools 0.14.0", "log", "recursive", "regex", @@ -1681,11 +1522,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -1693,11 +1533,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "recursive", "tokio", @@ -1705,9 +1544,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" dependencies = [ "arrow", "datafusion-common", @@ -1715,31 +1554,31 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" dependencies = [ "arrow", "datafusion-common", @@ -1750,18 +1589,19 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools", + "itertools 0.14.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -1776,9 +1616,9 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", - "itertools", + "itertools 0.14.0", "log", "num-traits", "parking_lot", @@ -1788,9 +1628,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" dependencies = [ "arrow", "chrono", @@ -1811,14 +1651,13 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost", - "rand 0.9.4", ] [[package]] name = "datafusion-proto-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" dependencies = [ "arrow", "datafusion-common", @@ -1827,9 +1666,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" dependencies = [ "arrow", "datafusion-common", @@ -1838,24 +1677,25 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", "log", ] [[package]] name = "datafusion-python" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c25cc11d7d6c3916cb97d7b967110b0679646edadec7f242f720fe64288117b" +checksum = "4b5da6aab44da83d488e6fb05855f5cd4480c7ab6b44f164166b495cba9fbbb8" dependencies = [ "arrow", "arrow-select", "async-trait", + "chrono", "cstr", "datafusion", "datafusion-ffi", "datafusion-proto", "datafusion-python-util", + "datafusion-spark", "futures", "log", "object_store", @@ -1874,13 +1714,14 @@ dependencies = [ [[package]] name = "datafusion-python-util" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d766b71c573229f5c9ffd32afd5678d1d2cbff1ff87cb3ef1d722aed7bed7db3" +checksum = "ba491c15adfebc9ae53aa11f65231814b5199627e9815e2fb62d2f81f4a17fb2" dependencies = [ "arrow", "datafusion", "datafusion-ffi", + "datafusion-proto", "prost", "pyo3", "tokio", @@ -1888,9 +1729,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" dependencies = [ "async-trait", "datafusion-common", @@ -1900,11 +1741,40 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "datafusion-spark" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "crc32fast", + "datafusion-catalog", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-functions-nested", + "log", + "num-traits", + "percent-encoding", + "rand 0.9.4", + "serde_json", + "sha1", + "sha2", + "twox-hash", + "url", +] + [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" dependencies = [ "arrow", "bigdecimal", @@ -1949,7 +1819,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2102,7 +1972,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2134,15 +2004,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generational-arena" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" -dependencies = [ - "cfg-if", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2248,21 +2109,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", ] -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - [[package]] name = "heck" version = "0.5.0" @@ -2521,12 +2376,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "1.1.0" @@ -2593,6 +2442,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2698,12 +2556,12 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" -version = "0.7.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if", - "winapi", + "windows-link", ] [[package]] @@ -2866,7 +2724,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -2944,7 +2801,7 @@ dependencies = [ "httparse", "humantime", "hyper", - "itertools", + "itertools 0.14.0", "md-5 0.10.6", "parking_lot", "percent-encoding", @@ -3104,7 +2961,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3150,7 +3007,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", ] [[package]] @@ -3179,7 +3045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -3189,7 +3055,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.117", + "syn", "tempfile", ] @@ -3200,10 +3066,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3299,6 +3165,7 @@ version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ + "python3-dll-a", "target-lexicon", ] @@ -3332,7 +3199,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3345,14 +3212,17 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.117", + "syn", ] [[package]] -name = "quad-rand" -version = "0.2.3" +name = "python3-dll-a" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" +checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b" +dependencies = [ + "cc", +] [[package]] name = "quick-xml" @@ -3503,7 +3373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3538,27 +3408,12 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-lite" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" - [[package]] name = "regex-syntax" version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "repr_offset" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" -dependencies = [ - "tstr", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -3592,7 +3447,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -3771,16 +3626,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -3798,7 +3643,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3837,17 +3682,34 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.3", +] + [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", + "cpufeatures", + "digest 0.11.3", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "shlex" version = "1.3.0" @@ -3918,9 +3780,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -3935,7 +3797,41 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", +] + +[[package]] +name = "stabby" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" +dependencies = [ + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -3957,28 +3853,16 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" - [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3987,17 +3871,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -4026,7 +3899,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4079,7 +3952,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4152,7 +4025,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4190,6 +4063,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.14.6" @@ -4228,7 +4131,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4253,7 +4156,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.117", + "syn", "tempfile", "tonic-build", ] @@ -4295,6 +4198,21 @@ dependencies = [ "url", ] +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "bitflags", + "bytes", + "http", + "percent-encoding", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -4327,7 +4245,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4345,32 +4263,14 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tstr" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" -dependencies = [ - "tstr_proc_macros", -] - -[[package]] -name = "tstr_proc_macros" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" - [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" - -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +dependencies = [ + "rand 0.9.4", +] [[package]] name = "typenum" @@ -4378,12 +4278,6 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" -[[package]] -name = "typewit" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" - [[package]] name = "unicase" version = "2.9.0" @@ -4446,7 +4340,6 @@ checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", - "serde_core", "wasm-bindgen", ] @@ -4541,7 +4434,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -4705,7 +4598,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4716,7 +4609,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4918,6 +4811,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -4954,7 +4856,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4970,7 +4872,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5037,7 +4939,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -5058,7 +4960,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5078,7 +4980,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -5118,7 +5020,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] diff --git a/python/Cargo.toml b/python/Cargo.toml index 8531c0f451..5b0b14cbbf 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -35,9 +35,9 @@ ballista = { path = "../ballista/client", version = "53.0.0" } ballista-core = { path = "../ballista/core", version = "53.0.0" } ballista-executor = { path = "../ballista/executor", version = "53.0.0", default-features = false } ballista-scheduler = { path = "../ballista/scheduler", version = "53.0.0", default-features = false } -datafusion = { version = "53", features = ["avro"] } -datafusion-proto = { version = "53" } -datafusion-python = { version = "53", default-features = false } +datafusion = { version = "54", features = ["avro"] } +datafusion-proto = { version = "54" } +datafusion-python = { version = "54", default-features = false } log = { version = "0.4" } pyo3 = { version = "0.28", features = ["extension-module", "abi3", "abi3-py310"] } pyo3-log = "0.13" diff --git a/python/pyproject.toml b/python/pyproject.toml index bbf077fd8c..b3a24c9cfc 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -41,7 +41,7 @@ classifiers = [ ] dependencies = [ "pyarrow>=21.0.0", - "datafusion==53", + "datafusion==54", "typing-extensions;python_version<'3.13'", ] dynamic = ["version"] diff --git a/python/uv.lock b/python/uv.lock index 9852555f5f..5935482cf4 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -52,7 +52,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "datafusion", specifier = "==53" }, + { name = "datafusion", specifier = "==54" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=8.0.0" }, { name = "pyarrow", specifier = ">=21.0.0" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, @@ -250,6 +250,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, ] +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -326,19 +335,25 @@ wheels = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "54.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "cloudpickle" }, { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" }, - { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" }, - { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" }, - { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" }, + { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" }, + { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" }, + { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" }, + { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" }, + { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" }, + { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" }, + { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" }, ] [[package]] From a8b3c79c80f7f1c0aa862ed1a76dee7a1ac67265 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 2 Jul 2026 10:50:56 -0600 Subject: [PATCH 018/107] feat: broadcast small build side of SortMergeJoinExec in the static planner (#1904) * feat(scheduler): broadcast small build side of SortMergeJoinExec Add ballista.optimizer.broadcast_sort_merge_join_enabled (default false). When enabled, the static distributed planner converts a SortMergeJoinExec whose smaller side is under broadcast_join_threshold_bytes into a CollectLeft hash join and lowers the build side as a broadcast stage, reusing the existing hash-join broadcast path. Redundant input sorts are dropped during conversion. TPC-H SF10 (AQE off): ~16% faster across the suite (q8 1.6x, q11 1.5x, q2 1.4x), identical row counts. * feat(config): default broadcast_sort_merge_join_enabled to true Enable the SortMergeJoinExec broadcast conversion by default so small build sides are broadcast without opt-in (~16% on TPC-H SF10, AQE off). Disable the flag explicitly in the sort-merge-join client test so it still exercises the plain SortMergeJoinExec execution path. * test: assert SMJ broadcast plans via string snapshots Rewrite the three SortMergeJoinExec broadcast tests to assert the stage plan as a string via the assert_plan! macro instead of walking the plan tree with manual downcasts, making the expected plan explicit and the tests easier to read. --- ballista/client/tests/context_checks.rs | 8 + ballista/core/src/config.rs | 25 +++ ballista/scheduler/src/planner.rs | 230 +++++++++++++++++++++--- 3 files changed, 242 insertions(+), 21 deletions(-) diff --git a/ballista/client/tests/context_checks.rs b/ballista/client/tests/context_checks.rs index 75d17f37ff..dc0576d9d6 100644 --- a/ballista/client/tests/context_checks.rs +++ b/ballista/client/tests/context_checks.rs @@ -960,6 +960,14 @@ mod supported { ctx: SessionContext, test_data: String, ) -> datafusion::error::Result<()> { + // Exercise the plain sort-merge-join execution path: disable the + // default SortMergeJoinExec broadcast conversion so the join stays a + // SortMergeJoinExec in the distributed plan. + ctx.sql("SET ballista.optimizer.broadcast_sort_merge_join_enabled = false") + .await? + .collect() + .await?; + ctx.register_parquet( "t0", &format!("{test_data}/alltypes_plain.parquet"), diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index fdad8d7f8b..9a9bc6d7f4 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -86,6 +86,12 @@ pub const BALLISTA_SHUFFLE_SORT_BASED_MEMORY_LIMIT_PER_TASK_BYTES: &str = pub const BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES: &str = "ballista.optimizer.broadcast_join_threshold_bytes"; +/// Configuration key to enable broadcasting a small build side of a +/// `SortMergeJoinExec` by converting it to a `CollectLeft` hash join in the +/// static distributed planner. Enabled by default. +pub const BALLISTA_BROADCAST_SORT_MERGE_JOIN_ENABLED: &str = + "ballista.optimizer.broadcast_sort_merge_join_enabled"; + /// Configuration key to enable AQE coalesce-shuffle-partitions rule. /// Disabled by default — opt in when the workload benefits from larger /// downstream tasks more than from preserved parallelism. @@ -199,6 +205,12 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| Set to 0 to disable promotion.".to_string(), DataType::UInt64, Some((10 * 1024 * 1024).to_string())), + ConfigEntry::new(BALLISTA_BROADCAST_SORT_MERGE_JOIN_ENABLED.to_string(), + "Broadcast a small build side of a SortMergeJoinExec by converting it \ + to a CollectLeft hash join in the static distributed planner. \ + The build side must also fit under broadcast_join_threshold_bytes.".to_string(), + DataType::Boolean, + Some(true.to_string())), ConfigEntry::new(BALLISTA_CLIENT_PULL.to_string(), "Should client employ pull or push job tracking. In pull mode client will make a request to server in the loop, until job finishes. Pull mode is kept for legacy clients.".to_string(), DataType::Boolean, @@ -496,6 +508,13 @@ impl BallistaConfig { self.get_usize_setting(BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES) } + /// Returns whether broadcasting a small build side of a `SortMergeJoinExec` + /// (by converting it to a `CollectLeft` hash join) is enabled in the static + /// distributed planner. + pub fn broadcast_sort_merge_join_enabled(&self) -> bool { + self.get_bool_setting(BALLISTA_BROADCAST_SORT_MERGE_JOIN_ENABLED) + } + /// Returns whether the AQE coalesce-shuffle-partitions rule is enabled. pub fn coalesce_enabled(&self) -> bool { self.get_bool_setting(BALLISTA_COALESCE_ENABLED) @@ -741,4 +760,10 @@ mod tests { assert_eq!(16777216, config.grpc_client_max_message_size()); Ok(()) } + + #[test] + fn broadcast_sort_merge_join_enabled_by_default() { + let config = BallistaConfig::default(); + assert!(config.broadcast_sort_merge_join_enabled()); + } } diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 6538cb45bb..d5602f57be 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -37,8 +37,12 @@ use datafusion::config::ConfigOptions; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_optimizer::enforce_sorting::EnforceSorting; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; +use datafusion::physical_plan::joins::{ + HashJoinExec, HashJoinExecBuilder, PartitionMode, SortMergeJoinExec, +}; +use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::{ ExecutionPlan, Partitioning, with_new_children_if_necessary, @@ -272,10 +276,6 @@ impl DefaultDistributedPlanner { plan: Arc, config: &ConfigOptions, ) -> Result> { - let Some(hash_join) = plan.downcast_ref::() else { - return Ok(plan); - }; - // DataFusion's own `JoinSelection` may have already stamped // `CollectLeft` on this join (it does so for any build side under // `datafusion.optimizer.hash_join_single_partition_threshold`, without @@ -283,8 +283,11 @@ impl DefaultDistributedPlanner { // side to every probe task, which is only correct for probe-driven join // types. If the join type is not broadcast-safe, demote it back to a // partitioned (shuffle) join. This is a correctness guard, so it runs - // regardless of the Ballista broadcast threshold below. - if *hash_join.partition_mode() == PartitionMode::CollectLeft { + // regardless of the Ballista broadcast threshold below. A + // `SortMergeJoinExec` falls through to the SMJ-broadcast path below. + if let Some(hash_join) = plan.downcast_ref::() + && *hash_join.partition_mode() == PartitionMode::CollectLeft + { if collect_left_broadcast_safe(*hash_join.join_type()) { return Ok(plan); } @@ -306,6 +309,31 @@ impl DefaultDistributedPlanner { debug!("broadcast check: threshold is 0, broadcast disabled"); return Ok(plan); } + + // The candidate is a `HashJoinExec(Partitioned)`: either the plan itself, + // or a `SortMergeJoinExec` converted to one when SMJ broadcast is enabled. + // `converted` owns the converted join so `hash_join` can borrow from it, + // and the original `plan` is returned unchanged when no promotion happens. + let smj_broadcast_enabled = config + .extensions + .get::() + .map(|c| c.broadcast_sort_merge_join_enabled()) + .unwrap_or_else(|| { + BallistaConfig::default().broadcast_sort_merge_join_enabled() + }); + let converted: Option> = + match plan.downcast_ref::() { + Some(smj) if smj_broadcast_enabled => { + Some(convert_sort_merge_to_hash_join(smj)?) + } + _ => None, + }; + let hash_join: &HashJoinExec = + match (plan.downcast_ref::(), &converted) { + (Some(hj), _) => hj, + (None, Some(hj)) => hj.as_ref(), + (None, None) => return Ok(plan), + }; debug!( "broadcast check: evaluating HashJoinExec mode={:?} join_type={:?} threshold={threshold_bytes}", hash_join.partition_mode(), @@ -421,9 +449,24 @@ impl DefaultDistributedPlanner { ) }; - let promoted_join = promoted + // `swap_inputs` may wrap the join in a `ProjectionExec` to restore the + // original column order. Locate the `HashJoinExec` (bare or under that + // projection) so the build side can be coalesced, then re-wrap if needed. + let (join_node, projection): ( + Arc, + Option>, + ) = if promoted.downcast_ref::().is_some() { + (promoted.clone(), None) + } else if let Some(proj) = promoted.downcast_ref::() { + (proj.input().clone(), Some(promoted.clone())) + } else { + debug!("broadcast check: unexpected promoted plan shape, skipping"); + return Ok(plan); + }; + + let promoted_join = join_node .downcast_ref::() - .expect("promoted plan must still be a HashJoinExec"); + .expect("promoted join node must be a HashJoinExec"); let new_left: Arc = if promoted_join .left() .properties() @@ -436,10 +479,16 @@ impl DefaultDistributedPlanner { promoted_join.left().clone() }; let new_right = promoted_join.right().clone(); - Ok(with_new_children_if_necessary( - promoted, - vec![new_left, new_right], - )?) + let rebuilt_join = + with_new_children_if_necessary(join_node, vec![new_left, new_right])?; + + // Re-wrap in the projection if `swap_inputs` added one. The recursive + // lowering descends into the projection and broadcasts the build side of + // the inner `HashJoinExec(CollectLeft)`. + match projection { + Some(proj) => Ok(with_new_children_if_necessary(proj, vec![rebuilt_join])?), + None => Ok(rebuilt_join), + } } /// Rewrites a `HashJoinExec(CollectLeft)` as an equivalent @@ -473,6 +522,33 @@ impl DefaultDistributedPlanner { } } +/// Strips a top-level `SortExec`, returning its input. A `SortMergeJoinExec` +/// requires sorted inputs, but the broadcast `CollectLeft` hash join converted +/// from it does not, so the sort is dropped during conversion. +fn strip_sort(plan: Arc) -> Arc { + if let Some(sort) = plan.downcast_ref::() { + sort.input().clone() + } else { + plan + } +} + +/// Converts a `SortMergeJoinExec` into an equivalent `HashJoinExec(Partitioned)`, +/// dropping the now-redundant input sorts. The result is then evaluated by the +/// normal hash-join broadcast path, which promotes it to `CollectLeft` when a +/// side fits under the threshold. +fn convert_sort_merge_to_hash_join(smj: &SortMergeJoinExec) -> Result> { + let left = strip_sort(smj.left().clone()); + let right = strip_sort(smj.right().clone()); + let hash_join = + HashJoinExecBuilder::new(left, right, smj.on().to_vec(), smj.join_type()) + .with_filter(smj.filter().clone()) + .with_partition_mode(PartitionMode::Partitioned) + .with_null_equality(smj.null_equality) + .build()?; + Ok(Arc::new(hash_join)) +} + fn create_unresolved_shuffle( shuffle_writer: &dyn ShuffleWriter, ) -> Arc { @@ -645,6 +721,7 @@ pub(crate) fn create_shuffle_writer_with_config( #[cfg(test)] mod test { + use crate::assert_plan; use crate::planner::{DefaultDistributedPlanner, DistributedPlanner}; use crate::test_utils::datafusion_test_context; use ballista_core::error::BallistaError; @@ -956,7 +1033,7 @@ order by async fn distributed_broadcast_join_plan() -> Result<(), BallistaError> { use datafusion::physical_plan::joins::PartitionMode; - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?; + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true, false)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -999,12 +1076,116 @@ order by Ok(()) } + #[tokio::test] + async fn distributed_broadcast_sort_merge_join_plan() -> Result<(), BallistaError> { + // prefer_hash_join=false -> DataFusion plans a SortMergeJoinExec. + // broadcast_sort_merge_join_enabled=true -> the small build side is + // converted to a broadcast CollectLeft hash join: the SortMergeJoinExec + // and its input SortExecs are gone, and the build input is an + // UnresolvedShuffleExec with broadcast=true. + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false, true)?; + + let df = ctx + .sql("select count(*) from big join small on big.k = small.k") + .await?; + + let plan = df.into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = + planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?; + + // Stage 1 holds the join: a broadcast CollectLeft hash join, no + // SortMergeJoinExec and no SortExec. + assert_plan!(stages[1].as_ref(), @r" + ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + ProjectionExec: expr=[] + ProjectionExec: expr=[k@1 as k, k@0 as k] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 1 + DataSourceExec: partitions=1, partition_sizes=[1] + "); + + Ok(()) + } + + #[tokio::test] + async fn distributed_sort_merge_join_unchanged_when_flag_disabled() + -> Result<(), BallistaError> { + // SMJ planned (prefer_hash_join=false), flag OFF -> no conversion: the + // join stays a SortMergeJoinExec over sorted inputs. + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false, false)?; + + let df = ctx + .sql("select count(*) from big join small on big.k = small.k") + .await?; + let plan = df.into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = + planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?; + + // Stage 0 holds the join, still a SortMergeJoinExec over sorted inputs + // (no HashJoinExec, no broadcast UnresolvedShuffleExec). + assert_plan!(stages[0].as_ref(), @r" + ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + ProjectionExec: expr=[] + SortMergeJoinExec: join_type=Inner, on=[(k@0, k@0)] + SortExec: expr=[k@0 ASC], preserve_partitioning=[false] + DataSourceExec: partitions=1, partition_sizes=[1] + SortExec: expr=[k@0 ASC], preserve_partitioning=[false] + DataSourceExec: partitions=1, partition_sizes=[1] + "); + + Ok(()) + } + + #[tokio::test] + async fn distributed_sort_merge_join_unchanged_when_sides_too_large() + -> Result<(), BallistaError> { + // Flag ON but threshold tiny (1 byte) -> neither side qualifies, so the + // join stays a SortMergeJoinExec over sorted inputs. + let (ctx, options) = make_broadcast_test_ctx(1, 1, false, true)?; + + let df = ctx + .sql("select count(*) from big join small on big.k = small.k") + .await?; + let plan = df.into_optimized_plan()?; + let plan = ctx.state().create_physical_plan(&plan).await?; + + let mut planner = DefaultDistributedPlanner::new(); + let job_uuid = Uuid::new_v4(); + let stages = + planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?; + + // Stage 0 holds the join, still a SortMergeJoinExec over sorted inputs + // (no HashJoinExec, no broadcast UnresolvedShuffleExec). + assert_plan!(stages[0].as_ref(), @r" + ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + ProjectionExec: expr=[] + SortMergeJoinExec: join_type=Inner, on=[(k@0, k@0)] + SortExec: expr=[k@0 ASC], preserve_partitioning=[false] + DataSourceExec: partitions=1, partition_sizes=[1] + SortExec: expr=[k@0 ASC], preserve_partitioning=[false] + DataSourceExec: partitions=1, partition_sizes=[1] + "); + + Ok(()) + } + #[tokio::test] async fn distributed_join_plan_no_broadcast_when_threshold_zero() -> Result<(), BallistaError> { use datafusion::physical_plan::joins::PartitionMode; - let (ctx, options) = make_broadcast_test_ctx(0, 1, true)?; + let (ctx, options) = make_broadcast_test_ctx(0, 1, true, false)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -1065,7 +1246,8 @@ order by ]; for sql in sqls { - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?; + let (ctx, options) = + make_broadcast_test_ctx(10 * 1024 * 1024, 1, true, false)?; let plan = ctx.sql(sql).await?.into_optimized_plan()?; let plan = ctx.state().create_physical_plan(&plan).await?; @@ -1197,7 +1379,7 @@ order by { use datafusion::physical_plan::joins::PartitionMode; - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?; + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true, false)?; let df = ctx .sql("select * from small left join big on small.k = big.k") @@ -1242,7 +1424,7 @@ order by { use datafusion::physical_plan::joins::PartitionMode; - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false)?; + let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false, false)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -1286,7 +1468,8 @@ order by // broadcast build stage has 3 input partitions and writes 3 shuffle // files. The broadcast UnresolvedShuffleExec must report // upstream_partition_count = 3. - let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024 * 1024, 3, true)?; + let (ctx, options) = + make_broadcast_test_ctx(10 * 1024 * 1024 * 1024, 3, true, false)?; let df = ctx .sql("select count(*) from big join small on big.k = small.k") @@ -1579,6 +1762,7 @@ order by threshold_bytes: usize, small_partitions: usize, prefer_hash_join: bool, + smj_broadcast_enabled: bool, ) -> Result< ( datafusion::prelude::SessionContext, @@ -1626,12 +1810,16 @@ order by let session_config = SessionConfig::new() .with_target_partitions(2) - .set_bool("datafusion.optimizer.prefer_hash_join", prefer_hash_join) + .with_ballista_broadcast_join_threshold_bytes(threshold_bytes) .set_usize( "datafusion.optimizer.hash_join_single_partition_threshold", 0, ) - .with_ballista_broadcast_join_threshold_bytes(threshold_bytes); + .set_bool("datafusion.optimizer.prefer_hash_join", prefer_hash_join) + .set_bool( + "ballista.optimizer.broadcast_sort_merge_join_enabled", + smj_broadcast_enabled, + ); let ctx = datafusion::prelude::SessionContext::new_with_config(session_config); ctx.register_batch("big", big_batch)?; From b75201135e732eedbbc3bba80bc1405fbef734c7 Mon Sep 17 00:00:00 2001 From: jeadie Date: Fri, 3 Jul 2026 09:45:34 +1000 Subject: [PATCH 019/107] fix(executor): address PR review feedback on shared-semaphore poll_loop - Handle AcquireError from semaphore acquire/acquire_owned by mapping to BallistaError instead of panicking, since the semaphore may now be externally owned and closed by the caller - Assert that the provided semaphore has at least one permit on entry to fail fast rather than deadlock - Rewrite poll_loop doc comment to be public-facing: remove references to internal variable names and implementation types, document shared- semaphore over-commit semantics and sizing tradeoffs in behavioral terms, and add a # Panics section for the zero-permit case --- ballista/executor/src/execution_loop.rs | 54 +++++++++++++++++++------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index bc67540555..5ece368485 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -51,17 +51,33 @@ use tonic::codegen::{Body, Bytes, StdError}; /// Main execution loop that polls the scheduler for available tasks. /// -/// This function runs indefinitely, periodically asking the scheduler for -/// work. When tasks are received, they are executed on a dedicated thread -/// pool and results are reported back to the scheduler. +/// Runs indefinitely, periodically asking the scheduler for work. When tasks +/// are received they are executed concurrently and results are reported back +/// to the scheduler. /// -/// The loop respects the executor's concurrent task limit via a semaphore, -/// ensuring no more than the configured number of tasks run simultaneously. +/// Concurrency is bounded by a semaphore. Pass `available_task_slots` to +/// supply your own semaphore — useful for sharing a single concurrency limit +/// across multiple poll loops or for observing executor load from outside. +/// Pass `None` to have the loop create a semaphore sized to the executor's +/// configured task-slot count. /// -/// `available_task_slots`, when provided, lets the caller supply the semaphore -/// controlling task concurrency (e.g. to share it across poll loops or to -/// observe executor busy state via `available_permits()`). When `None`, a -/// semaphore sized to the executor's task slots is created internally. +/// **Shared semaphores**: when one semaphore is shared across loops that +/// connect to different schedulers, each scheduler independently sees the +/// current free capacity and may dispatch up to that many tasks. The semaphore +/// still caps total concurrent execution — tasks that cannot run immediately +/// wait for capacity — but both schedulers may over-commit relative to what +/// the semaphore can actually admit at once. This is intentional: the +/// semaphore acts as an execution throttle, not a reservation system. +/// +/// **Semaphore sizing**: if the provided semaphore allows more concurrent +/// tasks than the executor's thread pool has threads, excess admitted tasks +/// will queue behind running ones. The caller is responsible for sizing the +/// semaphore appropriately for their thread pool. +/// +/// # Panics +/// +/// Panics on startup if `available_task_slots` is a semaphore with zero +/// permits, which would cause the loop to deadlock immediately. pub async fn poll_loop( mut scheduler: SchedulerGrpcClient, executor: Arc, @@ -84,6 +100,10 @@ where let available_task_slots = available_task_slots.unwrap_or_else(|| { Arc::new(Semaphore::new(executor_specification.task_slots as usize)) }); + assert!( + available_task_slots.available_permits() > 0, + "available_task_slots semaphore must have at least one permit; passing a closed or zero-permit semaphore would deadlock the poll loop" + ); let (task_status_sender, mut task_status_receiver) = std::sync::mpsc::channel::(); @@ -94,7 +114,10 @@ where loop { // Wait for task slots to be available before asking for new work - let permit = available_task_slots.acquire().await.unwrap(); + let permit = available_task_slots + .acquire() + .await + .map_err(|_| BallistaError::Internal("task slot semaphore closed".to_string()))?; // Make the slot available again drop(permit); @@ -140,8 +163,15 @@ where let task_status_sender = task_status_sender.clone(); // Acquire a permit/slot for the task - let permit = - available_task_slots.clone().acquire_owned().await.unwrap(); + let permit = available_task_slots + .clone() + .acquire_owned() + .await + .map_err(|_| { + BallistaError::Internal( + "task slot semaphore closed".to_string(), + ) + })?; let start_exec_time = SystemTime::now() .duration_since(UNIX_EPOCH) From 84512cb5cd43b8be7ca257309810db2bae71f308 Mon Sep 17 00:00:00 2001 From: Jogesh A Dinavahi Date: Thu, 2 Jul 2026 23:06:27 -0500 Subject: [PATCH 020/107] Fix/python cargo lock drift (#1899) --- .github/dependabot.yml | 15 +++++++++++++++ .github/workflows/dependencies.yml | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fcd9460025..dd4929e8fb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,6 +32,21 @@ updates: update-types: ["version-update:semver-major"] - dependency-name: "sqlparser" update-types: ["version-update:semver-major"] + - package-ecosystem: cargo + directory: "/python" + schedule: + interval: daily + open-pull-requests-limit: 10 + target-branch: main + labels: [auto-dependencies] + ignore: + # arrow and datafusion are bumped manually + - dependency-name: "arrow*" + update-types: ["version-update:semver-major"] + - dependency-name: "datafusion*" + update-types: ["version-update:semver-major"] + - dependency-name: "sqlparser" + update-types: ["version-update:semver-major"] - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 65d267be98..1a816243f3 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -17,6 +17,9 @@ name: Dependencies +permissions: + contents: read + concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true @@ -53,3 +56,23 @@ jobs: run: | cd dev/msrvcheck cargo run + + python-lock: + name: python Cargo.lock up to date + runs-on: macos-latest + steps: + - uses: actions/checkout@v7.0.0 + with: + fetch-depth: 1 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-macos-builder + with: + rust-version: stable + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - name: Verify python/Cargo.lock is in sync with manifests + run: | + cd python + if ! cargo metadata --locked --format-version 1 >/dev/null; then + echo "::error::python/Cargo.lock is out of date with the manifests. Run 'cd python && cargo update' and commit the updated Cargo.lock." + exit 1 + fi From 6472c7f21ad1a824b123037b2f18669bd1538bca Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 3 Jul 2026 08:03:21 -0600 Subject: [PATCH 021/107] feat(scheduler): allow submitting an already-built physical plan (#1924) Add a physical_plan field to ExecuteQueryParams.query so clients can submit an ExecutionPlan directly, bypassing logical plan creation on the scheduler. DefaultDistributedPlanner::plan_query_stages already splits physical plans into shuffle stages at RepartitionExec(Hash) boundaries, so this unblocks distributing plans that contain custom/FFI operators with no logical-plan representation. Thread the decoded plan through the existing job-submission event path (SchedulerServer::submit_job -> QueryStageSchedulerEvent::JobQueued -> SchedulerState::submit_job -> TaskManager::submit_job) as a new SubmitPlan enum (Logical | Physical) instead of a bare LogicalPlan. Physical-plan jobs always use the static distributed planner and skip create_physical_plan/EXPLAIN handling entirely; AQE is not available for this path since it requires a logical plan to re-plan against. Add execute_physical_plan() in ballista-core as a client-side helper that mirrors DistributedQueryExec's submission flow (serialize with the physical extension codec, submit, poll status, fetch partitions). --- Cargo.lock | 1 + ballista/client/Cargo.toml | 1 + .../client/tests/physical_plan_submission.rs | 132 ++++++++++++++++++ ballista/core/proto/ballista.proto | 5 + .../src/execution_plans/distributed_query.rs | 86 ++++++++++++ ballista/core/src/execution_plans/mod.rs | 2 +- ballista/core/src/serde/generated/ballista.rs | 8 +- .../scheduler/src/scheduler_server/event.rs | 27 +++- .../scheduler/src/scheduler_server/grpc.rs | 40 +++++- .../scheduler/src/scheduler_server/mod.rs | 15 +- ballista/scheduler/src/state/mod.rs | 14 +- ballista/scheduler/src/state/task_manager.rs | 94 ++++++++----- ballista/scheduler/src/test_utils.rs | 14 +- 13 files changed, 377 insertions(+), 62 deletions(-) create mode 100644 ballista/client/tests/physical_plan_submission.rs diff --git a/Cargo.lock b/Cargo.lock index 369a967188..eec4cd1d77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1012,6 +1012,7 @@ dependencies = [ "tokio", "tonic", "url", + "uuid", ] [[package]] diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml index e8d4eb9900..f3d9d2e3e8 100644 --- a/ballista/client/Cargo.toml +++ b/ballista/client/Cargo.toml @@ -46,6 +46,7 @@ datafusion-proto = { workspace = true } env_logger = { workspace = true } rstest = { workspace = true } tempfile = { workspace = true } +uuid = { workspace = true } tonic = { workspace = true } [features] diff --git a/ballista/client/tests/physical_plan_submission.rs b/ballista/client/tests/physical_plan_submission.rs new file mode 100644 index 0000000000..140ecdf222 --- /dev/null +++ b/ballista/client/tests/physical_plan_submission.rs @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end test for submitting an already-built physical plan directly to a +//! Ballista scheduler, bypassing logical plan creation on the scheduler side. + +mod common; + +#[cfg(test)] +#[cfg(feature = "standalone")] +mod physical_plan_submission_tests { + use crate::common::setup_test_cluster; + use ballista_core::config::BallistaConfig; + use ballista_core::execution_plans::execute_physical_plan; + use ballista_core::extension::SessionConfigExt; + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::common::test_util::batches_to_sort_string; + use datafusion::datasource::MemTable; + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::{self, ExecutionPlan, Partitioning}; + use datafusion::prelude::{SessionConfig, SessionContext}; + use datafusion_proto::protobuf::PhysicalPlanNode; + use std::sync::Arc; + + /// Builds a two-partition `MemTable`-backed scan wrapped in a hash + /// `RepartitionExec`, i.e. a physical plan with a shuffle boundary that + /// `plan_query_stages` will split into two stages. + async fn build_physical_plan() -> (Arc, SessionContext) { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + ])); + + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int32Array::from(vec![10, 20, 30])), + ], + ) + .unwrap(); + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![4, 5])), + Arc::new(Int32Array::from(vec![40, 50])), + ], + ) + .unwrap(); + + let table = MemTable::try_new(schema, vec![vec![batch1], vec![batch2]]).unwrap(); + + let ctx = SessionContext::new(); + ctx.register_table("t", Arc::new(table)).unwrap(); + + let scan = ctx + .table("t") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + + let repartitioned: Arc = Arc::new( + RepartitionExec::try_new( + scan, + Partitioning::Hash(vec![Arc::new(Column::new("id", 0))], 4), + ) + .unwrap(), + ); + + (repartitioned, ctx) + } + + /// Submits a hand-built physical plan (scan -> hash repartition) directly to + /// a standalone Ballista cluster via the new physical-plan submission path, + /// and checks that the rows returned match what executing the same plan + /// locally produces. The plan's `RepartitionExec(Hash)` boundary means the + /// scheduler must have split it into two shuffle stages for this to work. + #[tokio::test] + async fn should_execute_submitted_physical_plan_across_shuffle_stages() { + let (host, port) = setup_test_cluster().await; + let scheduler_url = format!("http://{host}:{port}"); + + let (plan, ctx) = build_physical_plan().await; + + let expected = physical_plan::collect(plan.clone(), ctx.task_ctx()) + .await + .unwrap(); + + let session_config = SessionConfig::new_with_ballista(); + let codec = session_config.ballista_physical_extension_codec(); + let session_id = uuid::Uuid::new_v4().to_string(); + + let stream = execute_physical_plan::( + scheduler_url, + &BallistaConfig::default(), + plan, + codec.as_ref(), + session_id, + session_config, + ) + .await + .expect("physical plan job should be submitted and executed"); + + let actual = physical_plan::common::collect(stream) + .await + .expect("collecting results from the submitted physical plan"); + + assert_eq!( + batches_to_sort_string(&expected), + batches_to_sort_string(&actual) + ); + } +} diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto index 2154b1a17d..50a3e25e70 100644 --- a/ballista/core/proto/ballista.proto +++ b/ballista/core/proto/ballista.proto @@ -630,6 +630,11 @@ message ExecuteQueryParams { oneof query { bytes logical_plan = 1; bytes substrait_plan = 6; + // An already-built physical plan, serialized with the scheduler's + // physical extension codec. Lets clients bypass logical-plan creation + // on the scheduler, e.g. for plans containing custom operators that + // have no logical-plan representation. + bytes physical_plan = 7; } // reserved after removing deprecated `sql` query type. reserved 2; diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs index a7d79ef824..137cde1038 100644 --- a/ballista/core/src/execution_plans/distributed_query.rs +++ b/ballista/core/src/execution_plans/distributed_query.rs @@ -46,6 +46,7 @@ use datafusion::prelude::SessionConfig; use datafusion_proto::logical_plan::{ AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec, }; +use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use log::{debug, error, info}; use parking_lot::Mutex; @@ -321,6 +322,91 @@ impl ExecutionPlan for DistributedQueryExec { } } +/// Submits an already-built physical plan directly to a Ballista scheduler for +/// distributed execution, bypassing logical plan creation on the scheduler side. +/// +/// This is a lower-level entry point than [DistributedQueryExec]: instead of a +/// [LogicalPlan] that the scheduler turns into a physical plan itself, the caller +/// supplies the physical plan up front - e.g. for plans containing custom +/// operators that have no logical-plan representation. +pub async fn execute_physical_plan( + scheduler_url: String, + config: &BallistaConfig, + physical_plan: Arc, + codec: &dyn PhysicalExtensionCodec, + session_id: String, + session_config: SessionConfig, +) -> Result { + let plan_message = + U::try_from_physical_plan(physical_plan.clone(), codec).map_err(|e| { + DataFusionError::Internal(format!("failed to serialize physical plan: {e:?}")) + })?; + let mut buf: Vec = vec![]; + plan_message.try_encode(&mut buf).map_err(|e| { + DataFusionError::Execution(format!("failed to encode physical plan: {e:?}")) + })?; + + let settings = session_config + .options() + .entries() + .iter() + .map( + |datafusion::config::ConfigEntry { key, value, .. }| KeyValuePair { + key: key.to_owned(), + value: value.clone(), + }, + ) + .collect(); + let operation_id = uuid::Uuid::now_v7().to_string(); + let query = ExecuteQueryParams { + query: Some(Query::PhysicalPlan(buf)), + settings, + session_id: session_id.clone(), + operation_id, + }; + + let max_message_size = config.grpc_client_max_message_size(); + let grpc_config = GrpcClientConfig::from(config); + let metrics = Arc::new(ExecutionPlanMetricsSet::new()); + let job_id_handle = Arc::new(Mutex::new(None)); + let partition = 0; + + let stream: std::pin::Pin> + Send>> = + if session_config.ballista_config().client_pull() { + Box::pin( + execute_query_pull( + scheduler_url, + session_id, + query, + max_message_size, + grpc_config, + metrics, + job_id_handle, + partition, + session_config, + ) + .await?, + ) + } else { + Box::pin( + execute_query_push( + scheduler_url, + query, + max_message_size, + grpc_config, + metrics, + job_id_handle, + partition, + session_config, + ) + .await?, + ) + }; + + let schema = physical_plan.schema(); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) +} + /// Client will periodically invoke scheduler to check /// job status. There is preconfigured wait period between /// pulls, which increases query latency. diff --git a/ballista/core/src/execution_plans/mod.rs b/ballista/core/src/execution_plans/mod.rs index f623685096..7c27f26d89 100644 --- a/ballista/core/src/execution_plans/mod.rs +++ b/ballista/core/src/execution_plans/mod.rs @@ -32,7 +32,7 @@ use std::path::{Path, PathBuf}; pub use chaos_exec::ChaosExec; use datafusion::common::exec_err; pub use distributed_explain_analyze::DistributedExplainAnalyzeExec; -pub use distributed_query::DistributedQueryExec; +pub use distributed_query::{DistributedQueryExec, execute_physical_plan}; pub use shuffle_reader::{CoalescePlan, PartitionGroup, ShuffleReaderExec}; pub use shuffle_reader::{stats_for_partition, stats_for_partitions}; pub use shuffle_writer::DEFAULT_SHUFFLE_CHANNEL_CAPACITY; diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs index 269827145e..d071546664 100644 --- a/ballista/core/src/serde/generated/ballista.rs +++ b/ballista/core/src/serde/generated/ballista.rs @@ -958,7 +958,7 @@ pub struct ExecuteQueryParams { /// client and scheduler #[prost(string, tag = "5")] pub operation_id: ::prost::alloc::string::String, - #[prost(oneof = "execute_query_params::Query", tags = "1, 6")] + #[prost(oneof = "execute_query_params::Query", tags = "1, 6, 7")] pub query: ::core::option::Option, } /// Nested message and enum types in `ExecuteQueryParams`. @@ -969,6 +969,12 @@ pub mod execute_query_params { LogicalPlan(::prost::alloc::vec::Vec), #[prost(bytes, tag = "6")] SubstraitPlan(::prost::alloc::vec::Vec), + /// An already-built physical plan, serialized with the scheduler's + /// physical extension codec. Lets clients bypass logical-plan creation + /// on the scheduler, e.g. for plans containing custom operators that + /// have no logical-plan representation. + #[prost(bytes, tag = "7")] + PhysicalPlan(::prost::alloc::vec::Vec), } } #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/ballista/scheduler/src/scheduler_server/event.rs b/ballista/scheduler/src/scheduler_server/event.rs index c02c14a997..2cb7e718b5 100644 --- a/ballista/scheduler/src/scheduler_server/event.rs +++ b/ballista/scheduler/src/scheduler_server/event.rs @@ -18,12 +18,35 @@ use std::fmt::{Debug, Formatter}; use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::ExecutionPlan; use crate::state::execution_graph::RunningTaskInfo; use ballista_core::{JobId, JobStatusSubscriber, serde::protobuf::TaskStatus}; use datafusion::prelude::SessionContext; use std::sync::Arc; +/// A plan submitted for execution by a client. +/// +/// Most jobs are submitted as a [LogicalPlan] which the scheduler optimizes +/// and turns into a physical plan itself. A client may instead submit an +/// already-built physical plan directly, bypassing logical planning on the +/// scheduler entirely - e.g. for plans containing custom operators that have +/// no logical-plan representation. Physical-plan jobs always use the static +/// distributed planner; the adaptive query planner (AQE) requires a logical +/// plan and is not available for this path. +// `LogicalPlan` is much larger than `Arc`, but `SubmitPlan` +// values are only ever handled by reference or already boxed by their callers +// (e.g. `QueryStageSchedulerEvent::JobQueued::plan`), so the size difference +// does not cause repeated large stack copies in practice. +#[allow(clippy::large_enum_variant)] +#[derive(Clone)] +pub enum SubmitPlan { + /// A logical plan that the scheduler will plan into a physical plan. + Logical(LogicalPlan), + /// An already-built physical plan supplied directly by the client. + Physical(Arc), +} + /// Events that drive the query stage scheduler state machine. #[derive(Clone)] pub enum QueryStageSchedulerEvent { @@ -35,8 +58,8 @@ pub enum QueryStageSchedulerEvent { job_name: String, /// Session context for the job. session_ctx: Arc, - /// Logical plan to execute. - plan: Box, + /// Plan to execute, either logical or an already-built physical plan. + plan: Box, /// Timestamp when the job was queued. queued_at: u64, /// job status subscriber diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs index 14a6952dab..1d34926698 100644 --- a/ballista/scheduler/src/scheduler_server/grpc.rs +++ b/ballista/scheduler/src/scheduler_server/grpc.rs @@ -51,9 +51,8 @@ use { use crate::cluster::{bind_task_bias, bind_task_round_robin}; use crate::config::TaskDistributionPolicy; -use crate::scheduler_server::event::QueryStageSchedulerEvent; +use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan}; use ballista_core::serde::protobuf::get_job_status_result::FlightProxy; -use datafusion::logical_expr::LogicalPlan; use datafusion::physical_plan::{DisplayFormatType, ExecutionPlan}; use datafusion::prelude::SessionContext; use std::ops::Deref; @@ -414,8 +413,8 @@ impl SchedulerGrpc })?; debug!( - "Decoded logical plan for execution:\n{}", - plan.display_indent() + "Decoded plan for execution:\n{}", + Self::describe_submitted_plan(&plan) ); log::trace!("setting job name: {job_name}"); @@ -499,8 +498,8 @@ impl SchedulerGrpc }; debug!( - "Decoded logical plan for execution:\n{}", - plan.display_indent() + "Decoded plan for execution:\n{}", + Self::describe_submitted_plan(&plan) ); log::trace!("setting job name: {job_name}"); @@ -785,7 +784,7 @@ impl SchedulerServer BResult { + ) -> BResult { match query { Query::LogicalPlan(message) => T::try_decode(message.as_slice()) .and_then(|m| { @@ -794,8 +793,22 @@ impl SchedulerServer { + let task_ctx = session_ctx.task_ctx(); + U::try_decode(message.as_slice()) + .and_then(|m| { + m.try_into_physical_plan( + task_ctx.as_ref(), + self.state.codec.physical_extension_codec(), + ) + }) + .map(SubmitPlan::Physical) + .map_err(|e| e.into()) + } + #[cfg(not(feature = "substrait"))] Query::SubstraitPlan(_) => { Err(BallistaError::NotImplemented("Received query type \"Substrait\", enable \"substrait\" feature to support Substrait plans.".to_string())) @@ -807,11 +820,24 @@ impl SchedulerServer String { + match plan { + SubmitPlan::Logical(plan) => plan.display_indent().to_string(), + SubmitPlan::Physical(plan) => { + datafusion::physical_plan::displayable(plan.as_ref()) + .indent(false) + .to_string() + } + } + } + fn flight_proxy_config(&self) -> Option { self.state .config diff --git a/ballista/scheduler/src/scheduler_server/mod.rs b/ballista/scheduler/src/scheduler_server/mod.rs index 17b955a98c..5886c65e04 100644 --- a/ballista/scheduler/src/scheduler_server/mod.rs +++ b/ballista/scheduler/src/scheduler_server/mod.rs @@ -25,7 +25,6 @@ use ballista_core::serde::protobuf::TaskStatus; use ballista_core::{JobId, JobStatusSubscriber}; use datafusion::execution::context::SessionState; -use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; @@ -36,7 +35,7 @@ use crate::metrics::SchedulerMetricsCollector; use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata}; use log::{debug, error, warn}; -use crate::scheduler_server::event::QueryStageSchedulerEvent; +use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan}; use crate::scheduler_server::query_stage_scheduler::QueryStageScheduler; use crate::state::executor_manager::ExecutorManager; @@ -222,7 +221,7 @@ impl SchedulerServer, - plan: &LogicalPlan, + plan: &SubmitPlan, subscriber: Option, ) -> Result { log::debug!("Received submit request for job {job_name}"); @@ -438,6 +437,7 @@ mod test { ExecutorSpecification, }; + use crate::scheduler_server::event::SubmitPlan; use crate::scheduler_server::{SchedulerServer, timestamp_millis}; use crate::test_utils::{ @@ -484,7 +484,14 @@ mod test { // Submit job scheduler .state - .submit_job(&job_id, "", ctx, &plan, 0, None) + .submit_job( + &job_id, + "", + ctx, + &SubmitPlan::Logical(plan.clone()), + 0, + None, + ) .await .expect("submitting plan"); diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index 7268942356..6ee5f59f50 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -17,7 +17,7 @@ use crate::cluster::{BallistaCluster, BoundTask, ExecutorSlot}; use crate::config::SchedulerConfig; -use crate::scheduler_server::event::QueryStageSchedulerEvent; +use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan}; use crate::state::execution_graph::TaskDescription; use crate::state::executor_manager::ExecutorManager; use crate::state::session_manager::SessionManager; @@ -28,7 +28,6 @@ use ballista_core::serde::BallistaCodec; use ballista_core::serde::protobuf::TaskStatus; use ballista_core::{JobId, JobStatusSubscriber}; use datafusion::execution::context::SessionContext; -use datafusion::logical_expr::LogicalPlan; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; use log::{debug, error, info, warn}; @@ -366,21 +365,14 @@ impl SchedulerState, - logical_plan: &LogicalPlan, + plan: &SubmitPlan, queued_at: u64, subscriber: Option, ) -> Result<()> { let start = Instant::now(); self.task_manager - .submit_job( - job_id, - job_name, - session_ctx, - logical_plan, - queued_at, - subscriber, - ) + .submit_job(job_id, job_name, session_ctx, plan, queued_at, subscriber) .await?; let elapsed = start.elapsed(); diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index 7bcbd68534..404d3dacb5 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -18,7 +18,7 @@ use crate::cluster::JobState; use crate::config::SchedulerConfig; use crate::planner::DefaultDistributedPlanner; -use crate::scheduler_server::event::QueryStageSchedulerEvent; +use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan}; use crate::state::aqe::AdaptiveExecutionGraph; use crate::state::distributed_explain::handle_explain_plan; use crate::state::execution_graph::{ @@ -37,7 +37,6 @@ use ballista_core::{JobId, JobStatusSubscriber}; use dashmap::DashMap; use datafusion::execution::config::SessionConfig; use datafusion::execution::context::SessionContext; -use datafusion::logical_expr::LogicalPlan; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; @@ -283,47 +282,76 @@ impl TaskManager job_id: &JobId, job_name: &str, ctx: Arc, - logical_plan: &LogicalPlan, + plan: &SubmitPlan, queued_at: u64, subscriber: Option, ) -> Result<()> { let mut planner = DefaultDistributedPlanner::new(); let session_state = ctx.state(); let session_config = session_state.config(); - let mut graph = if session_config.ballista_adaptive_query_planner_enabled() { - debug!("Using adaptive query planner (AQE) for job planning"); - warn!( - "Adaptive Query Planning is EXPERIMENTAL, should be used for testing purposes only!" - ); - Box::new( - AdaptiveExecutionGraph::try_new( + + let mut graph = match plan { + SubmitPlan::Logical(logical_plan) => { + if session_config.ballista_adaptive_query_planner_enabled() { + debug!("Using adaptive query planner (AQE) for job planning"); + warn!( + "Adaptive Query Planning is EXPERIMENTAL, should be used for testing purposes only!" + ); + Box::new( + AdaptiveExecutionGraph::try_new( + &self.scheduler_id, + job_id, + job_name, + &ctx, + logical_plan, + queued_at, + ) + .await?, + ) as ExecutionGraphBox + } else { + debug!("Using static query planner for job planning"); + let session_config = Arc::new(ctx.copied_config()); + + let physical_plan = + ctx.state().create_physical_plan(logical_plan).await?; + let physical_plan = + handle_explain_plan(job_id, &ctx, logical_plan, physical_plan) + .await?; + + Box::new(StaticExecutionGraph::new( + &self.scheduler_id, + job_id, + job_name, + &ctx.session_id(), + physical_plan, + queued_at, + session_config, + &mut planner, + Some(logical_plan.display_indent().to_string()), + )?) as ExecutionGraphBox + } + } + SubmitPlan::Physical(physical_plan) => { + if session_config.ballista_adaptive_query_planner_enabled() { + return Err(BallistaError::NotImplemented( + "Adaptive query planning (AQE) does not support jobs submitted as an already-built physical plan; disable AQE for this session or submit a logical plan instead.".to_string(), + )); + } + debug!("Using static query planner for physical-plan job submission"); + let session_config = Arc::new(ctx.copied_config()); + + Box::new(StaticExecutionGraph::new( &self.scheduler_id, job_id, job_name, - &ctx, - logical_plan, + &ctx.session_id(), + physical_plan.clone(), queued_at, - ) - .await?, - ) as ExecutionGraphBox - } else { - debug!("Using static query planner for job planning"); - let session_config = Arc::new(ctx.copied_config()); - - let plan = ctx.state().create_physical_plan(logical_plan).await?; - let plan = handle_explain_plan(job_id, &ctx, logical_plan, plan).await?; - - Box::new(StaticExecutionGraph::new( - &self.scheduler_id, - job_id, - job_name, - &ctx.session_id(), - plan, - queued_at, - session_config, - &mut planner, - Some(logical_plan.display_indent().to_string()), - )?) as ExecutionGraphBox + session_config, + &mut planner, + None, + )?) as ExecutionGraphBox + } }; let string_plan = datafusion::physical_plan::displayable(graph.physical_plan().as_ref()) diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs index 4f7b29651c..3ced8108ed 100644 --- a/ballista/scheduler/src/test_utils.rs +++ b/ballista/scheduler/src/test_utils.rs @@ -54,7 +54,7 @@ use datafusion::prelude::{CsvReadOptions, JoinType, col}; use datafusion::test_util::scan_empty_with_partitions; use crate::cluster::BallistaCluster; -use crate::scheduler_server::event::QueryStageSchedulerEvent; +use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan}; use crate::state::execution_graph::{ ExecutionGraph, ExecutionStage, StaticExecutionGraph, TaskDescription, @@ -503,7 +503,10 @@ impl SchedulerTest { .create_or_update_session("session_id", &self.session_config) .await?; - let job_id = self.scheduler.submit_job(job_name, ctx, plan, None).await?; + let job_id = self + .scheduler + .submit_job(job_name, ctx, &SubmitPlan::Logical(plan.clone()), None) + .await?; Ok(job_id) } @@ -642,7 +645,12 @@ impl SchedulerTest { let job_id = self .scheduler - .submit_job(job_name, ctx, plan, subscriber) + .submit_job( + job_name, + ctx, + &SubmitPlan::Logical(plan.clone()), + subscriber, + ) .await?; let mut receiver = self.status_receiver.take().unwrap(); From b880ca4193575fdd870900f822a2f6e6bfc2c5a0 Mon Sep 17 00:00:00 2001 From: jgrim Date: Sat, 4 Jul 2026 15:19:15 +0200 Subject: [PATCH 022/107] feat: cancel running stages and tasks on job failure or cancellation (#1903) * feat: failing tests for sibling tasks cancelled on stage failure * feat: cancel running stages and their tasks on job abort * refactor: implement abort_running per ExecutionGraph impl * feat: mark a failed stage's in-flight tasks as cancelled * Revert "refactor: implement abort_running per ExecutionGraph impl" This reverts commit 1626af4872587b400d1dc6e588f383160f3c4918. * refactor: use "killed" for aborted in-flight task error --- .../src/state/aqe/test/job_failure.rs | 139 ++++++++++++++++++ ballista/scheduler/src/state/aqe/test/mod.rs | 2 + .../scheduler/src/state/execution_graph.rs | 81 +++++++++- .../scheduler/src/state/execution_stage.rs | 31 +++- ballista/scheduler/src/state/task_manager.rs | 4 +- 5 files changed, 250 insertions(+), 7 deletions(-) create mode 100644 ballista/scheduler/src/state/aqe/test/job_failure.rs diff --git a/ballista/scheduler/src/state/aqe/test/job_failure.rs b/ballista/scheduler/src/state/aqe/test/job_failure.rs new file mode 100644 index 0000000000..22a926248c --- /dev/null +++ b/ballista/scheduler/src/state/aqe/test/job_failure.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Job-failure lifecycle tests for the adaptive execution graph. + +use crate::state::aqe::AdaptiveExecutionGraph; +use crate::state::execution_graph::ExecutionGraph; +use crate::state::execution_stage::ExecutionStage; +use crate::test_utils::mock_executor; +use ballista_core::error::Result; +use ballista_core::serde::protobuf::{ + FailedTask, JobStatus, failed_task, job_status, task_status, +}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::execution::context::{SessionConfig, SessionContext}; +use datafusion::functions_aggregate::sum::sum; +use datafusion::logical_expr::SortExpr; +use datafusion::prelude::{JoinType, col}; +use datafusion::test_util::scan_empty_with_partitions; + +/// Builds an adaptive graph for a join (two concurrent leaf stages). +async fn test_join_plan(partition: usize) -> AdaptiveExecutionGraph { + let mut config = SessionConfig::new().with_target_partitions(partition); + config + .options_mut() + .optimizer + .enable_round_robin_repartition = false; + let ctx = SessionContext::new_with_config(config); + + let schema = Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("gmv", DataType::UInt64, false), + ]); + + let left_plan = scan_empty_with_partitions(Some("left"), &schema, None, 2).unwrap(); + let right_plan = scan_empty_with_partitions(Some("right"), &schema, None, 2) + .unwrap() + .build() + .unwrap(); + let sort_expr = SortExpr::new(col("id"), false, false); + let logical_plan = left_plan + .join(right_plan, JoinType::Inner, (vec!["id"], vec!["id"]), None) + .unwrap() + .aggregate(vec![col("left.id")], vec![sum(col("left.gmv"))]) + .unwrap() + .sort(vec![sort_expr]) + .unwrap() + .build() + .unwrap(); + + AdaptiveExecutionGraph::try_new( + "localhost:50050", + &"job".into(), + "", + &ctx, + &logical_plan, + 0, + ) + .await + .unwrap() +} + +// Same contract as the static graph's test: aborting transitions every running +// stage to Failed and returns its in-flight tasks for cancellation. +#[tokio::test] +async fn test_abort_running_cancels_stages_and_returns_inflight_tasks() -> Result<()> { + let executor = mock_executor("executor-id1".to_string()); + let mut graph = test_join_plan(2).await; + + // Call revive to move the two leaf Resolved stages to Running + graph.revive(); + assert!( + graph.running_stages().len() >= 2, + "expected two concurrently running leaf stages, found {:?}", + graph.running_stages() + ); + + // Dispatch a task so there is an in-flight task to cancel + let _task = graph.pop_next_task(&executor.id)?.unwrap(); + + // Aborting cancels every running stage and returns its in-flight tasks + let cancelled = graph.abort_running("job aborted".to_string()); + + assert!( + !cancelled.is_empty(), + "abort_running must return the in-flight tasks to cancel" + ); + assert!( + graph.running_stages().is_empty(), + "every running stage must be cancelled, found {:?}", + graph.running_stages() + ); + assert!( + matches!( + graph.status(), + JobStatus { + status: Some(job_status::Status::Failed(_)), + .. + } + ), + "the job must be Failed after abort" + ); + + // In-flight tasks of the cancelled stage are recorded as Failed(TaskKilled) + let has_killed_task = graph.stages.values().any(|stage| match stage { + ExecutionStage::Failed(failed) => { + failed.task_infos.iter().flatten().any(|info| { + matches!( + &info.task_status, + task_status::Status::Failed(FailedTask { + failed_reason: Some(failed_task::FailedReason::TaskKilled(_)), + .. + }) + ) + }) + } + _ => false, + }); + assert!( + has_killed_task, + "in-flight tasks must be recorded as Failed(TaskKilled) after abort" + ); + + Ok(()) +} diff --git a/ballista/scheduler/src/state/aqe/test/mod.rs b/ballista/scheduler/src/state/aqe/test/mod.rs index c1237f216d..66272f62b7 100644 --- a/ballista/scheduler/src/state/aqe/test/mod.rs +++ b/ballista/scheduler/src/state/aqe/test/mod.rs @@ -19,6 +19,8 @@ mod alter_stages; /// Functional tests for the CoalescePartitionsRule end-to-end through the planner mod coalesce_rule; +/// Job-failure lifecycle tests for the adaptive graph +mod job_failure; /// covers join selection tests mod join_selection; /// Tests if plan is going to be split to stages correctly diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 51228a40b6..79cbf9c70b 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -219,6 +219,18 @@ pub trait ExecutionGraph: Debug { /// fail job with error message fn fail_job(&mut self, error: String); + /// Abort a running job: fail it, transition every running stage to Failed, + /// and return the in-flight tasks that should be cancelled. Used for both the + /// failure and cancellation teardown paths. + fn abort_running(&mut self, error: String) -> Vec { + let running_tasks = self.running_tasks(); + self.fail_job(error.clone()); + for stage_id in self.running_stages() { + self.fail_stage(stage_id, error.clone()); + } + running_tasks + } + /// Marks the job as successfully completed. /// /// This should only be called after all stages have completed successfully. @@ -1761,10 +1773,11 @@ mod test { use ballista_core::error::Result; use ballista_core::serde::protobuf::{ self, ExecutionError, FailedTask, FetchPartitionError, IoError, JobStatus, - TaskKilled, failed_task, job_status, + TaskKilled, failed_task, job_status, task_status, }; use crate::state::execution_graph::ExecutionGraph; + use crate::state::execution_stage::ExecutionStage; use crate::test_utils::{ mock_completed_task, mock_executor, mock_failed_task, revive_graph_and_complete_next_stage, @@ -2155,6 +2168,72 @@ mod test { Ok(()) } + // Aborting a running job (failure or cancellation) must transition every + // running stage to Failed and return its in-flight tasks for cancellation. + // `abort_running` is the shared teardown invoked by `abort_job`. + #[tokio::test] + async fn test_abort_running_cancels_stages_and_returns_inflight_tasks() -> Result<()> + { + let executor = mock_executor("executor-id1".to_string()); + let mut graph = test_join_plan(2).await; + + // Call revive to move the two leaf Resolved stages to Running + graph.revive(); + assert!( + graph.running_stages().len() >= 2, + "expected two concurrently running leaf stages, found {:?}", + graph.running_stages() + ); + + // Dispatch a task so there is an in-flight task to cancel + let _task = graph.pop_next_task(&executor.id)?.unwrap(); + + // Aborting cancels every running stage and returns its in-flight tasks + let cancelled = graph.abort_running("job aborted".to_string()); + + assert!( + !cancelled.is_empty(), + "abort_running must return the in-flight tasks to cancel" + ); + assert!( + graph.running_stages().is_empty(), + "every running stage must be cancelled, found {:?}", + graph.running_stages() + ); + assert!( + matches!( + graph.status(), + JobStatus { + status: Some(job_status::Status::Failed(_)), + .. + } + ), + "the job must be Failed after abort" + ); + + // In-flight tasks of the cancelled stage are recorded as Failed(TaskKilled) + let has_killed_task = graph.stages.values().any(|stage| match stage { + ExecutionStage::Failed(failed) => { + failed.task_infos.iter().flatten().any(|info| { + matches!( + &info.task_status, + task_status::Status::Failed(FailedTask { + failed_reason: Some(failed_task::FailedReason::TaskKilled(_)), + .. + }) + ) + }) + } + _ => false, + }); + assert!( + has_killed_task, + "in-flight tasks must be recorded as Failed(TaskKilled) after abort" + ); + + Ok(()) + } + #[tokio::test] async fn test_long_delayed_failed_task_after_executor_lost() -> Result<()> { let executor1 = mock_executor("executor-id1".to_string()); diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index e9e8f1b07f..698d951828 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -35,7 +35,7 @@ use ballista_core::error::{BallistaError, Result}; use ballista_core::execution_plans::{ShuffleWriterExec, SortShuffleWriterExec}; use ballista_core::serde::protobuf::failed_task::FailedReason; use ballista_core::serde::protobuf::{ - FailedTask, OperatorMetricsSet, ResultLost, SuccessfulTask, TaskStatus, + FailedTask, OperatorMetricsSet, ResultLost, SuccessfulTask, TaskKilled, TaskStatus, }; use ballista_core::serde::protobuf::{RunningTask, task_status}; use ballista_core::serde::scheduler::PartitionLocation; @@ -554,15 +554,40 @@ impl RunningStage { } } - /// Converts this running stage to a failed stage with the given error message. + /// Converts this running stage to a failed stage. Still-running tasks are recorded + /// as cancelled (`Failed(TaskKilled)`) since failing the stage cancels them. pub fn to_failed(&self, error_message: String) -> FailedStage { + let task_infos = self + .task_infos + .iter() + .map(|task_info| { + task_info.as_ref().map(|info| { + if matches!(info.task_status, task_status::Status::Running(_)) { + TaskInfo { + task_status: task_status::Status::Failed(FailedTask { + error: "killed".to_string(), + retryable: false, + count_to_failures: false, + failed_reason: Some(FailedReason::TaskKilled( + TaskKilled {}, + )), + }), + ..info.clone() + } + } else { + info.clone() + } + }) + }) + .collect(); + FailedStage { stage_id: self.stage_id, stage_attempt_num: self.stage_attempt_num, partitions: self.partitions, output_links: self.output_links.clone(), plan: self.plan.clone(), - task_infos: self.task_infos.clone(), + task_infos, stage_metrics: self.stage_metrics.clone(), error_message, } diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index 404d3dacb5..6283ac7eef 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -576,7 +576,7 @@ impl TaskManager let mut guard = graph.write().await; let pending_tasks = guard.available_tasks(); - let running_tasks = guard.running_tasks(); + let running_tasks = guard.abort_running(failure_reason); info!( "Cancelling {} running tasks for job {}", @@ -584,8 +584,6 @@ impl TaskManager job_id ); - guard.fail_job(failure_reason); - self.state.save_job(job_id, &guard).await?; (running_tasks, pending_tasks) From 3b870187645bae7cde26feb1668c9a3a01a069d7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 4 Jul 2026 08:58:09 -0600 Subject: [PATCH 023/107] refactor(scheduler): keep submit_job non-breaking, add submit_physical_plan (#1941) --- .../scheduler/src/scheduler_server/grpc.rs | 4 +- .../scheduler/src/scheduler_server/mod.rs | 102 ++++++++++++++++-- ballista/scheduler/src/state/mod.rs | 2 +- ballista/scheduler/src/state/task_manager.rs | 51 ++++++++- ballista/scheduler/src/test_utils.rs | 14 +-- 5 files changed, 147 insertions(+), 26 deletions(-) diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs index 1d34926698..14a0204d89 100644 --- a/ballista/scheduler/src/scheduler_server/grpc.rs +++ b/ballista/scheduler/src/scheduler_server/grpc.rs @@ -429,7 +429,7 @@ impl SchedulerGrpc }); let job_id = self - .submit_job(&job_name, session_ctx, &plan, Some(subscriber)) + .submit_plan(&job_name, session_ctx, &plan, Some(subscriber)) .await .map_err(|e| { let msg = @@ -504,7 +504,7 @@ impl SchedulerGrpc log::trace!("setting job name: {job_name}"); let job_id = self - .submit_job(&job_name, session_ctx, &plan, None) + .submit_plan(&job_name, session_ctx, &plan, None) .await .map_err(|e| { let msg = diff --git a/ballista/scheduler/src/scheduler_server/mod.rs b/ballista/scheduler/src/scheduler_server/mod.rs index 5886c65e04..17fdc5b54a 100644 --- a/ballista/scheduler/src/scheduler_server/mod.rs +++ b/ballista/scheduler/src/scheduler_server/mod.rs @@ -25,6 +25,8 @@ use ballista_core::serde::protobuf::TaskStatus; use ballista_core::{JobId, JobStatusSubscriber}; use datafusion::execution::context::SessionState; +use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; @@ -216,8 +218,8 @@ impl SchedulerServer, @@ -241,6 +243,37 @@ impl SchedulerServer, + plan: &LogicalPlan, + subscriber: Option, + ) -> Result { + self.submit_plan( + job_name, + ctx, + &SubmitPlan::Logical(plan.clone()), + subscriber, + ) + .await + } + + /// Submit an already-built physical plan for distributed execution. The physical + /// plan is planned with the static distributed planner; adaptive query planning + /// (AQE) is not available for this path. + pub async fn submit_physical_plan( + &self, + job_name: &str, + ctx: Arc, + plan: Arc, + subscriber: Option, + ) -> Result { + self.submit_plan(job_name, ctx, &SubmitPlan::Physical(plan), subscriber) + .await + } + /// It just send task status update event to the channel, /// and will not guarantee the event processing completed after return pub(crate) async fn update_task_status( @@ -437,7 +470,6 @@ mod test { ExecutorSpecification, }; - use crate::scheduler_server::event::SubmitPlan; use crate::scheduler_server::{SchedulerServer, timestamp_millis}; use crate::test_utils::{ @@ -484,14 +516,8 @@ mod test { // Submit job scheduler .state - .submit_job( - &job_id, - "", - ctx, - &SubmitPlan::Logical(plan.clone()), - 0, - None, - ) + .task_manager + .submit_job(&job_id, "", ctx, &plan, 0, None) .await .expect("submitting plan"); @@ -564,6 +590,60 @@ mod test { Ok(()) } + // Verify a job can be submitted as an already-built physical plan. + #[tokio::test] + async fn test_submit_physical_plan() -> Result<()> { + let logical_plan = test_plan(); + let task_slots = 4; + + let scheduler = test_scheduler(TaskSchedulingPolicy::PullStaged).await?; + + let executors = test_executors(task_slots); + for (executor_metadata, executor_data) in executors { + scheduler + .state + .executor_manager + .register_executor(executor_metadata, executor_data) + .await?; + } + + let config = + SessionConfig::new_with_ballista().with_target_partitions(task_slots); + + let ctx = scheduler + .state + .session_manager + .create_or_update_session("session_id", &config) + .await?; + + // Plan the logical plan into a physical plan and submit it directly. + let physical_plan = ctx.state().create_physical_plan(&logical_plan).await?; + + let job_id: JobId = "physical_job".into(); + + scheduler + .state + .task_manager + .queue_job(&job_id, "", timestamp_millis())?; + + scheduler + .state + .task_manager + .submit_physical_plan(&job_id, "", ctx, physical_plan, 0, None) + .await + .expect("submitting physical plan"); + + assert!( + scheduler + .state + .task_manager + .get_active_execution_graph(&job_id) + .is_some() + ); + + Ok(()) + } + #[tokio::test] async fn test_push_scheduling() -> Result<()> { let plan = test_plan(); diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index 6ee5f59f50..ce007330e7 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -372,7 +372,7 @@ impl SchedulerState TaskManager /// Generate an ExecutionGraph for the job and save it to the persistent state. /// By default, this job will be curated by the scheduler which receives it. /// Then we will also save it to the active execution graph + /// + /// Shared entry point for logical and physical submissions. #[allow(clippy::too_many_arguments)] - pub async fn submit_job( + pub async fn submit_plan( &self, job_id: &JobId, job_name: &str, @@ -370,6 +373,52 @@ impl TaskManager Ok(()) } + /// Submit a logical plan for distributed execution. + #[allow(clippy::too_many_arguments)] + pub async fn submit_job( + &self, + job_id: &JobId, + job_name: &str, + ctx: Arc, + plan: &LogicalPlan, + queued_at: u64, + subscriber: Option, + ) -> Result<()> { + self.submit_plan( + job_id, + job_name, + ctx, + &SubmitPlan::Logical(plan.clone()), + queued_at, + subscriber, + ) + .await + } + + /// Submit an already-built physical plan for distributed execution. The physical + /// plan is planned with the static distributed planner; adaptive query planning + /// (AQE) is not available for this path. + #[allow(clippy::too_many_arguments)] + pub async fn submit_physical_plan( + &self, + job_id: &JobId, + job_name: &str, + ctx: Arc, + plan: Arc, + queued_at: u64, + subscriber: Option, + ) -> Result<()> { + self.submit_plan( + job_id, + job_name, + ctx, + &SubmitPlan::Physical(plan), + queued_at, + subscriber, + ) + .await + } + /// Returns a snapshot of currently running jobs from the cache. pub fn get_running_job_cache(&self) -> Arc> { let ret = self diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs index 3ced8108ed..4f7b29651c 100644 --- a/ballista/scheduler/src/test_utils.rs +++ b/ballista/scheduler/src/test_utils.rs @@ -54,7 +54,7 @@ use datafusion::prelude::{CsvReadOptions, JoinType, col}; use datafusion::test_util::scan_empty_with_partitions; use crate::cluster::BallistaCluster; -use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan}; +use crate::scheduler_server::event::QueryStageSchedulerEvent; use crate::state::execution_graph::{ ExecutionGraph, ExecutionStage, StaticExecutionGraph, TaskDescription, @@ -503,10 +503,7 @@ impl SchedulerTest { .create_or_update_session("session_id", &self.session_config) .await?; - let job_id = self - .scheduler - .submit_job(job_name, ctx, &SubmitPlan::Logical(plan.clone()), None) - .await?; + let job_id = self.scheduler.submit_job(job_name, ctx, plan, None).await?; Ok(job_id) } @@ -645,12 +642,7 @@ impl SchedulerTest { let job_id = self .scheduler - .submit_job( - job_name, - ctx, - &SubmitPlan::Logical(plan.clone()), - subscriber, - ) + .submit_job(job_name, ctx, plan, subscriber) .await?; let mut receiver = self.status_receiver.take().unwrap(); From ea9adfbea58d0ed3c9f6b308307eb4c1da0a51d7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 4 Jul 2026 16:48:57 -0600 Subject: [PATCH 024/107] feat(benchmarks): support reading TPC-H data from S3-compatible object storage (#1942) --- benchmarks/Cargo.toml | 4 +++- benchmarks/src/bin/tpch.rs | 30 ++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 29694a3b4f..ab969b4777 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -32,7 +32,9 @@ default = ["mimalloc"] [dependencies] ballista = { path = "../ballista/client", version = "53.0.0" } -ballista-core = { path = "../ballista/core", version = "53.0.0" } +ballista-core = { path = "../ballista/core", version = "53.0.0", features = [ + "build-binary", +] } clap = { workspace = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index 282eb241f2..0947fe5790 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -19,6 +19,9 @@ use ballista::extension::SessionConfigExt; use ballista::prelude::SessionContextExt; +use ballista_core::object_store::{ + session_config_with_s3_support, session_state_with_s3_support, +}; use datafusion::arrow::array::*; use datafusion::arrow::datatypes::SchemaBuilder; use datafusion::arrow::util::display::array_value_to_string; @@ -92,9 +95,11 @@ struct BallistaBenchmarkOpt { #[structopt(short = "s", long = "batch-size", default_value = "8192")] batch_size: usize, - /// Path to data files - #[structopt(parse(from_os_str), required = true, short = "p", long = "path")] - path: PathBuf, + /// Path to data files. May be a local path or an object-store URL such as + /// `s3://bucket/prefix` (credentials/endpoint via AWS_* env vars or + /// `-c s3.*` config overrides). + #[structopt(required = true, short = "p", long = "path")] + path: String, /// File format: `csv` or `parquet` #[structopt(short = "f", long = "format", default_value = "csv")] @@ -423,7 +428,7 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> { let ctx = SessionContext::new_with_config(cfg); register_datafusion_tables( &ctx, - opt.path.to_str().unwrap(), + opt.path.as_str(), opt.file_format.as_str(), opt.partitions, ) @@ -439,7 +444,7 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> { for query in query_numbers { let mut query_run = QueryRun::new(query); - let mut config = SessionConfig::new_with_ballista() + let mut config = session_config_with_s3_support() .with_target_partitions(opt.partitions) .with_ballista_job_name(&format!("Query derived from TPC-H q{}", query)) .with_batch_size(opt.batch_size) @@ -459,14 +464,11 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> { } } - let state = SessionStateBuilder::new() - .with_default_features() - .with_config(config) - .build(); + let state = session_state_with_s3_support(config)?; let ctx = SessionContext::remote_with_state(&address, state).await?; // register tables with Ballista context - let path = opt.path.to_str().unwrap(); + let path = opt.path.as_str(); let file_format = opt.file_format.as_str(); register_tables(path, file_format, &ctx, opt.debug).await?; @@ -781,6 +783,14 @@ async fn register_tables( } fn find_path(path: &str, table: &str, ext: &str) -> Result { + // Object-store URLs (e.g. `s3://bucket/prefix`) cannot be probed with the + // local filesystem, so register the per-table directory under the base URL + // directly. The trailing slash marks it as a directory so the listing table + // enumerates the Parquet files inside (e.g. `s3://bucket/prefix/lineitem/`). + if path.contains("://") { + return Ok(format!("{path}/{table}/")); + } + let path1 = format!("{path}/{table}.{ext}"); let path2 = format!("{path}/{table}"); if Path::new(&path1).exists() { From 6ccbcd54c38ca3b1072f8b1695855d6f1d973e8d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 4 Jul 2026 16:49:28 -0600 Subject: [PATCH 025/107] docs: add versioned upgrade guide (54.0.0) (#1939) --- docs/source/contributors-guide/development.md | 9 ++ docs/source/index.rst | 6 + docs/source/upgrading/54.0.0.md | 125 ++++++++++++++++++ docs/source/upgrading/index.rst | 24 ++++ 4 files changed, 164 insertions(+) create mode 100644 docs/source/upgrading/54.0.0.md create mode 100644 docs/source/upgrading/index.rst diff --git a/docs/source/contributors-guide/development.md b/docs/source/contributors-guide/development.md index a54ae08f2e..a361c897fd 100644 --- a/docs/source/contributors-guide/development.md +++ b/docs/source/contributors-guide/development.md @@ -156,3 +156,12 @@ This includes instructions for: - Running benchmarks against DataFusion and Ballista - Comparing performance with Apache Spark - Running load tests + +## Upgrade notes + +When a change breaks a public API, renames or removes a configuration key or CLI +flag, changes a default that alters query plans or results, or changes the +wire/serialization format, add an entry describing the change and the required +action to the current unreleased page under +[`docs/source/upgrading/`](../upgrading/index.rst). This keeps the upgrade guide +complete for the next release. diff --git a/docs/source/index.rst b/docs/source/index.rst index b14743be68..e66e237528 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -64,6 +64,12 @@ Table of content changelog/index +.. toctree:: + :maxdepth: 1 + :caption: Upgrade Guides + + upgrading/index + .. _toc.contributors: .. toctree:: diff --git a/docs/source/upgrading/54.0.0.md b/docs/source/upgrading/54.0.0.md new file mode 100644 index 0000000000..0b3a282498 --- /dev/null +++ b/docs/source/upgrading/54.0.0.md @@ -0,0 +1,125 @@ + + +# Upgrade Guides + +## Ballista 54.0.0 + +**Note:** Ballista `54.0.0` has not been released yet. The information provided +in this section pertains to features and changes that have already been merged +to the main branch and are awaiting release in this version. + +### For library and embedding users + +#### Upgrade to DataFusion 54 + +Ballista `54.0.0` upgrades its DataFusion dependency to `54.0.0`. Code built on +`ballista-core` / DataFusion APIs must move with it. See DataFusion's own +[Upgrade Guide for 54.0.0](https://datafusion.apache.org/library-user-guide/upgrading/54.0.0.html) +for the DataFusion-side API changes. + +#### `job_id` is now a newtype + +Scheduler job identifiers are no longer bare `String`s. `ballista-core` now +exports a transparent `JobId` newtype (`ballista_core::JobId`) so a job id can +no longer be confused with another string identifier at the type level. Public +signatures that previously carried a `String`/`&str` job id now carry a `JobId` +/ `&JobId` — for example `DistributedQueryExec::job_id()` now returns +`Option` instead of `Option`. Construct one with +`JobId::new(...)` or `.into()`, and recover the inner string with `.as_str()` +or `.into_inner()`. + +```rust +// before +let id: Option = query_exec.job_id(); + +// after +use ballista_core::JobId; +let id: Option = query_exec.job_id(); +let id_str: Option = id.map(|id| id.into_inner()); +``` + +### For cluster operators + +#### CORS origins and methods are configured via CLI flags + +The scheduler REST API no longer reads CORS configuration from environment +variables. The `BALLISTA_CORS_ALLOWED_ORIGINS` and +`BALLISTA_CORS_ALLOWED_METHODS` environment variables are removed and replaced +by the `--cors-allowed-origins` and `--cors-allowed-methods` scheduler CLI +flags (each a comma-separated list; `*` allows any). If you set these +environment variables, move the values to the flags — otherwise the scheduler +falls back to its defaults (origins `http://localhost:8080` and +`https://nightlies.apache.org`; methods `GET`, `PATCH`, `OPTIONS`). + +``` +# before (environment variables) +BALLISTA_CORS_ALLOWED_ORIGINS=https://example.com \ +BALLISTA_CORS_ALLOWED_METHODS=GET,PATCH \ + ballista-scheduler + +# after (scheduler CLI flags) +ballista-scheduler \ + --cors-allowed-origins https://example.com \ + --cors-allowed-methods GET,PATCH +``` + +### Planner and execution behavior changes + +#### The static planner now broadcasts small join build sides + +Ballista previously disabled `CollectLeft` broadcast joins entirely (both +`datafusion.optimizer.hash_join_single_partition_threshold` and +`hash_join_single_partition_threshold_rows` defaulted to `0`). In `54.0.0` +those defaults are raised to `10 MB` (`10485760`) and `1000000` rows, and the +static `DefaultDistributedPlanner` now converts a `SortMergeJoinExec` whose +smaller side fits under `ballista.optimizer.broadcast_join_threshold_bytes` +(default `10 MB`) into a broadcast `CollectLeft` hash join. This conversion is +governed by the new `ballista.optimizer.broadcast_sort_merge_join_enabled` +config key, which defaults to `true`. + +The consequence is that join plans — and therefore performance and shuffle +behavior — change on upgrade **even with AQE off**. Broadcast is applied only +to join types that are safe to broadcast (inner and right-side variants); other +join types remain repartitioned. To restore the previous behavior, set +`ballista.optimizer.broadcast_sort_merge_join_enabled = false`, +`ballista.optimizer.broadcast_join_threshold_bytes = 0`, +`datafusion.optimizer.hash_join_single_partition_threshold = 0`, and +`hash_join_single_partition_threshold_rows = 0`. + +#### Serialized empty projections + +The plan serialization for a filter that projects to zero columns changed. In +`53.x` an empty projection (`Some([])`) and "all columns" (`None`) both encoded +to an empty list and decoded back to `None`, shifting downstream column indices; +the scheduler now rewrites such filters into a serde-safe form before +serialization. Together with the DataFusion 54 protobuf bump, this means a +`54.0.0` scheduler and a `53.x` executor (or vice versa) cannot reliably +exchange serialized plans. Upgrade schedulers and executors together — do not +run a mixed-version cluster across this upgrade. + +#### AQE join behavior (opt-in) + +If you have enabled `ballista.planner.adaptive.enabled = true`, adaptive join +selection now delays the join decision and can promote a small build side to a +broadcast (`CollectLeft`) hash join at runtime, and empty-join inputs are +short-circuited by the propagate-empty rule. This changes the runtime join +plans (and their results' timing/shape) for adaptive clusters; it is gated by +`ballista.planner.adaptive_join.enabled` (default `true`). Clusters using the +default (AQE off) are unaffected. diff --git a/docs/source/upgrading/index.rst b/docs/source/upgrading/index.rst new file mode 100644 index 0000000000..078853ee91 --- /dev/null +++ b/docs/source/upgrading/index.rst @@ -0,0 +1,24 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Upgrade Guides +============== + +.. toctree:: + :maxdepth: 1 + + Ballista 54.0.0 <54.0.0> From e05fe5ea3a8f47bec2db27443f97ebd61088bfe2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 6 Jul 2026 07:58:08 -0600 Subject: [PATCH 026/107] feat: bound shuffle fetch with a reduce-side in-flight governor (#1951) --- ballista/client/tests/sort_shuffle.rs | 55 ++- ballista/core/src/client.rs | 12 + ballista/core/src/config.rs | 60 +++ .../src/execution_plans/distributed_query.rs | 2 + .../src/execution_plans/shuffle_reader.rs | 382 ++++++++++++++++-- ballista/core/src/utils.rs | 25 +- ballista/executor/src/client_pool.rs | 2 + examples/examples/standalone-substrait.rs | 2 + 8 files changed, 507 insertions(+), 33 deletions(-) diff --git a/ballista/client/tests/sort_shuffle.rs b/ballista/client/tests/sort_shuffle.rs index 48a27715fb..0a9cc4bf72 100644 --- a/ballista/client/tests/sort_shuffle.rs +++ b/ballista/client/tests/sort_shuffle.rs @@ -30,7 +30,12 @@ mod common; mod sort_shuffle_tests { use ballista::prelude::{SessionConfigExt, SessionContextExt}; use ballista_core::config::{ - BALLISTA_ADAPTIVE_PLANNER_ENABLED, BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ, + BALLISTA_ADAPTIVE_PLANNER_ENABLED, + BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE, + BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE, + BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ, + BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS, + BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT, BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT, BALLISTA_SHUFFLE_SORT_BASED_ENABLED, }; @@ -90,6 +95,28 @@ mod sort_shuffle_tests { SessionContext::standalone_with_state(state).await.unwrap() } + /// Remote-flight sort-shuffle context with a deliberately tiny governor + /// budget (64 KiB in-flight, 2 blocks/address) but a large connection window + /// (8 MiB). This forces many partition fetches to serialize through the + /// governor while multiplexing over the pooled connection — the scenario + /// that deadlocked before the governor existed. The query must still + /// complete and return correct results. + async fn create_tiny_budget_remote_context() -> SessionContext { + let config = SessionConfig::new_with_ballista() + .set_str(BALLISTA_SHUFFLE_SORT_BASED_ENABLED, "true") + .set_str(BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ, "true") + .set_str(BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT, "true") + .set_str(BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT, "65536") + .set_str(BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS, "2") + .set_str(BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE, "8388608") + .set_str(BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE, "8388608"); + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .build(); + SessionContext::standalone_with_state(state).await.unwrap() + } + /// Creates a standalone session context with hash-based shuffle. async fn create_hash_shuffle_context() -> SessionContext { let config = SessionConfig::new_with_ballista() @@ -568,4 +595,30 @@ mod sort_shuffle_tests { assert_result_eq(expected, &results); Ok(()) } + + // ==================== Governor Budget ==================== + + #[tokio::test] + async fn test_shuffle_completes_under_tiny_governor_budget() -> Result<()> { + let ctx = create_tiny_budget_remote_context().await; + register_test_data(&ctx).await; + + // GROUP BY forces a shuffle; the reducer fetches every upstream partition + // remotely under the tiny in-flight budget. + let df = ctx + .sql("SELECT bool_col, COUNT(*) as cnt FROM test GROUP BY bool_col ORDER BY bool_col") + .await?; + let results = df.collect().await?; + + let expected = vec![ + "+----------+-----+", + "| bool_col | cnt |", + "+----------+-----+", + "| false | 4 |", + "| true | 4 |", + "+----------+-----+", + ]; + assert_result_eq(expected, &results); + Ok(()) + } } diff --git a/ballista/core/src/client.rs b/ballista/core/src/client.rs index e3c26b158b..267ef83008 100644 --- a/ballista/core/src/client.rs +++ b/ballista/core/src/client.rs @@ -62,6 +62,7 @@ pub struct BallistaClient { impl BallistaClient { /// Create a new BallistaClient to connect to the executor listening on the specified /// host and port + #[allow(clippy::too_many_arguments)] pub async fn try_new( host: &str, port: u16, @@ -70,6 +71,8 @@ impl BallistaClient { customize_endpoint: Option>, io_retries_times: u8, io_retry_wait_time_ms: u64, + initial_connection_window_size: u32, + initial_stream_window_size: u32, ) -> BResult { let scheme = if use_tls { "https" } else { "http" }; @@ -83,6 +86,15 @@ impl BallistaClient { )) })?; + if initial_connection_window_size > 0 { + endpoint = endpoint + .initial_connection_window_size(Some(initial_connection_window_size)); + } + if initial_stream_window_size > 0 { + endpoint = + endpoint.initial_stream_window_size(Some(initial_stream_window_size)); + } + if let Some(customize) = customize_endpoint { endpoint = customize .configure_endpoint(endpoint) diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index 9a9bc6d7f4..087a646b8e 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -41,6 +41,21 @@ pub const BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ: &str = /// Configuration key to prefer Flight protocol for remote shuffle reads. pub const BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT: &str = "ballista.shuffle.remote_read_prefer_flight"; +/// Configuration key for the reduce-side in-flight-bytes governor budget. +pub const BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT: &str = + "ballista.shuffle.reader.max_bytes_in_flight"; +/// Configuration key for the per-address in-flight block cap. +pub const BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS: &str = + "ballista.shuffle.reader.max_blocks_in_flight_per_address"; +/// Configuration key for the assumed block size when partition stats lack a byte count. +pub const BALLISTA_SHUFFLE_READER_DEFAULT_BLOCK_SIZE: &str = + "ballista.shuffle.reader.default_block_size_bytes"; +/// Configuration key for the gRPC client HTTP/2 initial connection-level flow-control window. +pub const BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE: &str = + "ballista.client.initial_connection_window_size"; +/// Configuration key for the gRPC client HTTP/2 initial stream-level flow-control window. +pub const BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE: &str = + "ballista.client.initial_stream_window_size"; /// max message size for gRPC clients pub const BALLISTA_CLIENT_GRPC_MAX_MESSAGE_SIZE: &str = "ballista.client.grpc_max_message_size"; @@ -156,6 +171,26 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| "Forces the shuffle reader to use flight reader instead of block reader for remote read. Block reader usually has better performance and resource utilization".to_string(), DataType::Boolean, Some((false).to_string())), + ConfigEntry::new(BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT.to_string(), + "Reduce-side shuffle governor: maximum total in-flight bytes across concurrent remote partition fetches. Mirrors Spark's spark.reducer.maxSizeInFlight. Values above 4 GiB are clamped to 4 GiB (u32 semaphore limit).".to_string(), + DataType::UInt64, + Some((50331648).to_string())), + ConfigEntry::new(BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS.to_string(), + "Reduce-side shuffle governor: maximum concurrent in-flight partition fetches to a single executor address.".to_string(), + DataType::UInt64, + Some((128).to_string())), + ConfigEntry::new(BALLISTA_SHUFFLE_READER_DEFAULT_BLOCK_SIZE.to_string(), + "Assumed per-partition byte size charged to the shuffle governor when partition stats carry no byte count.".to_string(), + DataType::UInt64, + Some((1048576).to_string())), + ConfigEntry::new(BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE.to_string(), + "HTTP/2 initial connection-level flow-control window for gRPC data-plane clients, in bytes. Should be >= the shuffle governor byte budget so the governor, not the transport window, is the binding backpressure. 0 leaves the tonic default.".to_string(), + DataType::UInt64, + Some((67108864).to_string())), + ConfigEntry::new(BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE.to_string(), + "HTTP/2 initial stream-level flow-control window for gRPC data-plane clients, in bytes. 0 leaves the tonic default.".to_string(), + DataType::UInt64, + Some((16777216).to_string())), ConfigEntry::new(BALLISTA_CLIENT_GRPC_MAX_MESSAGE_SIZE.to_string(), "Configuration for max message size in gRPC clients".to_string(), DataType::UInt64, @@ -425,6 +460,31 @@ impl BallistaConfig { self.get_usize_setting(BALLISTA_SHUFFLE_READER_MAX_REQUESTS) } + /// Reduce-side shuffle governor byte budget (`max_bytes_in_flight`). + pub fn shuffle_reader_max_bytes_in_flight(&self) -> u64 { + self.get_usize_setting(BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT) as u64 + } + + /// Reduce-side shuffle governor per-address in-flight block cap. + pub fn shuffle_reader_max_blocks_in_flight_per_address(&self) -> usize { + self.get_usize_setting(BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS) + } + + /// Assumed block size charged to the governor when stats lack a byte count. + pub fn shuffle_reader_default_block_size_bytes(&self) -> u64 { + self.get_usize_setting(BALLISTA_SHUFFLE_READER_DEFAULT_BLOCK_SIZE) as u64 + } + + /// HTTP/2 initial connection-level flow-control window (bytes) for data-plane clients. + pub fn grpc_client_initial_connection_window_size(&self) -> u32 { + self.get_usize_setting(BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE) as u32 + } + + /// HTTP/2 initial stream-level flow-control window (bytes) for data-plane clients. + pub fn grpc_client_initial_stream_window_size(&self) -> u32 { + self.get_usize_setting(BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE) as u32 + } + /// Returns the gRPC client connection timeout in seconds. pub fn grpc_client_connect_timeout_seconds(&self) -> usize { self.get_usize_setting(BALLISTA_CLIENT_GRPC_CONNECT_TIMEOUT_SECONDS) diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs index 137cde1038..20f3234cfe 100644 --- a/ballista/core/src/execution_plans/distributed_query.rs +++ b/ballista/core/src/execution_plans/distributed_query.rs @@ -825,6 +825,8 @@ async fn fetch_partition( customize_endpoint, io_retries_times, io_retry_wait_time_ms, + 0, + 0, ) .await .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index a9955ac5f5..89ae76f73f 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -50,13 +50,15 @@ use rand::rng; use std::collections::HashMap; use std::fmt::Debug; use std::fs::File; +use std::future::Future; use std::io::BufReader; use std::path::Path; use std::pin::Pin; use std::result; use std::sync::Arc; use std::task::{Context, Poll}; -use tokio::sync::{Semaphore, mpsc}; +use std::time::Duration; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; use tokio_stream::wrappers::ReceiverStream; /// Coalesce plan attached to a `ShuffleReaderExec` or `UnresolvedShuffleExec`. @@ -612,6 +614,77 @@ impl Stream for AbortableReceiverStream { .map_err(|e| ArrowError::ExternalError(Box::new(e))) } } + +/// In-flight bytes charged to the governor for a block. Uses the partition's +/// recorded byte size, falling back to `default` when stats carry none. Never 0, +/// so every block occupies at least one permit. +fn block_size(location: &PartitionLocation, default: u64) -> u64 { + location + .partition_stats + .num_bytes() + .unwrap_or(default) + .max(1) +} + +/// Size of the byte semaphore. tokio permits are `usize`; clamp so it is at least +/// 1 and never exceeds `u32::MAX` (the `acquire_many` argument type). +fn byte_permits_cap(max_bytes: u64) -> usize { + max_bytes.clamp(1, u32::MAX as u64) as usize +} + +/// Byte permits to acquire for a block: `min(size, max_bytes)`, clamped to +/// `[1, u32::MAX]`. Capping at `max_bytes` means an oversized block requests the +/// entire budget and can only proceed once all other fetches drain — the +/// application-layer analog of Spark's `bytesInFlight == 0` progress clause. +fn byte_permits_for(size: u64, max_bytes: u64) -> u32 { + let cap = max_bytes.clamp(1, u32::MAX as u64); + size.clamp(1, cap) as u32 +} + +/// Wraps a fetched partition stream and holds the governor permits for its +/// lifetime. Dropping the stream — on normal end, consumer cancellation, or a +/// mid-body error — releases all three permits, freeing budget for the next +/// fetch. This is the release-on-body-completion behavior the governor needs. +struct GovernedStream { + inner: SendableRecordBatchStream, + _byte_permit: OwnedSemaphorePermit, + _req_permit: OwnedSemaphorePermit, + _addr_permit: OwnedSemaphorePermit, +} + +impl GovernedStream { + fn new( + inner: SendableRecordBatchStream, + byte_permit: OwnedSemaphorePermit, + req_permit: OwnedSemaphorePermit, + addr_permit: OwnedSemaphorePermit, + ) -> Self { + Self { + inner, + _byte_permit: byte_permit, + _req_permit: req_permit, + _addr_permit: addr_permit, + } + } +} + +impl Stream for GovernedStream { + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.inner.poll_next_unpin(cx) + } +} + +impl RecordBatchStream for GovernedStream { + fn schema(&self) -> SchemaRef { + self.inner.schema() + } +} + /// Splits the provided partition locations into local and remote partitions. /// Local partitions are read directly from local Arrow IPC files, /// while remote partitions are fetched using the Arrow Flight client. @@ -636,11 +709,30 @@ fn send_fetch_partitions( config: &SessionConfig, client_pool: Option>, ) -> AbortableReceiverStream { - let max_request_num = config.ballista_shuffle_reader_maximum_concurrent_requests(); + let ballista_config = config.ballista_config(); + let max_reqs = config.ballista_shuffle_reader_maximum_concurrent_requests(); + let max_bytes = ballista_config.shuffle_reader_max_bytes_in_flight(); + let max_blocks_per_addr = + ballista_config.shuffle_reader_max_blocks_in_flight_per_address(); + let default_block_size = ballista_config.shuffle_reader_default_block_size_bytes(); let sort_shuffle_enabled = config.ballista_sort_shuffle_enabled(); - let (response_sender, response_receiver) = mpsc::channel(max_request_num); - let semaphore = Arc::new(Semaphore::new(max_request_num)); + let (response_sender, response_receiver) = mpsc::channel(max_reqs.max(1)); + + // Reduce-side in-flight governor. Each remote fetch acquires: + // * `byte_sem` — min(block_size, max_bytes) permits (in-flight-bytes budget) + // * `req_sem` — 1 permit (in-flight-request count) + // * an addr semaphore — 1 permit (per-address in-flight cap) + // All three are held for the lifetime of the returned stream (see + // GovernedStream), so budget is released only when the body is fully + // consumed. Because total in-flight bytes stay <= the sized h2 window, the + // governor — not the 64 KB transport window — is the binding backpressure, + // which is what makes multiplexing over few connections safe. + let byte_sem = Arc::new(Semaphore::new(byte_permits_cap(max_bytes))); + let req_sem = Arc::new(Semaphore::new(max_reqs.max(1))); + let addr_sems: Arc>>> = + Arc::new(std::sync::Mutex::new(HashMap::new())); + let mut spawned_tasks: Vec> = vec![]; let (local_locations, remote_locations): (Vec<_>, Vec<_>) = local_remote_read_split( @@ -657,10 +749,6 @@ fn send_fetch_partitions( // keep local shuffle files reading in serial order for memory control. let response_sender_c = response_sender.clone(); - - // - // fetching local partitions (read from file) - // let work_dir = work_dir.to_string(); spawned_tasks.push(SpawnedTask::spawn_blocking({ move || { @@ -673,38 +761,74 @@ fn send_fetch_partitions( } })); - // - // fetching remote partitions (uses grpc flight protocol) - // let grpc_config: Arc = Arc::new((&config.ballista_config()).into()); let customize_endpoint = config.ballista_override_create_grpc_client_endpoint(); let prefer_flight = config.ballista_shuffle_reader_remote_prefer_flight(); + // The reduce-side with_retry owns fetch retries; disable the client's inner + // establish-retry loop (set to a single attempt) so the two layers don't + // multiply. Read the retry budget BEFORE overriding it for the client. + let outer_retries = grpc_config.io_retries_times; + let io_wait = grpc_config.io_retry_wait_time_ms; + let client_grpc_config: Arc = { + let mut c = (*grpc_config).clone(); + c.io_retries_times = 1; + Arc::new(c) + }; + for p in remote_locations.into_iter() { - let semaphore = semaphore.clone(); + let byte_sem = byte_sem.clone(); + let req_sem = req_sem.clone(); + let addr_sems = addr_sems.clone(); let response_sender = response_sender.clone(); + let customize_endpoint = customize_endpoint.clone(); + let client_grpc_config = client_grpc_config.clone(); + let client_pool = client_pool.clone(); + + spawned_tasks.push(SpawnedTask::spawn(async move { + let addr = p.executor_meta.id.clone(); + let size = block_size(&p, default_block_size); + + // Acquire the cheap count/address gates first, then the byte budget + // last, so a fetch does not hold scarce byte permits while blocked + // on a per-address slot. All acquires are on Arcs that + // are never closed, so `unwrap()` cannot panic. + let req_permit = req_sem.acquire_owned().await.unwrap(); + let addr_sem = { + let mut map = addr_sems.lock().unwrap(); + map.entry(addr.clone()) + .or_insert_with(|| { + Arc::new(Semaphore::new(max_blocks_per_addr.max(1))) + }) + .clone() + }; + let addr_permit = addr_sem.acquire_owned().await.unwrap(); + let byte_permit = byte_sem + .acquire_many_owned(byte_permits_for(size, max_bytes)) + .await + .unwrap(); - spawned_tasks.push(SpawnedTask::spawn({ - let customize_endpoint = customize_endpoint.clone(); - let grpc_config = grpc_config.clone(); - let client_pool = client_pool.clone(); - async move { - // Block if exceeds max request number. - let permit = semaphore.acquire_owned().await.unwrap(); - let r = fetch_partition_remote( + let r = with_retry(outer_retries, io_wait, is_retriable_fetch_error, || { + fetch_partition_buffered( &p, - grpc_config, + client_grpc_config.clone(), prefer_flight, - customize_endpoint, - client_pool, + customize_endpoint.clone(), + client_pool.clone(), ) - .await; - // Block if the channel buffer is full. - if let Err(e) = response_sender.send(r).await { - error!("Fail to send response event to the channel due to {e}"); - } - // Increase semaphore by dropping existing permits. - drop(permit); + }) + .await + .map(|(schema, batches)| { + Box::pin(GovernedStream::new( + buffered_stream(schema, batches), + byte_permit, + req_permit, + addr_permit, + )) as SendableRecordBatchStream + }); + + if let Err(e) = response_sender.send(r).await { + error!("Fail to send response event to the channel due to {e}"); } })); } @@ -712,6 +836,101 @@ fn send_fetch_partitions( AbortableReceiverStream::create(response_receiver, spawned_tasks) } +/// Retry an idempotent async operation up to `retries` times after the initial +/// attempt, sleeping `wait_ms` between tries. Shuffle partition fetches are +/// idempotent (the server re-reads the file), so a transport error — including +/// one mid-body — can be recovered by refetching the whole partition. Only +/// errors accepted by `should_retry` are retried; anything else is returned +/// immediately on the first failure. +async fn with_retry( + retries: u8, + wait_ms: u64, + should_retry: impl Fn(&BallistaError) -> bool, + mut f: F, +) -> result::Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut attempt: u8 = 0; + loop { + match f().await { + Ok(v) => return Ok(v), + Err(e) => { + if attempt >= retries || !should_retry(&e) { + return Err(e); + } + attempt += 1; + debug!( + "retrying shuffle fetch (attempt {attempt}/{retries}) after error: {e}" + ); + if wait_ms > 0 { + tokio::time::sleep(Duration::from_millis(wait_ms)).await; + } + } + } + } +} + +/// Transport/fetch failures worth refetching an idempotent shuffle block for. +/// Deterministic failures (bad config, schema/decoding logic errors) are not +/// retried. Mirrors the transport-only retry policy of the client's inner loop. +fn is_retriable_fetch_error(e: &BallistaError) -> bool { + matches!( + e, + BallistaError::GrpcConnectionError(_) | BallistaError::FetchFailed(..) + ) +} + +/// Fetch a remote partition and buffer its entire body into memory, turning the +/// open-ended stream into a discrete, refetchable unit. Buffering is what makes +/// a mid-body transport failure retriable without emitting duplicate batches +/// downstream. The governor charges each block its compressed (serialized) size, +/// so the byte budget bounds concurrent in-flight **wire** bytes — which is the +/// quantity the h2 window must accommodate. The decoded in-memory footprint is +/// larger by the Arrow/compression expansion ratio, so peak buffered RAM exceeds +/// the byte budget by that factor. Bounding decoded memory precisely is the +/// deferred disk-spill work. +async fn fetch_partition_buffered( + location: &PartitionLocation, + config: Arc, + prefer_flight: bool, + customize_endpoint: Option>, + client_pool: Option>, +) -> result::Result<(SchemaRef, Vec), BallistaError> { + let stream = fetch_partition_remote( + location, + config, + prefer_flight, + customize_endpoint, + client_pool, + ) + .await?; + let schema = stream.schema(); + let metadata = &location.executor_meta; + let partition_id = &location.partition_id; + let batches = stream.try_collect::>().await.map_err(|e| { + BallistaError::FetchFailed( + metadata.id.clone(), + partition_id.stage_id, + partition_id.partition_id, + e.to_string(), + ) + })?; + Ok((schema, batches)) +} + +/// Build an in-memory `SendableRecordBatchStream` from already-buffered batches. +fn buffered_stream( + schema: SchemaRef, + batches: Vec, +) -> SendableRecordBatchStream { + Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches.into_iter().map(Ok)), + )) +} + async fn new_ballista_client( host: &str, port: u16, @@ -731,6 +950,8 @@ async fn new_ballista_client( customize_endpoint, io_retries_times, io_retry_wait_time_ms, + config.initial_connection_window_size, + config.initial_stream_window_size, ) .await } @@ -1845,3 +2066,104 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod governor_tests { + use super::*; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + + #[test] + fn byte_permits_for_caps_at_budget() { + // A block larger than the budget requests exactly the whole budget, + // so it can only run when all other fetches have drained. + assert_eq!(byte_permits_for(10, 100), 10); + assert_eq!(byte_permits_for(500, 100), 100); + // Never zero (a zero acquire is a no-op and would under-account). + assert_eq!(byte_permits_for(0, 100), 1); + } + + #[test] + fn byte_permits_cap_is_nonzero_and_bounded() { + assert_eq!(byte_permits_cap(0), 1); + assert_eq!(byte_permits_cap(48 * 1024 * 1024), 48 * 1024 * 1024); + } + + #[tokio::test] + async fn governed_stream_releases_permits_on_drop() { + let byte_sem = Arc::new(Semaphore::new(100)); + let req_sem = Arc::new(Semaphore::new(4)); + let addr_sem = Arc::new(Semaphore::new(2)); + + let byte = byte_sem.clone().acquire_many_owned(30).await.unwrap(); + let req = req_sem.clone().acquire_owned().await.unwrap(); + let addr = addr_sem.clone().acquire_owned().await.unwrap(); + assert_eq!(byte_sem.available_permits(), 70); + assert_eq!(req_sem.available_permits(), 3); + assert_eq!(addr_sem.available_permits(), 1); + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + let empty = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::empty(), + )) as SendableRecordBatchStream; + let governed = GovernedStream::new(empty, byte, req, addr); + drop(governed); + + assert_eq!(byte_sem.available_permits(), 100); + assert_eq!(req_sem.available_permits(), 4); + assert_eq!(addr_sem.available_permits(), 2); + } + + #[tokio::test] + async fn with_retry_succeeds_after_transient_failures() { + use std::sync::atomic::{AtomicU8, Ordering}; + let calls = AtomicU8::new(0); + let result: result::Result = + with_retry(3, 0, is_retriable_fetch_error, || { + let n = calls.fetch_add(1, Ordering::SeqCst); + async move { + if n < 2 { + Err(BallistaError::GrpcConnectionError("transient".to_string())) + } else { + Ok(42) + } + } + }) + .await; + assert_eq!(result.unwrap(), 42); + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn with_retry_gives_up_after_max_attempts() { + use std::sync::atomic::{AtomicU8, Ordering}; + let calls = AtomicU8::new(0); + let result: result::Result = + with_retry(2, 0, is_retriable_fetch_error, || { + calls.fetch_add(1, Ordering::SeqCst); + async move { + Err(BallistaError::GrpcConnectionError("always".to_string())) + } + }) + .await; + assert!(result.is_err()); + // initial attempt + 2 retries = 3 calls + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn with_retry_does_not_retry_non_retriable() { + use std::sync::atomic::{AtomicU8, Ordering}; + let calls = AtomicU8::new(0); + let result: result::Result = + with_retry(3, 0, is_retriable_fetch_error, || { + calls.fetch_add(1, Ordering::SeqCst); + async move { Err(BallistaError::General("deterministic".to_string())) } + }) + .await; + assert!(result.is_err()); + // A non-retriable error must not trigger any retry attempts. + assert_eq!(calls.load(Ordering::SeqCst), 1); + } +} diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs index 8428884680..ffe7804c83 100644 --- a/ballista/core/src/utils.rs +++ b/ballista/core/src/utils.rs @@ -72,6 +72,10 @@ pub struct GrpcClientConfig { pub io_retries_times: u8, /// Wait time in milliseconds between IO retries. pub io_retry_wait_time_ms: u64, + /// HTTP/2 initial connection-level flow-control window in bytes. 0 = tonic default. + pub initial_connection_window_size: u32, + /// HTTP/2 initial stream-level flow-control window in bytes. 0 = tonic default. + pub initial_stream_window_size: u32, } impl From<&BallistaConfig> for GrpcClientConfig { @@ -87,6 +91,9 @@ impl From<&BallistaConfig> for GrpcClientConfig { max_message_size: config.grpc_client_max_message_size(), io_retries_times: config.io_retries_times() as u8, io_retry_wait_time_ms: config.io_retry_wait_time_ms() as u64, + initial_connection_window_size: config + .grpc_client_initial_connection_window_size(), + initial_stream_window_size: config.grpc_client_initial_stream_window_size(), } } } @@ -102,6 +109,8 @@ impl Default for GrpcClientConfig { max_message_size: 16 * 1024 * 1024, io_retries_times: 3, io_retry_wait_time_ms: 3000, + initial_connection_window_size: 67108864, + initial_stream_window_size: 16777216, } } } @@ -294,7 +303,7 @@ where { let endpoint = tonic::transport::Endpoint::new(dst)?; if let Some(config) = config { - Ok(endpoint + let mut endpoint = endpoint .connect_timeout(Duration::from_secs(config.connect_timeout_seconds)) .timeout(Duration::from_secs(config.timeout_seconds)) .tcp_nodelay(true) @@ -303,7 +312,17 @@ where config.http2_keepalive_interval_seconds, )) .keep_alive_timeout(Duration::from_secs(20)) - .keep_alive_while_idle(true)) + .keep_alive_while_idle(true); + if config.initial_connection_window_size > 0 { + endpoint = endpoint.initial_connection_window_size(Some( + config.initial_connection_window_size, + )); + } + if config.initial_stream_window_size > 0 { + endpoint = endpoint + .initial_stream_window_size(Some(config.initial_stream_window_size)); + } + Ok(endpoint) } else { Ok(endpoint) } @@ -396,6 +415,8 @@ mod tests { max_message_size: 16 * 1024 * 1024, io_retries_times: 3, io_retry_wait_time_ms: 3000, + initial_connection_window_size: 67108864, + initial_stream_window_size: 16777216, }; let result = create_grpc_client_endpoint("http://localhost:50051", Some(&config)); assert!(result.is_ok()); diff --git a/ballista/executor/src/client_pool.rs b/ballista/executor/src/client_pool.rs index ed1260c0a0..cb87a012b3 100644 --- a/ballista/executor/src/client_pool.rs +++ b/ballista/executor/src/client_pool.rs @@ -185,6 +185,8 @@ impl BallistaClientPool for DefaultBallistaClientPool { customize_endpoint, config.io_retries_times, config.io_retry_wait_time_ms, + config.initial_connection_window_size, + config.initial_stream_window_size, ) .await? } diff --git a/examples/examples/standalone-substrait.rs b/examples/examples/standalone-substrait.rs index 762a9c82e4..54388e5907 100644 --- a/examples/examples/standalone-substrait.rs +++ b/examples/examples/standalone-substrait.rs @@ -422,6 +422,8 @@ impl SubstraitSchedulerClient { None, io_retries_times, io_retry_wait_time_ms, + 0, + 0, ) .await .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; From ce9eefabc6a2306f80e78eaeb5e39669e00c2fd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:14:03 +0200 Subject: [PATCH 027/107] chore(deps): bump ctor from 1.0.7 to 1.0.8 (#1957) Bumps [ctor](https://github.com/mmastrac/linktime) from 1.0.7 to 1.0.8. - [Release notes](https://github.com/mmastrac/linktime/releases) - [Changelog](https://github.com/mmastrac/linktime/blob/master/CHANGELOG.md) - [Commits](https://github.com/mmastrac/linktime/compare/ctor-1.0.7...ctor-1.0.8) --- updated-dependencies: - dependency-name: ctor dependency-version: 1.0.8 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eec4cd1d77..0ec612c077 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2094,9 +2094,9 @@ dependencies = [ [[package]] name = "ctor" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01334b89b69ff726750c5ce5073fc8bd860e99aa9a8fc5ca11b04730e3aee97a" +checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb" dependencies = [ "link-section", "linktime-proc-macro", @@ -4432,9 +4432,9 @@ dependencies = [ [[package]] name = "link-section" -version = "0.18.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2b1dd6fe32e55c0fc0ea9493aa57459ca3cf4ff3c857c7d0302290150da6e4f" +checksum = "e333fe507b738576d6da5bb3f1a7d7a1c80307ed9ef31624c057d844c19c93e9" [[package]] name = "linktime-proc-macro" From 182222743749b40625d9be63174b32b54cd2af4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:14:30 +0200 Subject: [PATCH 028/107] chore(deps): bump bytesize from 2.4.0 to 2.4.2 (#1956) Bumps [bytesize](https://github.com/bytesize-rs/bytesize) from 2.4.0 to 2.4.2. - [Release notes](https://github.com/bytesize-rs/bytesize/releases) - [Changelog](https://github.com/bytesize-rs/bytesize/blob/master/CHANGELOG.md) - [Commits](https://github.com/bytesize-rs/bytesize/compare/bytesize-v2.4.0...bytesize-v2.4.2) --- updated-dependencies: - dependency-name: bytesize dependency-version: 2.4.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ec612c077..50ee692b8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,9 +1524,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.4.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" +checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e" [[package]] name = "bzip2" From e0855643d6a231cb7aa71cc2593672e194c34cf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:15:00 +0200 Subject: [PATCH 029/107] chore(deps): bump astral-sh/setup-uv from 8.2.0 to 8.3.0 (#1955) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 8.2.0 to 8.3.0. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/fac544c07dec837d0ccb6301d7b5580bf5edae39...d31148d669074a8d0a63714ba94f3201e7020bc3) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 8.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4aea4d977..79018b07e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true @@ -77,7 +77,7 @@ jobs: with: python-version: "3.12" - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true @@ -100,7 +100,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7.0.0 - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true - name: Generate license file @@ -148,7 +148,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true @@ -194,7 +194,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true @@ -236,7 +236,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true @@ -278,7 +278,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: true From a96c381fb1eb038ae19fcf4d180b9a640401462e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:16:12 +0200 Subject: [PATCH 030/107] chore(deps): bump taiki-e/install-action from 2.82.6 to 2.82.7 (#1918) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.82.6 to 2.82.7. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/9bcaee1dcae34154180f412e2fa69355a7cda9f6...16b05812d776ae1dfaabc8277e421fb6d2506419) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.82.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bhargava Vadlamani <11091419+coderfender@users.noreply.github.com> --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index da851b7402..076eb711c8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 38dd179c24..2bed39bef4 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index a3ea4838d9..2290f62c19 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,12 +64,12 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 with: tool: trunk@0.21.14 - name: Install cargo-get - uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 + uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 with: tool: cargo-get@1.4.0 From 24e95f9ffc64002af81002cb50fea98c1b1c90c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Tue, 7 Jul 2026 16:45:49 +0200 Subject: [PATCH 031/107] fix AQE planning with S3 (#1962) --- ballista/scheduler/src/state/aqe/planner.rs | 22 ++- examples/tests/object_store.rs | 143 ++++++++++++++++++++ 2 files changed, 161 insertions(+), 4 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/planner.rs b/ballista/scheduler/src/state/aqe/planner.rs index 844bcc32b1..4981912271 100644 --- a/ballista/scheduler/src/state/aqe/planner.rs +++ b/ballista/scheduler/src/state/aqe/planner.rs @@ -31,6 +31,7 @@ use datafusion::common; use datafusion::common::{HashMap, exec_err}; use datafusion::error::DataFusionError; use datafusion::execution::context::SessionContext; +use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::logical_expr::LogicalPlan; use datafusion::physical_optimizer::PhysicalOptimizerRule; @@ -87,6 +88,7 @@ impl AdaptivePlanner { /// # Arguments: /// /// * `session_config` - The session configuration for the job. + /// * `runtime_env` - runtime environment /// * `plan` - The physical execution plan for the job. /// * `job_name` - The name of the job. /// * `physical_optimizer_rules` - A list of physical optimizer rules to apply. @@ -95,12 +97,16 @@ impl AdaptivePlanner { /// A new instance of `AdaptivePlanner` or an error if the initialization fails. pub fn try_new_with_optimizers( session_config: &SessionConfig, + runtime_env: Arc, plan: Arc, job_name: String, physical_optimizer_rules: Vec, ) -> common::Result { - let session_state = - Self::create_session_state(session_config, physical_optimizer_rules); + let session_state = Self::create_session_state( + session_config, + runtime_env, + physical_optimizer_rules, + ); let planner = DefaultPhysicalPlanner::default(); let plan = planner.optimize_physical_plan(plan, &session_state, |_, _| {})?; @@ -133,6 +139,7 @@ impl AdaptivePlanner { let plan_id_generator = Arc::new(AtomicUsize::new(0)); Self::try_new_with_optimizers( session_config, + RuntimeEnv::default().into(), plan, job_name, Self::default_optimizers(plan_id_generator), @@ -159,12 +166,16 @@ impl AdaptivePlanner { // running standard set of optimizers, which will // after each stage. let plan_id_generator = Arc::new(AtomicUsize::new(0)); - let state = Self::create_session_state( + let runtime_env = ctx.runtime_env(); + let plan_preparation_stage = Self::create_session_state( ctx.state().config(), + ctx.runtime_env(), Self::plan_preparation_optimizers(plan_id_generator.clone()), ); - let plan = state.create_physical_plan(logical_plan).await?; + let plan = plan_preparation_stage + .create_physical_plan(logical_plan) + .await?; // Note: the signature requires a JobId, but we are passing a JobName. The below is a // dirty fix but this seems like a bug or a design flaw. @@ -175,6 +186,7 @@ impl AdaptivePlanner { Self::try_new_with_optimizers( ctx.state().config(), + runtime_env, plan, job_name, Self::default_optimizers(plan_id_generator), @@ -546,11 +558,13 @@ impl AdaptivePlanner { /// A new `SessionState` instance. fn create_session_state( session_config: &SessionConfig, + runtime_env: Arc, physical_optimizers: Vec, ) -> SessionState { SessionStateBuilder::new_with_default_features() .with_physical_optimizer_rules(physical_optimizers) .with_config(session_config.clone()) + .with_runtime_env(runtime_env) .build() } /// Recursively finds runnable exchanges in the execution plan. diff --git a/examples/tests/object_store.rs b/examples/tests/object_store.rs index b79712ea9b..6202bb8f9a 100644 --- a/examples/tests/object_store.rs +++ b/examples/tests/object_store.rs @@ -371,6 +371,149 @@ mod custom_s3_config { Ok(()) } + #[tokio::test] + async fn should_configure_aqe_s3_execute_sql_write_remote() + -> datafusion::error::Result<()> { + let test_data = examples_test_data(); + + // + // Minio cluster setup + // + let container = crate::common::create_minio_container(); + let node = container.start().await.unwrap(); + + node.exec(crate::common::create_bucket_command()) + .await + .unwrap(); + + let endpoint_host = node.get_host().await.unwrap(); + let endpoint_port = node.get_host_port_ipv4(9000).await.unwrap(); + + log::info!( + "MINIO testcontainers host: {}, port: {}", + endpoint_host, + endpoint_port + ); + + // + // Session Context and Ballista cluster setup + // + + // Setting up configuration producer + // + // configuration producer registers user defined config extension + // S3Option with relevant S3 configuration + let config_producer = + Arc::new(ballista_core::object_store::session_config_with_s3_support); + // Setting up runtime producer + // + // Runtime producer creates object store registry + // which can create object store connecter based on + // S3Option configuration. + let runtime_producer: RuntimeProducer = + Arc::new(ballista_core::object_store::runtime_env_with_s3_support); + + // Session builder creates SessionState + // + // which is configured using runtime and configuration producer, + // producing same runtime environment, and providing same + // object store registry. + + let session_builder = + Arc::new(ballista_core::object_store::session_state_with_s3_support); + + let state = session_builder(config_producer())?; + + // setting up ballista cluster with new runtime, configuration, and session state producers + let (host, port) = crate::common::setup_test_cluster_with_builders( + config_producer, + runtime_producer, + session_builder, + ) + .await; + let url = format!("df://{host}:{port}"); + + // establishing cluster connection, + let ctx: SessionContext = SessionContext::remote_with_state(&url, state).await?; + + ctx.sql("SET ballista.planner.adaptive.enabled = true") + .await? + .show() + .await?; + // setting up relevant S3 options + ctx.sql("SET s3.allow_http = true").await?.show().await?; + ctx.sql(&format!("SET s3.access_key_id = '{}'", ACCESS_KEY_ID)) + .await? + .show() + .await?; + ctx.sql(&format!("SET s3.secret_access_key = '{}'", SECRET_KEY)) + .await? + .show() + .await?; + ctx.sql(&format!( + "SET s3.endpoint = 'http://{}:{}'", + endpoint_host, endpoint_port + )) + .await? + .show() + .await?; + ctx.sql("SET s3.allow_http = true").await?.show().await?; + + // verifying that we have set S3Options + ctx.sql("select name, value from information_schema.df_settings where name like 's3.%'").await?.show().await?; + + ctx.register_parquet( + "test", + &format!("{test_data}/alltypes_plain.parquet"), + Default::default(), + ) + .await?; + + let write_dir_path = + &format!("s3://{}/write_test.parquet", crate::common::BUCKET); + + ctx.sql("select * from test") + .await? + .write_parquet(write_dir_path, Default::default(), Default::default()) + .await?; + + ctx.register_parquet("written_table", write_dir_path, Default::default()) + .await?; + + let result = ctx + .sql("select id, string_col, timestamp_col from written_table where id > 4") + .await? + .collect() + .await?; + let expected = [ + "+----+------------+---------------------+", + "| id | string_col | timestamp_col |", + "+----+------------+---------------------+", + "| 5 | 31 | 2009-03-01T00:01:00 |", + "| 6 | 30 | 2009-04-01T00:00:00 |", + "| 7 | 31 | 2009-04-01T00:01:00 |", + "+----+------------+---------------------+", + ]; + + assert_batches_eq!(expected, &result); + + let result = ctx + .sql("select max(id) as max, min(id) as min from written_table") + .await? + .collect() + .await?; + let expected = [ + "+-----+-----+", + "| max | min |", + "+-----+-----+", + "| 7 | 0 |", + "+-----+-----+", + ]; + + assert_batches_eq!(expected, &result); + Ok(()) + } + // this test shows how to register external ObjectStoreRegistry and configure it // using infrastructure provided by ballista standalone. // From ab34429d1f1fe1c80d1eb2b5896a38d7ac18f0dd Mon Sep 17 00:00:00 2001 From: jeadie Date: Wed, 8 Jul 2026 16:56:20 +1000 Subject: [PATCH 032/107] style: cargo fmt execution_loop.rs --- ballista/executor/src/execution_loop.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index 5ece368485..6ace64e69a 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -114,10 +114,9 @@ where loop { // Wait for task slots to be available before asking for new work - let permit = available_task_slots - .acquire() - .await - .map_err(|_| BallistaError::Internal("task slot semaphore closed".to_string()))?; + let permit = available_task_slots.acquire().await.map_err(|_| { + BallistaError::Internal("task slot semaphore closed".to_string()) + })?; // Make the slot available again drop(permit); @@ -163,15 +162,14 @@ where let task_status_sender = task_status_sender.clone(); // Acquire a permit/slot for the task - let permit = available_task_slots - .clone() - .acquire_owned() - .await - .map_err(|_| { - BallistaError::Internal( - "task slot semaphore closed".to_string(), - ) - })?; + let permit = + available_task_slots.clone().acquire_owned().await.map_err( + |_| { + BallistaError::Internal( + "task slot semaphore closed".to_string(), + ) + }, + )?; let start_exec_time = SystemTime::now() .duration_since(UNIX_EPOCH) From 4f1d7da525624f1642efc408bdc241ea6b8999a6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 8 Jul 2026 14:00:30 -0600 Subject: [PATCH 033/107] Add TPC-H distributed plan-stability test suite (#1966) * test: add dataless stats-injecting table provider for plan stability * test: add TPC-H fixtures and distributed staged-plan text helper Add the 8 TPC-H schemas (copied verbatim from benchmarks/src/bin/tpch.rs), SF100 row-count constants, and staged_plan_text(query_name) which loads a TPC-H query's SQL, registers stats-only tables, builds the physical plan with the Ballista-configured planner (target_partitions=16), splits it into distributed query stages via DefaultDistributedPlanner, and renders the stages to normalized text. Copy the 22 TPC-H query SQL files and add helper tests covering a single-statement query (q1) and a multi-statement query with view DDL (q15). * test: add TPC-H distributed plan-stability suite with approved plans * ci: add regenerate script and README for TPC-H plan-stability suite Add dev/update-tpch-plan-stability.sh to regenerate approved golden plans, and document the suite's scope, usage, and existing CI coverage. No workflow changes needed: rust.yml already runs workspace-wide cargo test/clippy/fmt jobs that pick up the new [[test]] target automatically. * style: apply cargo fmt to TPC-H plan-stability test files Fix outstanding cargo fmt --check deltas in fixtures.rs, stats_table.rs, and the plan_stability_test! macro invocation in main.rs. * ci: exclude plan-stability goldens from RAT and add license header * test: read TPC-H queries from benchmarks/queries instead of copying --- Cargo.lock | 1 + ballista/scheduler/Cargo.toml | 5 + .../tests/tpch_plan_stability/README.md | 59 +++++ .../tests/tpch_plan_stability/approved/q1.txt | 18 ++ .../tpch_plan_stability/approved/q10.txt | 60 +++++ .../tpch_plan_stability/approved/q11.txt | 87 +++++++ .../tpch_plan_stability/approved/q12.txt | 30 +++ .../tpch_plan_stability/approved/q13.txt | 32 +++ .../tpch_plan_stability/approved/q14.txt | 25 ++ .../tpch_plan_stability/approved/q15.txt | 57 +++++ .../tpch_plan_stability/approved/q16.txt | 46 ++++ .../tpch_plan_stability/approved/q17.txt | 36 +++ .../tpch_plan_stability/approved/q18.txt | 46 ++++ .../tpch_plan_stability/approved/q19.txt | 26 ++ .../tests/tpch_plan_stability/approved/q2.txt | 137 +++++++++++ .../tpch_plan_stability/approved/q20.txt | 69 ++++++ .../tpch_plan_stability/approved/q21.txt | 82 +++++++ .../tpch_plan_stability/approved/q22.txt | 35 +++ .../tests/tpch_plan_stability/approved/q3.txt | 40 +++ .../tests/tpch_plan_stability/approved/q4.txt | 31 +++ .../tests/tpch_plan_stability/approved/q5.txt | 89 +++++++ .../tests/tpch_plan_stability/approved/q6.txt | 6 + .../tests/tpch_plan_stability/approved/q7.txt | 89 +++++++ .../tests/tpch_plan_stability/approved/q8.txt | 118 +++++++++ .../tests/tpch_plan_stability/approved/q9.txt | 85 +++++++ .../tests/tpch_plan_stability/fixtures.rs | 229 ++++++++++++++++++ .../tests/tpch_plan_stability/main.rs | 112 +++++++++ .../tests/tpch_plan_stability/stats_table.rs | 152 ++++++++++++ dev/release/rat_exclude_files.txt | 1 + dev/update-tpch-plan-stability.sh | 25 ++ 30 files changed, 1828 insertions(+) create mode 100644 ballista/scheduler/tests/tpch_plan_stability/README.md create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt create mode 100644 ballista/scheduler/tests/tpch_plan_stability/fixtures.rs create mode 100644 ballista/scheduler/tests/tpch_plan_stability/main.rs create mode 100644 ballista/scheduler/tests/tpch_plan_stability/stats_table.rs create mode 100755 dev/update-tpch-plan-stability.sh diff --git a/Cargo.lock b/Cargo.lock index 50ee692b8e..4d0b4e1d48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,6 +1195,7 @@ dependencies = [ "prost", "prost-types", "rand 0.10.1", + "regex", "rstest", "serde", "tokio", diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index f882d26f38..92339a5c50 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -79,7 +79,12 @@ tracing-appender = { workspace = true, optional = true } tracing-subscriber = { workspace = true, optional = true } uuid = { workspace = true } +[[test]] +name = "tpch_plan_stability" +path = "tests/tpch_plan_stability/main.rs" + [dev-dependencies] +regex = "1" rstest = { workspace = true } [build-dependencies] diff --git a/ballista/scheduler/tests/tpch_plan_stability/README.md b/ballista/scheduler/tests/tpch_plan_stability/README.md new file mode 100644 index 0000000000..5b62c7f9af --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/README.md @@ -0,0 +1,59 @@ + + +# TPC-H plan-stability suite + +Freezes each TPC-H query's **distributed staged plan** (static planner, SF100 +table statistics, `target_partitions=16`) as an approved text file under +`approved/`. The suite fails if a code change alters plan shape — join strategy, +shuffle/stage boundaries, or broadcast decisions. + +- Run: `cargo test -p ballista-scheduler --test tpch_plan_stability` +- Regenerate after an intended change: `dev/update-tpch-plan-stability.sh` + (or `BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability`), + then review the diff under `approved/`. + +Scope: TPC-H only, static planner, Ballista default config (SortMergeJoin). +Tables are dataless providers with injected SF100 cardinalities (`fixtures.rs`). + +Query SQL is read directly from the canonical `benchmarks/queries/` at test time +(not copied), so a change to a benchmark query flows into the planned plan and +surfaces as a golden diff, prompting a deliberate regeneration. + +## CI coverage + +This suite is registered as a `[[test]]` target in `ballista/scheduler/Cargo.toml` +(`tpch_plan_stability`), so it already runs wherever CI exercises the workspace's +default cargo tests — no dedicated job was added: + +- `.github/workflows/rust.yml` → `linux-test` (`cargo test --profile ci --features=testcontainers`) and `macos-test` (`cargo test --profile ci --locked`) both run from the workspace root without `-p`/`--workspace` + scoping. Since the root `Cargo.toml` sets no `default-members`, this tests + every workspace member, including `ballista-scheduler`, which picks up this + target automatically. +- `.github/workflows/rust.yml` → `clippy` already runs `cargo clippy --all-targets --package ballista-scheduler --all-features -- -D warnings`, + which lints this test target too. +- `.github/workflows/rust.yml` → `lint` runs `cargo fmt --all -- --check`, + which covers these files as well. + +The generated `approved/*.txt` golden plans carry no license header (the test +compares their exact bytes), so they are excluded from the Apache RAT license +check in `.github/workflows/dev.yml` via +`ballista/scheduler/tests/tpch_plan_stability/approved/*` in +`dev/release/rat_exclude_files.txt` — mirroring the existing +`ballista/scheduler/testdata/*` exclusion. diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt new file mode 100644 index 0000000000..3f26e2a588 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt @@ -0,0 +1,18 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 16) + AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + ProjectionExec: expr=[l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] + FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_extendedprice@1, l_discount@2, l_quantity@0, l_tax@3, l_returnflag@4, l_linestatus@5] + StatsExec: rows=600037902 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] + AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) + +=== Stage 3 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt new file mode 100644 index 0000000000..ef52dce9cb --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt @@ -0,0 +1,60 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + StatsExec: rows=15000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16) + FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1] + StatsExec: rows=150000000 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@7], 16) + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_nationkey@3 as c_nationkey, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, o_orderkey@7 as o_orderkey] + SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] + StatsExec: rows=600037902 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([c_nationkey@3], 16) + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_nationkey@3 as c_nationkey, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@9 as l_extendedprice, l_discount@10 as l_discount] + SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)] + SortExec: expr=[o_orderkey@7 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@7], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) + AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@10 as n_name] + ProjectionExec: expr=[c_custkey@2 as c_custkey, c_name@3 as c_name, c_address@4 as c_address, c_nationkey@5 as c_nationkey, c_phone@6 as c_phone, c_acctbal@7 as c_acctbal, c_comment@8 as c_comment, l_extendedprice@9 as l_extendedprice, l_discount@10 as l_discount, n_nationkey@0 as n_nationkey, n_name@1 as n_name] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([c_nationkey@3], 16) + +=== Stage 9 === +ShuffleWriterExec: partitioning: None + SortExec: TopK(fetch=20), expr=[revenue@2 DESC], preserve_partitioning=[true] + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] + AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) + +=== Stage 10 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [revenue@2 DESC], fetch=20 + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@3, c_phone@6, n_name@4, c_address@5, c_comment@7], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt new file mode 100644 index 0000000000..059c25242a --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt @@ -0,0 +1,87 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] + StatsExec: rows=25 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) + StatsExec: rows=80000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) + ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@4 as s_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)] + SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@0], 16) + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 6 === +ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + ProjectionExec: expr=[ps_availqty@0 as ps_availqty, ps_supplycost@1 as ps_supplycost] + ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@3 as s_nationkey, n_nationkey@0 as n_nationkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] + StatsExec: rows=25 + +=== Stage 8 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1], 16) + StatsExec: rows=80000000 + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 11 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16) + ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@5 as s_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)] + SortExec: expr=[ps_suppkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@1], 16) + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 12 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) + AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost] + ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@4 as s_nationkey, n_nationkey@0 as n_nationkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@3], 16) + +=== Stage 13 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[value@1 DESC], preserve_partitioning=[true] + ProjectionExec: expr=[ps_partkey@1 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2 as value] + NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2] + ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)] + AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + CoalescePartitionsExec + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1] + AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + +=== Stage 14 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [value@1 DESC] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt new file mode 100644 index 0000000000..9802dd3a7b --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt @@ -0,0 +1,30 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4] + StatsExec: rows=600037902 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + StatsExec: rows=150000000 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([l_shipmode@0], 16) + AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] + ProjectionExec: expr=[l_shipmode@1 as l_shipmode, o_orderpriority@3 as o_orderpriority] + SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)] + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] + AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] + UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 16) + +=== Stage 5 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt new file mode 100644 index 0000000000..252b08f089 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt @@ -0,0 +1,32 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + StatsExec: rows=15000000 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16) + FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1] + StatsExec: rows=150000000 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([c_count@0], 16) + AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] + ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count] + AggregateExec: mode=SinglePartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], ordering_mode=Sorted + ProjectionExec: expr=[c_custkey@0 as c_custkey, o_orderkey@1 as o_orderkey] + SortMergeJoinExec: join_type=Left, on=[(c_custkey@0, o_custkey@1)] + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] + ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] + UnresolvedShuffleExec: partitioning: Hash([c_count@0], 16) + +=== Stage 5 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC] + UnresolvedShuffleExec: partitioning: Hash([c_count@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt new file mode 100644 index 0000000000..172add80cf --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt @@ -0,0 +1,25 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16) + FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2] + StatsExec: rows=600037902 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + StatsExec: rows=20000000 + +=== Stage 3 === +ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, p_type@4 as p_type] + SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)] + SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] + AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + CoalescePartitionsExec + UnresolvedShuffleExec: partitioning: Hash([p_partkey@3], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt new file mode 100644 index 0000000000..2a357f08de --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt @@ -0,0 +1,57 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16) + AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] + StatsExec: rows=600037902 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)] + ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] + AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([max(revenue0.total_revenue)@0], 16) + AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)] + CoalescePartitionsExec + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@0], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([max(revenue0.total_revenue)@0], 16) + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16) + AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] + StatsExec: rows=600037902 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([total_revenue@4], 16) + ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address, s_phone@3 as s_phone, total_revenue@5 as total_revenue] + SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, supplier_no@0)] + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + SortExec: expr=[supplier_no@0 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] + AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@0], 16) + +=== Stage 8 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address, s_phone@3 as s_phone, total_revenue@4 as total_revenue] + ProjectionExec: expr=[s_suppkey@1 as s_suppkey, s_name@2 as s_name, s_address@3 as s_address, s_phone@4 as s_phone, total_revenue@5 as total_revenue, max(revenue0.total_revenue)@0 as max(revenue0.total_revenue)] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([total_revenue@4], 16) + +=== Stage 9 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([total_revenue@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt new file mode 100644 index 0000000000..4403c35874 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt @@ -0,0 +1,46 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0] + StatsExec: rows=1000000 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) + StatsExec: rows=80000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) + StatsExec: rows=20000000 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) + ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey, p_brand@3 as p_brand, p_type@4 as p_type, p_size@5 as p_size] + SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, p_partkey@0)] + SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 16) + AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] + AggregateExec: mode=SinglePartitioned, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] + HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@0], 16) + +=== Stage 7 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] + UnresolvedShuffleExec: partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) + +=== Stage 8 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt new file mode 100644 index 0000000000..e3b6744bb2 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt @@ -0,0 +1,36 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16) + StatsExec: rows=600037902 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0] + StatsExec: rows=20000000 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16) + AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] + StatsExec: rows=600037902 + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)] + ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice] + SortMergeJoinExec: join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1 + ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@3 as p_partkey] + SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)] + SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] + AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + +=== Stage 5 === +ShuffleWriterExec: partitioning: None + ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] + AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)] + CoalescePartitionsExec + UnresolvedShuffleExec: partitioning: Hash([l_partkey@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt new file mode 100644 index 0000000000..190add99b8 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt @@ -0,0 +1,46 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + StatsExec: rows=15000000 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16) + StatsExec: rows=150000000 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@2], 16) + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, o_orderkey@2 as o_orderkey, o_totalprice@4 as o_totalprice, o_orderdate@5 as o_orderdate] + SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + StatsExec: rows=600037902 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] + StatsExec: rows=600037902 + +=== Stage 6 === +ShuffleWriterExec: partitioning: None + SortExec: TopK(fetch=100), expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=SinglePartitioned, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)], ordering_mode=PartiallySorted([2]) + SortMergeJoinExec: join_type=LeftSemi, on=[(o_orderkey@2, l_orderkey@0)] + ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, o_orderkey@2 as o_orderkey, o_totalprice@3 as o_totalprice, o_orderdate@4 as o_orderdate, l_quantity@6 as l_quantity] + SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)] + SortExec: expr=[o_orderkey@2 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@2], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] + AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + +=== Stage 7 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], fetch=100 + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt new file mode 100644 index 0000000000..a3960ed6eb --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt @@ -0,0 +1,26 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16) + FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] + StatsExec: rows=600037902 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_size@2 >= 1 AND (p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) + StatsExec: rows=20000000 + +=== Stage 3 === +ShuffleWriterExec: partitioning: None + AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount] + SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15 + SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] + AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + CoalescePartitionsExec + UnresolvedShuffleExec: partitioning: Hash([p_partkey@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt new file mode 100644 index 0000000000..5d3dba4594 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt @@ -0,0 +1,137 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) + FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] + StatsExec: rows=5 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1] + StatsExec: rows=20000000 + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) + StatsExec: rows=80000000 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@2], 16) + ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, ps_suppkey@3 as ps_suppkey, ps_supplycost@4 as ps_supplycost] + SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, ps_partkey@0)] + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@4], 16) + ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@5 as s_name, s_address@6 as s_address, s_nationkey@7 as s_nationkey, s_phone@8 as s_phone, s_acctbal@9 as s_acctbal, s_comment@10 as s_comment, ps_supplycost@3 as ps_supplycost] + SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@2, s_suppkey@0)] + SortExec: expr=[ps_suppkey@2 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@2], 16) + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([n_regionkey@9], 16) + ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@2 as s_name, s_address@3 as s_address, s_phone@5 as s_phone, s_acctbal@6 as s_acctbal, s_comment@7 as s_comment, ps_supplycost@8 as ps_supplycost, n_name@10 as n_name, n_regionkey@11 as n_regionkey] + ProjectionExec: expr=[p_partkey@3 as p_partkey, p_mfgr@4 as p_mfgr, s_name@5 as s_name, s_address@6 as s_address, s_nationkey@7 as s_nationkey, s_phone@8 as s_phone, s_acctbal@9 as s_acctbal, s_comment@10 as s_comment, ps_supplycost@11 as ps_supplycost, n_nationkey@0 as n_nationkey, n_name@1 as n_name, n_regionkey@2 as n_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@4], 16) + +=== Stage 11 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 16) + ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@2 as s_name, s_address@3 as s_address, s_phone@4 as s_phone, s_acctbal@5 as s_acctbal, s_comment@6 as s_comment, ps_supplycost@7 as ps_supplycost, n_name@8 as n_name] + ProjectionExec: expr=[p_partkey@1 as p_partkey, p_mfgr@2 as p_mfgr, s_name@3 as s_name, s_address@4 as s_address, s_phone@5 as s_phone, s_acctbal@6 as s_acctbal, s_comment@7 as s_comment, ps_supplycost@8 as ps_supplycost, n_name@9 as n_name, n_regionkey@10 as n_regionkey, r_regionkey@0 as r_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([n_regionkey@9], 16) + +=== Stage 12 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0, ps_supplycost@7], 16) + +=== Stage 13 === +SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) + FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] + StatsExec: rows=5 + +=== Stage 14 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + +=== Stage 15 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 16 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 17 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1], 16) + StatsExec: rows=80000000 + +=== Stage 18 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 19 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) + ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@4 as s_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)] + SortExec: expr=[ps_suppkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@1], 16) + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 20 === +SortShuffleWriterExec: partitioning=Hash([n_regionkey@2], 16) + ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@1 as ps_supplycost, n_regionkey@4 as n_regionkey] + ProjectionExec: expr=[ps_partkey@2 as ps_partkey, ps_supplycost@3 as ps_supplycost, s_nationkey@4 as s_nationkey, n_nationkey@0 as n_nationkey, n_regionkey@1 as n_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + +=== Stage 21 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) + AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] + ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@1 as ps_supplycost] + ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@3 as n_regionkey, r_regionkey@0 as r_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([n_regionkey@2], 16) + +=== Stage 22 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16) + ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey] + AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + +=== Stage 23 === +ShuffleWriterExec: partitioning: None + SortExec: TopK(fetch=100), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@8 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16) + +=== Stage 24 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=100 + UnresolvedShuffleExec: partitioning: Hash([p_partkey@3, min(partsupp.ps_supplycost)@9], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt new file mode 100644 index 0000000000..a684c1a4d2 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt @@ -0,0 +1,69 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0] + StatsExec: rows=25 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16) + StatsExec: rows=1000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address] + ProjectionExec: expr=[s_suppkey@1 as s_suppkey, s_name@2 as s_name, s_address@3 as s_address, s_nationkey@4 as s_nationkey, n_nationkey@0 as n_nationkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@3], 16) + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) + StatsExec: rows=80000000 + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0] + StatsExec: rows=20000000 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([ps_partkey@0, ps_suppkey@1], 16) + SortMergeJoinExec: join_type=LeftSemi, on=[(ps_partkey@0, p_partkey@0)] + SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 16) + AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] + FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] + StatsExec: rows=600037902 + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) + ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey] + SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1 + SortExec: expr=[ps_partkey@0 ASC, ps_suppkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0, ps_suppkey@1], 16) + SortExec: expr=[l_partkey@1 ASC, l_suppkey@2 ASC], preserve_partitioning=[true] + ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] + AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@0, l_suppkey@1], 16) + +=== Stage 10 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[s_name@1 as s_name, s_address@2 as s_address] + SortMergeJoinExec: join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)] + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@0], 16) + +=== Stage 11 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [s_name@0 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt new file mode 100644 index 0000000000..693152cd68 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt @@ -0,0 +1,82 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0] + StatsExec: rows=25 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16) + FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + StatsExec: rows=600037902 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@2], 16) + ProjectionExec: expr=[s_name@1 as s_name, s_nationkey@2 as s_nationkey, l_orderkey@3 as l_orderkey, l_suppkey@4 as l_suppkey] + SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)] + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1], 16) + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0] + StatsExec: rows=150000000 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@1], 16) + ProjectionExec: expr=[s_name@0 as s_name, s_nationkey@1 as s_nationkey, l_orderkey@2 as l_orderkey, l_suppkey@3 as l_suppkey] + SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@2, o_orderkey@0)] + SortExec: expr=[l_orderkey@2 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@2], 16) + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@1], 16) + ProjectionExec: expr=[s_name@0 as s_name, l_orderkey@2 as l_orderkey, l_suppkey@3 as l_suppkey] + ProjectionExec: expr=[s_name@1 as s_name, s_nationkey@2 as s_nationkey, l_orderkey@3 as l_orderkey, l_suppkey@4 as l_suppkey, n_nationkey@0 as n_nationkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@1], 16) + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + StatsExec: rows=600037902 + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + StatsExec: rows=600037902 + +=== Stage 11 === +SortShuffleWriterExec: partitioning=Hash([s_name@0], 16) + AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] + ProjectionExec: expr=[s_name@0 as s_name] + SortMergeJoinExec: join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 + SortMergeJoinExec: join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 + SortExec: expr=[l_orderkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@1], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + +=== Stage 12 === +ShuffleWriterExec: partitioning: None + SortExec: TopK(fetch=100), expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] + UnresolvedShuffleExec: partitioning: Hash([s_name@0], 16) + +=== Stage 13 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], fetch=100 + UnresolvedShuffleExec: partitioning: Hash([s_name@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt new file mode 100644 index 0000000000..4dd6095874 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt @@ -0,0 +1,35 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]) + StatsExec: rows=15000000 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@0], 16) + StatsExec: rows=150000000 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([cntrycode@0], 16) + AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] + ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal] + NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2] + AggregateExec: mode=Single, gby=[], aggr=[avg(customer.c_acctbal)] + FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] + StatsExec: rows=15000000 + ProjectionExec: expr=[c_phone@1 as c_phone, c_acctbal@2 as c_acctbal, CAST(c_acctbal@2 AS Decimal128(19, 6)) as join_proj_push_down_1] + SortMergeJoinExec: join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)] + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + SortExec: expr=[o_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@0], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] + UnresolvedShuffleExec: partitioning: Hash([cntrycode@0], 16) + +=== Stage 5 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([cntrycode@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt new file mode 100644 index 0000000000..a8d755ca9c --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt @@ -0,0 +1,40 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0] + StatsExec: rows=15000000 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16) + FilterExec: o_orderdate@2 < 1995-03-15 + StatsExec: rows=150000000 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + ProjectionExec: expr=[o_orderkey@1 as o_orderkey, o_orderdate@3 as o_orderdate, o_shippriority@4 as o_shippriority] + SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] + StatsExec: rows=600037902 + +=== Stage 5 === +ShuffleWriterExec: partitioning: None + SortExec: TopK(fetch=10), expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] + AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], ordering_mode=PartiallySorted([0]) + ProjectionExec: expr=[o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority, l_orderkey@3 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] + SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)] + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + +=== Stage 6 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], fetch=10 + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt new file mode 100644 index 0000000000..bbb97d34bd --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt @@ -0,0 +1,31 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2] + StatsExec: rows=150000000 + +=== Stage 2 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0] + StatsExec: rows=600037902 + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([o_orderpriority@0], 16) + AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] + ProjectionExec: expr=[o_orderpriority@1 as o_orderpriority] + SortMergeJoinExec: join_type=LeftSemi, on=[(o_orderkey@0, l_orderkey@0)] + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] + UnresolvedShuffleExec: partitioning: Hash([o_orderpriority@0], 16) + +=== Stage 5 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([o_orderpriority@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt new file mode 100644 index 0000000000..c159882e44 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt @@ -0,0 +1,89 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) + FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0] + StatsExec: rows=5 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + StatsExec: rows=15000000 + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16) + FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1] + StatsExec: rows=150000000 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@1], 16) + ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@2 as o_orderkey] + SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + StatsExec: rows=600037902 + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@1, c_nationkey@0], 16) + ProjectionExec: expr=[c_nationkey@0 as c_nationkey, l_suppkey@3 as l_suppkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] + SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)] + SortExec: expr=[o_orderkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@1], 16) + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0, s_nationkey@1], 16) + StatsExec: rows=1000000 + +=== Stage 11 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) + ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@5 as s_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@1, s_suppkey@0), (c_nationkey@0, s_nationkey@1)] + SortExec: expr=[l_suppkey@1 ASC, c_nationkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1, c_nationkey@0], 16) + SortExec: expr=[s_suppkey@0 ASC, s_nationkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0, s_nationkey@1], 16) + +=== Stage 12 === +SortShuffleWriterExec: partitioning=Hash([n_regionkey@3], 16) + ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@4 as n_name, n_regionkey@5 as n_regionkey] + ProjectionExec: expr=[l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, s_nationkey@5 as s_nationkey, n_nationkey@0 as n_nationkey, n_name@1 as n_name, n_regionkey@2 as n_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + +=== Stage 13 === +SortShuffleWriterExec: partitioning=Hash([n_name@0], 16) + AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@2 as n_name] + ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, n_name@3 as n_name, n_regionkey@4 as n_regionkey, r_regionkey@0 as r_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([n_regionkey@3], 16) + +=== Stage 14 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] + ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] + AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + UnresolvedShuffleExec: partitioning: Hash([n_name@0], 16) + +=== Stage 15 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [revenue@1 DESC] + UnresolvedShuffleExec: partitioning: Hash([n_name@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt new file mode 100644 index 0000000000..1da12a0cb1 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt @@ -0,0 +1,6 @@ +=== Stage 1 === +ShuffleWriterExec: partitioning: None + ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue] + AggregateExec: mode=Single, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] + FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2] + StatsExec: rows=600037902 diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt new file mode 100644 index 0000000000..77cbc87a8d --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt @@ -0,0 +1,89 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY + StatsExec: rows=25 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16) + FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 + StatsExec: rows=600037902 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@1], 16) + ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_orderkey@2 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, l_shipdate@6 as l_shipdate] + SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)] + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1], 16) + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + StatsExec: rows=150000000 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@4], 16) + ProjectionExec: expr=[s_nationkey@0 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, o_custkey@6 as o_custkey] + SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)] + SortExec: expr=[l_orderkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@1], 16) + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + StatsExec: rows=15000000 + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@0], 16) + ProjectionExec: expr=[s_nationkey@0 as s_nationkey, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@6 as c_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(o_custkey@4, c_custkey@0)] + SortExec: expr=[o_custkey@4 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@4], 16) + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([c_nationkey@3], 16) + ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@6 as n_name] + ProjectionExec: expr=[s_nationkey@2 as s_nationkey, l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, l_shipdate@5 as l_shipdate, c_nationkey@6 as c_nationkey, n_nationkey@0 as n_nationkey, n_name@1 as n_name] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@0], 16) + +=== Stage 11 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([c_nationkey@3], 16) + +=== Stage 12 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE + StatsExec: rows=25 + +=== Stage 13 === +SortShuffleWriterExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 16) + AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] + ProjectionExec: expr=[n_name@4 as supp_nation, n_name@6 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 14 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] + UnresolvedShuffleExec: partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) + +=== Stage 15 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt new file mode 100644 index 0000000000..d70be863e9 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt @@ -0,0 +1,118 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) + FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0] + StatsExec: rows=5 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 4 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0] + StatsExec: rows=20000000 + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@1], 16) + StatsExec: rows=600037902 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16) + ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_suppkey@3 as l_suppkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] + SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, l_partkey@1)] + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@1], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@5 as s_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@1, s_suppkey@0)] + SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1], 16) + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31 + StatsExec: rows=150000000 + +=== Stage 11 === +SortShuffleWriterExec: partitioning=Hash([o_custkey@3], 16) + ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_custkey@5 as o_custkey, o_orderdate@6 as o_orderdate] + SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)] + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + +=== Stage 12 === +SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) + StatsExec: rows=15000000 + +=== Stage 13 === +SortShuffleWriterExec: partitioning=Hash([c_nationkey@4], 16) + ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, s_nationkey@2 as s_nationkey, o_orderdate@4 as o_orderdate, c_nationkey@6 as c_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(o_custkey@3, c_custkey@0)] + SortExec: expr=[o_custkey@3 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_custkey@3], 16) + SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + +=== Stage 14 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) + ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, s_nationkey@2 as s_nationkey, o_orderdate@3 as o_orderdate, n_regionkey@6 as n_regionkey] + ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_orderdate@5 as o_orderdate, c_nationkey@6 as c_nationkey, n_nationkey@0 as n_nationkey, n_regionkey@1 as n_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([c_nationkey@4], 16) + +=== Stage 15 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + +=== Stage 16 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 17 === +SortShuffleWriterExec: partitioning=Hash([n_regionkey@3], 16) + ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@6 as n_name] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 18 === +SortShuffleWriterExec: partitioning=Hash([o_year@0], 16) + AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@4 as nation] + ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@5 as n_name, r_regionkey@0 as r_regionkey] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([n_regionkey@3], 16) + +=== Stage 19 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] + AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + UnresolvedShuffleExec: partitioning: Hash([o_year@0], 16) + +=== Stage 20 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [o_year@0 ASC NULLS LAST] + UnresolvedShuffleExec: partitioning: Hash([o_year@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt new file mode 100644 index 0000000000..3d91e1dd21 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt @@ -0,0 +1,85 @@ +=== Stage 1 === +SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) + StatsExec: rows=25 + +=== Stage 2 === +ShuffleWriterExec: partitioning: None + UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + +=== Stage 3 === +SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) + FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0] + StatsExec: rows=20000000 + +=== Stage 4 === +SortShuffleWriterExec: partitioning=Hash([l_partkey@1], 16) + StatsExec: rows=600037902 + +=== Stage 5 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@2], 16) + ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount] + SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, l_partkey@1)] + SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_partkey@1], 16) + +=== Stage 6 === +SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) + StatsExec: rows=1000000 + +=== Stage 7 === +SortShuffleWriterExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 16) + ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_partkey@1 as l_partkey, l_suppkey@2 as l_suppkey, l_quantity@3 as l_quantity, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, s_nationkey@7 as s_nationkey] + SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@2, s_suppkey@0)] + SortExec: expr=[l_suppkey@2 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@2], 16) + SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + +=== Stage 8 === +SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 16) + StatsExec: rows=80000000 + +=== Stage 9 === +SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) + ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_quantity@3 as l_quantity, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, s_nationkey@6 as s_nationkey, ps_supplycost@9 as ps_supplycost] + SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)] + SortExec: expr=[l_suppkey@2 ASC, l_partkey@1 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_suppkey@2, l_partkey@1], 16) + SortExec: expr=[ps_suppkey@1 ASC, ps_partkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@1, ps_partkey@0], 16) + +=== Stage 10 === +SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) + StatsExec: rows=150000000 + +=== Stage 11 === +SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16) + ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@7 as o_orderdate] + SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)] + SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] + UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + +=== Stage 12 === +SortShuffleWriterExec: partitioning=Hash([nation@0, o_year@1], 16) + AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] + ProjectionExec: expr=[n_name@7 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) - ps_supplycost@4 * l_quantity@0 as amount] + ProjectionExec: expr=[l_quantity@2 as l_quantity, l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, s_nationkey@5 as s_nationkey, ps_supplycost@6 as ps_supplycost, o_orderdate@7 as o_orderdate, n_nationkey@0 as n_nationkey, n_name@1 as n_name] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)] + UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: partitioning: Hash([s_nationkey@3], 16) + +=== Stage 13 === +ShuffleWriterExec: partitioning: None + SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] + ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] + UnresolvedShuffleExec: partitioning: Hash([nation@0, o_year@1], 16) + +=== Stage 14 === +ShuffleWriterExec: partitioning: None + SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] + UnresolvedShuffleExec: partitioning: Hash([nation@0, o_year@1], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/fixtures.rs b/ballista/scheduler/tests/tpch_plan_stability/fixtures.rs new file mode 100644 index 0000000000..811370875e --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/fixtures.rs @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! TPC-H schema/fixture helpers and the distributed staged-plan text helper. +//! +//! `staged_plan_text` loads a TPC-H query's SQL, registers the 8 TPC-H tables +//! (via [`TpchStatsTable`]) with SF100 row-count statistics, builds the +//! physical plan with the static (Ballista) planner, breaks it into +//! distributed query stages with [`DefaultDistributedPlanner`], and renders +//! the stages to a normalized text representation suitable for snapshotting. + +use std::sync::Arc; + +use ballista_core::JobId; +use ballista_core::extension::SessionConfigExt; +use ballista_scheduler::planner::{DefaultDistributedPlanner, DistributedPlanner}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::prelude::{SessionConfig, SessionContext}; + +use crate::stats_table::TpchStatsTable; + +const TARGET_PARTITIONS: usize = 16; +const JOB_ID: &str = "plan_stability"; + +/// The 8 TPC-H table names and their SF100 row counts. +pub const SF100_ROWS: &[(&str, usize)] = &[ + ("region", 5), + ("nation", 25), + ("supplier", 1_000_000), + ("customer", 15_000_000), + ("part", 20_000_000), + ("partsupp", 80_000_000), + ("orders", 150_000_000), + ("lineitem", 600_037_902), +]; + +/// Returns the schema for a TPC-H table, copied verbatim from +/// `benchmarks/src/bin/tpch.rs::get_schema` (that function lives in a binary +/// and cannot be imported directly). +pub fn tpch_schema(table: &str) -> Schema { + match table { + "part" => Schema::new(vec![ + Field::new("p_partkey", DataType::Int64, false), + Field::new("p_name", DataType::Utf8, false), + Field::new("p_mfgr", DataType::Utf8, false), + Field::new("p_brand", DataType::Utf8, false), + Field::new("p_type", DataType::Utf8, false), + Field::new("p_size", DataType::Int32, false), + Field::new("p_container", DataType::Utf8, false), + Field::new("p_retailprice", DataType::Decimal128(15, 2), false), + Field::new("p_comment", DataType::Utf8, false), + ]), + + "supplier" => Schema::new(vec![ + Field::new("s_suppkey", DataType::Int64, false), + Field::new("s_name", DataType::Utf8, false), + Field::new("s_address", DataType::Utf8, false), + Field::new("s_nationkey", DataType::Int64, false), + Field::new("s_phone", DataType::Utf8, false), + Field::new("s_acctbal", DataType::Decimal128(15, 2), false), + Field::new("s_comment", DataType::Utf8, false), + ]), + + "partsupp" => Schema::new(vec![ + Field::new("ps_partkey", DataType::Int64, false), + Field::new("ps_suppkey", DataType::Int64, false), + Field::new("ps_availqty", DataType::Int32, false), + Field::new("ps_supplycost", DataType::Decimal128(15, 2), false), + Field::new("ps_comment", DataType::Utf8, false), + ]), + + "customer" => Schema::new(vec![ + Field::new("c_custkey", DataType::Int64, false), + Field::new("c_name", DataType::Utf8, false), + Field::new("c_address", DataType::Utf8, false), + Field::new("c_nationkey", DataType::Int64, false), + Field::new("c_phone", DataType::Utf8, false), + Field::new("c_acctbal", DataType::Decimal128(15, 2), false), + Field::new("c_mktsegment", DataType::Utf8, false), + Field::new("c_comment", DataType::Utf8, false), + ]), + + "orders" => Schema::new(vec![ + Field::new("o_orderkey", DataType::Int64, false), + Field::new("o_custkey", DataType::Int64, false), + Field::new("o_orderstatus", DataType::Utf8, false), + Field::new("o_totalprice", DataType::Decimal128(15, 2), false), + Field::new("o_orderdate", DataType::Date32, false), + Field::new("o_orderpriority", DataType::Utf8, false), + Field::new("o_clerk", DataType::Utf8, false), + Field::new("o_shippriority", DataType::Int32, false), + Field::new("o_comment", DataType::Utf8, false), + ]), + + "lineitem" => Schema::new(vec![ + Field::new("l_orderkey", DataType::Int64, false), + Field::new("l_partkey", DataType::Int64, false), + Field::new("l_suppkey", DataType::Int64, false), + Field::new("l_linenumber", DataType::Int32, false), + Field::new("l_quantity", DataType::Decimal128(15, 2), false), + Field::new("l_extendedprice", DataType::Decimal128(15, 2), false), + Field::new("l_discount", DataType::Decimal128(15, 2), false), + Field::new("l_tax", DataType::Decimal128(15, 2), false), + Field::new("l_returnflag", DataType::Utf8, false), + Field::new("l_linestatus", DataType::Utf8, false), + Field::new("l_shipdate", DataType::Date32, false), + Field::new("l_commitdate", DataType::Date32, false), + Field::new("l_receiptdate", DataType::Date32, false), + Field::new("l_shipinstruct", DataType::Utf8, false), + Field::new("l_shipmode", DataType::Utf8, false), + Field::new("l_comment", DataType::Utf8, false), + ]), + + "nation" => Schema::new(vec![ + Field::new("n_nationkey", DataType::Int64, false), + Field::new("n_name", DataType::Utf8, false), + Field::new("n_regionkey", DataType::Int64, false), + Field::new("n_comment", DataType::Utf8, false), + ]), + + "region" => Schema::new(vec![ + Field::new("r_regionkey", DataType::Int64, false), + Field::new("r_name", DataType::Utf8, false), + Field::new("r_comment", DataType::Utf8, false), + ]), + + other => panic!("unknown tpch table {other}"), + } +} + +fn make_ctx() -> SessionContext { + let config = + SessionConfig::new_with_ballista().with_target_partitions(TARGET_PARTITIONS); + let ctx = SessionContext::new_with_config(config); + for (table, rows) in SF100_ROWS { + let schema = Arc::new(tpch_schema(table)); + ctx.register_table(*table, Arc::new(TpchStatsTable::new(schema, *rows))) + .unwrap(); + } + ctx +} + +fn is_query_stmt(stmt: &str) -> bool { + let u = stmt.trim_start().to_uppercase(); + u.starts_with("SELECT") || u.starts_with("WITH") +} + +/// Produce the normalized distributed staged-plan text for a TPC-H query. +pub async fn staged_plan_text(query_name: &str) -> String { + // Read the query SQL directly from the canonical benchmark location rather + // than a copy, so a change to a benchmark query surfaces as a golden diff. + let sql_path = format!( + "{}/../../benchmarks/queries/{query_name}.sql", + env!("CARGO_MANIFEST_DIR") + ); + let sql = std::fs::read_to_string(&sql_path) + .unwrap_or_else(|e| panic!("read {sql_path}: {e}")); + + let ctx = make_ctx(); + + // Split into statements; execute DDL, capture the physical plan of the answer + // (last SELECT/WITH) statement. Single-statement queries take the one statement. + let stmts: Vec<&str> = sql + .split(';') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + let answer_idx = stmts + .iter() + .rposition(|s| is_query_stmt(s)) + .expect("no SELECT/WITH statement in query"); + + let mut physical = None; + for (i, stmt) in stmts.iter().enumerate() { + if i == answer_idx { + physical = Some( + ctx.sql(stmt) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(), + ); + } else { + // DDL such as CREATE VIEW / DROP VIEW (q15) — apply it. + ctx.sql(stmt).await.unwrap().collect().await.unwrap(); + } + } + let physical = physical.unwrap(); + + let mut planner = DefaultDistributedPlanner::new(); + let state = ctx.state(); + let job_id: JobId = JOB_ID.into(); + let stages = planner + .plan_query_stages(&job_id, physical, state.config().options()) + .unwrap(); + + let mut out = String::new(); + for stage in &stages { + out.push_str(&format!("=== Stage {} ===\n", stage.stage_id())); + let ep: &dyn datafusion::physical_plan::ExecutionPlan = stage.as_ref(); + out.push_str(&DisplayableExecutionPlan::new(ep).indent(false).to_string()); + out.push('\n'); + } + normalize(&out) +} + +fn normalize(plan: &str) -> String { + // Strip the fixed job id and any hex addresses so output is byte-stable. + let s = plan.replace(JOB_ID, ""); + // remove 0x… addresses if any appear + let re = regex::Regex::new(r"0x[0-9a-fA-F]+").unwrap(); + re.replace_all(&s, "0x").into_owned() +} diff --git a/ballista/scheduler/tests/tpch_plan_stability/main.rs b/ballista/scheduler/tests/tpch_plan_stability/main.rs new file mode 100644 index 0000000000..38d6d5f2a5 --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/main.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod fixtures; +mod stats_table; + +use std::path::PathBuf; +use std::sync::Arc; + +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::common::stats::Precision; +use datafusion::prelude::SessionContext; + +use stats_table::TpchStatsTable; + +#[tokio::test] +async fn stats_table_reports_injected_rows() { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + let ctx = SessionContext::new(); + ctx.register_table( + "t", + Arc::new(TpchStatsTable::new(Arc::clone(&schema), 12_345)), + ) + .unwrap(); + + let plan = ctx + .sql("SELECT a FROM t") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + + let stats = plan.partition_statistics(None).unwrap(); + assert_eq!(stats.num_rows, Precision::Inexact(12_345)); +} + +#[tokio::test] +async fn staged_plan_text_is_nonempty_and_shuffled() { + let text = fixtures::staged_plan_text("q1").await; + assert!( + text.contains("ShuffleWriterExec"), + "q1 plan should contain shuffle stages:\n{text}" + ); + assert!( + !text.contains("plan_stability"), + "job id should be normalized out" + ); + assert!(text.contains("=== Stage"), "stage banners present"); +} + +#[tokio::test] +async fn multi_statement_q15_plans() { + let text = fixtures::staged_plan_text("q15").await; + assert!( + text.contains("=== Stage"), + "q15 (create/select/drop view) should plan:\n{text}" + ); +} + +fn golden_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/tpch_plan_stability/approved") + .join(format!("{name}.txt")) +} + +async fn check_query(name: &str) { + let actual = fixtures::staged_plan_text(name).await; + let path = golden_path(name); + if std::env::var("BALLISTA_GENERATE_GOLDEN").is_ok() { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, actual.trim_end().to_string() + "\n").unwrap(); + return; + } + let expected = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!("missing golden {path:?}; regenerate with BALLISTA_GENERATE_GOLDEN=1") + }); + assert_eq!( + actual.trim_end(), + expected.trim_end(), + "distributed plan drift for {name}. Review the change; if intended, regenerate with \ + BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability" + ); +} + +macro_rules! plan_stability_test { + ($($name:ident),+ $(,)?) => { + $( + #[tokio::test] + async fn $name() { check_query(stringify!($name)).await; } + )+ + }; +} + +plan_stability_test!( + q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, q15, q16, q17, q18, q19, + q20, q21, q22 +); diff --git a/ballista/scheduler/tests/tpch_plan_stability/stats_table.rs b/ballista/scheduler/tests/tpch_plan_stability/stats_table.rs new file mode 100644 index 0000000000..2d8d7b1b5d --- /dev/null +++ b/ballista/scheduler/tests/tpch_plan_stability/stats_table.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::{Session, TableProvider}; +use datafusion::common::stats::Precision; +use datafusion::common::{Result, Statistics}; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::{Expr, TableType}; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::execution_plan::{ + Boundedness, EmissionType, SchedulingType, +}; +use datafusion::physical_plan::memory::MemoryStream; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + SendableRecordBatchStream, +}; + +use async_trait::async_trait; + +/// A leaf execution plan that produces no rows but reports a fixed row-count +/// statistic, so the physical optimizer makes realistic join/broadcast choices +/// without any data on disk. +#[derive(Debug)] +pub struct StatsExec { + schema: SchemaRef, + num_rows: usize, + cache: Arc, +} + +impl StatsExec { + pub fn new(schema: SchemaRef, num_rows: usize) -> Self { + let cache = PlanProperties::new( + EquivalenceProperties::new(Arc::clone(&schema)), + Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + ) + .with_scheduling_type(SchedulingType::Cooperative); + Self { + schema, + num_rows, + cache: Arc::new(cache), + } + } +} + +impl DisplayAs for StatsExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "StatsExec: rows={}", self.num_rows) + } + DisplayFormatType::TreeRender => write!(f, ""), + } + } +} + +impl ExecutionPlan for StatsExec { + fn name(&self) -> &'static str { + "StatsExec" + } + fn properties(&self) -> &Arc { + &self.cache + } + fn children(&self) -> Vec<&Arc> { + vec![] + } + fn with_new_children( + self: Arc, + _: Vec>, + ) -> Result> { + Ok(self) + } + fn execute( + &self, + _partition: usize, + _context: Arc, + ) -> Result { + Ok(Box::pin(MemoryStream::try_new( + vec![], + Arc::clone(&self.schema), + None, + )?)) + } + fn partition_statistics(&self, _partition: Option) -> Result> { + Ok(Arc::new( + Statistics::new_unknown(&self.schema) + .with_num_rows(Precision::Inexact(self.num_rows)), + )) + } +} + +/// A dataless `TableProvider` with a real schema and a fixed row-count statistic. +#[derive(Debug)] +pub struct TpchStatsTable { + schema: SchemaRef, + num_rows: usize, +} + +impl TpchStatsTable { + pub fn new(schema: SchemaRef, num_rows: usize) -> Self { + Self { schema, num_rows } + } +} + +#[async_trait] +impl TableProvider for TpchStatsTable { + fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } + fn table_type(&self) -> TableType { + TableType::Base + } + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + let schema = match projection { + Some(p) => Arc::new(self.schema.project(p)?), + None => Arc::clone(&self.schema), + }; + Ok(Arc::new(StatsExec::new(schema, self.num_rows))) + } + fn statistics(&self) -> Option { + Some( + Statistics::new_unknown(&self.schema) + .with_num_rows(Precision::Inexact(self.num_rows)), + ) + } +} diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 47a5ee2eb8..83d47821cc 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -34,6 +34,7 @@ requirements.txt .gitattributes benchmarks/queries/q*.sql ballista/scheduler/testdata/* +ballista/scheduler/tests/tpch_plan_stability/approved/* **/yarn.lock python/requirements*.txt **/testdata/* diff --git a/dev/update-tpch-plan-stability.sh b/dev/update-tpch-plan-stability.sh new file mode 100755 index 0000000000..072e2a1b61 --- /dev/null +++ b/dev/update-tpch-plan-stability.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Regenerate the approved TPC-H distributed plans for the plan-stability suite. +# Run after an intended planner/plan-shape change, then review the diff. +set -euo pipefail +cd "$(dirname "$0")/.." +BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability +echo "Regenerated. Review changes under ballista/scheduler/tests/tpch_plan_stability/approved/" From 5298a9f1ff0e57a20b2b6f23695e930b41167de8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 8 Jul 2026 22:19:12 -0600 Subject: [PATCH 034/107] feat: display input stage id on UnresolvedShuffleExec (#1967) (#1969) --- ballista/client/tests/context_checks.rs | 2 +- .../src/execution_plans/unresolved_shuffle.rs | 22 +++++++-- ballista/scheduler/src/planner.rs | 20 ++++---- .../tests/tpch_plan_stability/approved/q1.txt | 4 +- .../tpch_plan_stability/approved/q10.txt | 18 ++++---- .../tpch_plan_stability/approved/q11.txt | 26 +++++------ .../tpch_plan_stability/approved/q12.txt | 8 ++-- .../tpch_plan_stability/approved/q13.txt | 8 ++-- .../tpch_plan_stability/approved/q14.txt | 6 +-- .../tpch_plan_stability/approved/q15.txt | 16 +++---- .../tpch_plan_stability/approved/q16.txt | 14 +++--- .../tpch_plan_stability/approved/q17.txt | 8 ++-- .../tpch_plan_stability/approved/q18.txt | 12 ++--- .../tpch_plan_stability/approved/q19.txt | 6 +-- .../tests/tpch_plan_stability/approved/q2.txt | 46 +++++++++---------- .../tpch_plan_stability/approved/q20.txt | 20 ++++---- .../tpch_plan_stability/approved/q21.txt | 24 +++++----- .../tpch_plan_stability/approved/q22.txt | 8 ++-- .../tests/tpch_plan_stability/approved/q3.txt | 10 ++-- .../tests/tpch_plan_stability/approved/q4.txt | 8 ++-- .../tests/tpch_plan_stability/approved/q5.txt | 28 +++++------ .../tests/tpch_plan_stability/approved/q7.txt | 28 +++++------ .../tests/tpch_plan_stability/approved/q8.txt | 38 +++++++-------- .../tests/tpch_plan_stability/approved/q9.txt | 26 +++++------ 24 files changed, 211 insertions(+), 195 deletions(-) diff --git a/ballista/client/tests/context_checks.rs b/ballista/client/tests/context_checks.rs index dc0576d9d6..272abc94bd 100644 --- a/ballista/client/tests/context_checks.rs +++ b/ballista/client/tests/context_checks.rs @@ -1120,7 +1120,7 @@ mod supported { "| | ShuffleWriterExec: partitioning: None |", "| | ProjectionExec: expr=[count(Int64(1))@1 as count(*), id@0 as id] |", "| | AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[count(Int64(1))] |", - "| | UnresolvedShuffleExec: partitioning: Hash([id@0], 16) |", + "| | UnresolvedShuffleExec: stage=1, partitioning: Hash([id@0], 16) |", "| | |", "| | |", "+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", diff --git a/ballista/core/src/execution_plans/unresolved_shuffle.rs b/ballista/core/src/execution_plans/unresolved_shuffle.rs index 64bfd8fec9..f544f0b0a4 100644 --- a/ballista/core/src/execution_plans/unresolved_shuffle.rs +++ b/ballista/core/src/execution_plans/unresolved_shuffle.rs @@ -155,13 +155,14 @@ impl DisplayAs for UnresolvedShuffleExec { if self.broadcast { write!( f, - "UnresolvedShuffleExec: broadcast=true, upstream_partitions: {}", - self.upstream_partition_count, + "UnresolvedShuffleExec: stage={}, broadcast=true, upstream_partitions: {}", + self.stage_id, self.upstream_partition_count, ) } else { write!( f, - "UnresolvedShuffleExec: partitioning: {}", + "UnresolvedShuffleExec: stage={}, partitioning: {}", + self.stage_id, self.properties().output_partitioning() )?; if let Some(c) = &self.coalesce { @@ -266,4 +267,19 @@ mod tests { "expected ', coalesce: 3 of 8' annotation; got: {s}" ); } + + #[test] + fn display_includes_stage_id() { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let exec = + UnresolvedShuffleExec::new(3, schema, Partitioning::UnknownPartitioning(4)); + let s = format!( + "{}", + datafusion::physical_plan::displayable(&exec).indent(false) + ); + assert!( + s.contains("stage=3"), + "expected stage id in display, got: {s}" + ); + } } diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index d5602f57be..97cc1a7ac9 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -800,11 +800,11 @@ mod test { SortExec: expr=[l_returnflag@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[l_returnflag@0 as l_returnflag, sum(lineitem.l_extendedprice * Int64(1))@1 as sum_disc_price] AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag], aggr=[sum(lineitem.l_extendedprice * Int64(1))] - UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0], 2) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_returnflag@0], 2) ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0], 2) + UnresolvedShuffleExec: stage=2, partitioning: Hash([l_returnflag@0], 2) */ assert_eq!(3, stages.len()); @@ -918,18 +918,18 @@ order by ShuffleWriterExec: partitioning: Hash([l_shipmode@0], 2) AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_shipmode@1, o_orderpriority@3] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 2) - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 2) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_orderkey@0], 2) + UnresolvedShuffleExec: stage=2, partitioning: Hash([o_orderkey@0], 2) ShuffleWriterExec: partitioning: None SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] - UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 2) + UnresolvedShuffleExec: stage=3, partitioning: Hash([l_shipmode@0], 2) ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 2) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_shipmode@0], 2) */ assert_eq!(5, stages.len()); @@ -1099,13 +1099,13 @@ order by // Stage 1 holds the join: a broadcast CollectLeft hash join, no // SortMergeJoinExec and no SortExec. - assert_plan!(stages[1].as_ref(), @r" + assert_plan!(stages[1].as_ref(), @" ShuffleWriterExec: partitioning: None AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] ProjectionExec: expr=[] ProjectionExec: expr=[k@1 as k, k@0 as k] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 1 + UnresolvedShuffleExec: stage=1, broadcast=true, upstream_partitions: 1 DataSourceExec: partitions=1, partition_sizes=[1] "); @@ -1647,12 +1647,12 @@ order by FilterExec: rank() PARTITION BY [lineitem.l_shipmode] ORDER BY [lineitem.l_shipdate DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 100 BoundedWindowAggExec: wdw=[rank() PARTITION BY [lineitem.l_shipmode] ORDER BY [lineitem.l_shipdate DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [lineitem.l_shipmode] ORDER BY [lineitem.l_shipdate DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] SortExec: expr=[l_shipmode@1 ASC NULLS LAST, l_shipdate@0 DESC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_shipmode@1], 2) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_shipmode@1], 2) Stage 2: ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [l_shipdate@1 ASC NULLS LAST, rk@2 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 2) + UnresolvedShuffleExec: stage=2, partitioning: Hash([l_shipmode@0], 2) */ diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt index 3f26e2a588..b01d7f2aff 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt @@ -10,9 +10,9 @@ ShuffleWriterExec: partitioning: None SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] - UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) === Stage 3 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([l_returnflag@0, l_linestatus@1], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt index ef52dce9cb..cf63a2940c 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt @@ -4,7 +4,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([n_nationkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) @@ -20,9 +20,9 @@ SortShuffleWriterExec: partitioning=Hash([o_orderkey@7], 16) ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_nationkey@3 as c_nationkey, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, o_orderkey@7 as o_orderkey] SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([c_custkey@0], 16) SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([o_custkey@1], 16) === Stage 6 === SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) @@ -34,9 +34,9 @@ SortShuffleWriterExec: partitioning=Hash([c_nationkey@3], 16) ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_nationkey@3 as c_nationkey, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@9 as l_extendedprice, l_discount@10 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)] SortExec: expr=[o_orderkey@7 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@7], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([o_orderkey@7], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([l_orderkey@0], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) @@ -44,17 +44,17 @@ SortShuffleWriterExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@10 as n_name] ProjectionExec: expr=[c_custkey@2 as c_custkey, c_name@3 as c_name, c_address@4 as c_address, c_nationkey@5 as c_nationkey, c_phone@6 as c_phone, c_acctbal@7 as c_acctbal, c_comment@8 as c_comment, l_extendedprice@9 as l_extendedprice, l_discount@10 as l_discount, n_nationkey@0 as n_nationkey, n_name@1 as n_name] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([c_nationkey@3], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=7, partitioning: Hash([c_nationkey@3], 16) === Stage 9 === ShuffleWriterExec: partitioning: None SortExec: TopK(fetch=20), expr=[revenue@2 DESC], preserve_partitioning=[true] ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16) === Stage 10 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [revenue@2 DESC], fetch=20 - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@3, c_phone@6, n_name@4, c_address@5, c_comment@7], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@3, c_phone@6, n_name@4, c_address@5, c_comment@7], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt index 059c25242a..9a68df95d8 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([n_nationkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) @@ -20,9 +20,9 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@4 as s_nationkey] SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)] SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([ps_suppkey@0], 16) SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([s_suppkey@0], 16) === Stage 6 === ShuffleWriterExec: partitioning: None @@ -30,8 +30,8 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[ps_availqty@0 as ps_availqty, ps_supplycost@1 as ps_supplycost] ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@3 as s_nationkey, n_nationkey@0 as n_nationkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=5, partitioning: Hash([s_nationkey@2], 16) === Stage 7 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -40,7 +40,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 8 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([n_nationkey@0], 16) === Stage 9 === SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1], 16) @@ -55,9 +55,9 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16) ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@5 as s_nationkey] SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)] SortExec: expr=[ps_suppkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@1], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([ps_suppkey@1], 16) SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([s_suppkey@0], 16) === Stage 12 === SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) @@ -65,8 +65,8 @@ SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost] ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@4 as s_nationkey, n_nationkey@0 as n_nationkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@3], 16) + UnresolvedShuffleExec: stage=8, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=11, partitioning: Hash([s_nationkey@3], 16) === Stage 13 === ShuffleWriterExec: partitioning: None @@ -76,12 +76,12 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)] AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] CoalescePartitionsExec - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([s_nationkey@2], 16) ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1] AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + UnresolvedShuffleExec: stage=12, partitioning: Hash([ps_partkey@0], 16) === Stage 14 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [value@1 DESC] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + UnresolvedShuffleExec: stage=13, partitioning: Hash([ps_partkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt index 9802dd3a7b..3b86494e39 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt @@ -13,18 +13,18 @@ SortShuffleWriterExec: partitioning=Hash([l_shipmode@0], 16) ProjectionExec: expr=[l_shipmode@1 as l_shipmode, o_orderpriority@3 as o_orderpriority] SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)] SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_orderkey@0], 16) SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([o_orderkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] - UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([l_shipmode@0], 16) === Stage 5 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_shipmode@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt index 252b08f089..10bc255aeb 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt @@ -15,18 +15,18 @@ SortShuffleWriterExec: partitioning=Hash([c_count@0], 16) ProjectionExec: expr=[c_custkey@0 as c_custkey, o_orderkey@1 as o_orderkey] SortMergeJoinExec: join_type=Left, on=[(c_custkey@0, o_custkey@1)] SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16) SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16) === Stage 4 === ShuffleWriterExec: partitioning: None SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] - UnresolvedShuffleExec: partitioning: Hash([c_count@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([c_count@0], 16) === Stage 5 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC] - UnresolvedShuffleExec: partitioning: Hash([c_count@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([c_count@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt index 172add80cf..5f8721f421 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt @@ -13,13 +13,13 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, p_type@4 as p_type] SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)] SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16) SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] CoalescePartitionsExec - UnresolvedShuffleExec: partitioning: Hash([p_partkey@3], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@3], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt index 2a357f08de..3863e7bc8e 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt @@ -9,17 +9,17 @@ ShuffleWriterExec: partitioning: None AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)] ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_suppkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([max(revenue0.total_revenue)@0], 16) AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)] CoalescePartitionsExec - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([l_suppkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([max(revenue0.total_revenue)@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([max(revenue0.total_revenue)@0], 16) === Stage 5 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) @@ -36,11 +36,11 @@ SortShuffleWriterExec: partitioning=Hash([total_revenue@4], 16) ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address, s_phone@3 as s_phone, total_revenue@5 as total_revenue] SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, supplier_no@0)] SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([s_suppkey@0], 16) SortExec: expr=[supplier_no@0 ASC], preserve_partitioning=[true] ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([l_suppkey@0], 16) === Stage 8 === ShuffleWriterExec: partitioning: None @@ -48,10 +48,10 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address, s_phone@3 as s_phone, total_revenue@4 as total_revenue] ProjectionExec: expr=[s_suppkey@1 as s_suppkey, s_name@2 as s_name, s_address@3 as s_address, s_phone@4 as s_phone, total_revenue@5 as total_revenue, max(revenue0.total_revenue)@0 as max(revenue0.total_revenue)] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([total_revenue@4], 16) + UnresolvedShuffleExec: stage=4, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=7, partitioning: Hash([total_revenue@4], 16) === Stage 9 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([total_revenue@4], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([total_revenue@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt index 4403c35874..2947632db4 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([s_suppkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) @@ -21,26 +21,26 @@ SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey, p_brand@3 as p_brand, p_type@4 as p_type, p_size@5 as p_size] SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, p_partkey@0)] SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([ps_partkey@0], 16) SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([p_partkey@0], 16) === Stage 6 === SortShuffleWriterExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 16) AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] AggregateExec: mode=SinglePartitioned, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@0], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=5, partitioning: Hash([ps_suppkey@0], 16) === Stage 7 === ShuffleWriterExec: partitioning: None SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] - UnresolvedShuffleExec: partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) === Stage 8 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt index e3b6744bb2..a74197f02d 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt @@ -20,17 +20,17 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@3 as p_partkey] SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)] SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16) SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16) SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true] ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([l_partkey@0], 16) === Stage 5 === ShuffleWriterExec: partitioning: None ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)] CoalescePartitionsExec - UnresolvedShuffleExec: partitioning: Hash([l_partkey@4], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_partkey@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt index 190add99b8..7cfabb07ef 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt @@ -11,9 +11,9 @@ SortShuffleWriterExec: partitioning=Hash([o_orderkey@2], 16) ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, o_orderkey@2 as o_orderkey, o_totalprice@4 as o_totalprice, o_orderdate@5 as o_orderdate] SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16) SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16) === Stage 4 === SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) @@ -32,15 +32,15 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, o_orderkey@2 as o_orderkey, o_totalprice@3 as o_totalprice, o_orderdate@4 as o_orderdate, l_quantity@6 as l_quantity] SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)] SortExec: expr=[o_orderkey@2 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@2], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderkey@2], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@0], 16) === Stage 7 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], fetch=100 - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@2], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([o_orderkey@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt index a3960ed6eb..acd9d8cf14 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt @@ -14,13 +14,13 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15 SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16) SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] CoalescePartitionsExec - UnresolvedShuffleExec: partitioning: Hash([p_partkey@4], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@4], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt index 5d3dba4594..a3448fbdf5 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([r_regionkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -13,7 +13,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([n_nationkey@0], 16) === Stage 5 === SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) @@ -29,9 +29,9 @@ SortShuffleWriterExec: partitioning=Hash([ps_suppkey@2], 16) ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, ps_suppkey@3 as ps_suppkey, ps_supplycost@4 as ps_supplycost] SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, ps_partkey@0)] SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([p_partkey@0], 16) SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([ps_partkey@0], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) @@ -42,29 +42,29 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@4], 16) ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@5 as s_name, s_address@6 as s_address, s_nationkey@7 as s_nationkey, s_phone@8 as s_phone, s_acctbal@9 as s_acctbal, s_comment@10 as s_comment, ps_supplycost@3 as ps_supplycost] SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@2, s_suppkey@0)] SortExec: expr=[ps_suppkey@2 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@2], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([ps_suppkey@2], 16) SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([s_suppkey@0], 16) === Stage 10 === SortShuffleWriterExec: partitioning=Hash([n_regionkey@9], 16) ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@2 as s_name, s_address@3 as s_address, s_phone@5 as s_phone, s_acctbal@6 as s_acctbal, s_comment@7 as s_comment, ps_supplycost@8 as ps_supplycost, n_name@10 as n_name, n_regionkey@11 as n_regionkey] ProjectionExec: expr=[p_partkey@3 as p_partkey, p_mfgr@4 as p_mfgr, s_name@5 as s_name, s_address@6 as s_address, s_nationkey@7 as s_nationkey, s_phone@8 as s_phone, s_acctbal@9 as s_acctbal, s_comment@10 as s_comment, ps_supplycost@11 as ps_supplycost, n_nationkey@0 as n_nationkey, n_name@1 as n_name, n_regionkey@2 as n_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@4], 16) + UnresolvedShuffleExec: stage=4, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=9, partitioning: Hash([s_nationkey@4], 16) === Stage 11 === SortShuffleWriterExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 16) ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@2 as s_name, s_address@3 as s_address, s_phone@4 as s_phone, s_acctbal@5 as s_acctbal, s_comment@6 as s_comment, ps_supplycost@7 as ps_supplycost, n_name@8 as n_name] ProjectionExec: expr=[p_partkey@1 as p_partkey, p_mfgr@2 as p_mfgr, s_name@3 as s_name, s_address@4 as s_address, s_phone@5 as s_phone, s_acctbal@6 as s_acctbal, s_comment@7 as s_comment, ps_supplycost@8 as ps_supplycost, n_name@9 as n_name, n_regionkey@10 as n_regionkey, r_regionkey@0 as r_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([n_regionkey@9], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=10, partitioning: Hash([n_regionkey@9], 16) === Stage 12 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0, ps_supplycost@7], 16) + UnresolvedShuffleExec: stage=11, partitioning: Hash([p_partkey@0, ps_supplycost@7], 16) === Stage 13 === SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) @@ -73,7 +73,7 @@ SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) === Stage 14 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + UnresolvedShuffleExec: stage=13, partitioning: Hash([r_regionkey@0], 16) === Stage 15 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -81,7 +81,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 16 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=15, partitioning: Hash([n_nationkey@0], 16) === Stage 17 === SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1], 16) @@ -96,17 +96,17 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@4 as s_nationkey] SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)] SortExec: expr=[ps_suppkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@1], 16) + UnresolvedShuffleExec: stage=17, partitioning: Hash([ps_suppkey@1], 16) SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=18, partitioning: Hash([s_suppkey@0], 16) === Stage 20 === SortShuffleWriterExec: partitioning=Hash([n_regionkey@2], 16) ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@1 as ps_supplycost, n_regionkey@4 as n_regionkey] ProjectionExec: expr=[ps_partkey@2 as ps_partkey, ps_supplycost@3 as ps_supplycost, s_nationkey@4 as s_nationkey, n_nationkey@0 as n_nationkey, n_regionkey@1 as n_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + UnresolvedShuffleExec: stage=16, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=19, partitioning: Hash([s_nationkey@2], 16) === Stage 21 === SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) @@ -114,24 +114,24 @@ SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@1 as ps_supplycost] ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@3 as n_regionkey, r_regionkey@0 as r_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([n_regionkey@2], 16) + UnresolvedShuffleExec: stage=14, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=20, partitioning: Hash([n_regionkey@2], 16) === Stage 22 === SortShuffleWriterExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16) ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey] AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + UnresolvedShuffleExec: stage=21, partitioning: Hash([ps_partkey@0], 16) === Stage 23 === ShuffleWriterExec: partitioning: None SortExec: TopK(fetch=100), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@8 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16) + UnresolvedShuffleExec: stage=12, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=22, partitioning: Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16) === Stage 24 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=100 - UnresolvedShuffleExec: partitioning: Hash([p_partkey@3, min(partsupp.ps_supplycost)@9], 16) + UnresolvedShuffleExec: stage=23, partitioning: Hash([p_partkey@3, min(partsupp.ps_supplycost)@9], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt index a684c1a4d2..0066e48430 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([n_nationkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16) @@ -16,8 +16,8 @@ SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address] ProjectionExec: expr=[s_suppkey@1 as s_suppkey, s_name@2 as s_name, s_address@3 as s_address, s_nationkey@4 as s_nationkey, n_nationkey@0 as n_nationkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@3], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=3, partitioning: Hash([s_nationkey@3], 16) === Stage 5 === SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16) @@ -32,9 +32,9 @@ SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) SortShuffleWriterExec: partitioning=Hash([ps_partkey@0, ps_suppkey@1], 16) SortMergeJoinExec: join_type=LeftSemi, on=[(ps_partkey@0, p_partkey@0)] SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([ps_partkey@0], 16) SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([p_partkey@0], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 16) @@ -47,11 +47,11 @@ SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16) ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey] SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1 SortExec: expr=[ps_partkey@0 ASC, ps_suppkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_partkey@0, ps_suppkey@1], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([ps_partkey@0, ps_suppkey@1], 16) SortExec: expr=[l_partkey@1 ASC, l_suppkey@2 ASC], preserve_partitioning=[true] ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@0, l_suppkey@1], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([l_partkey@0, l_suppkey@1], 16) === Stage 10 === ShuffleWriterExec: partitioning: None @@ -59,11 +59,11 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[s_name@1 as s_name, s_address@2 as s_address] SortMergeJoinExec: join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)] SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([s_suppkey@0], 16) SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@0], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([ps_suppkey@0], 16) === Stage 11 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [s_name@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([s_suppkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt index 693152cd68..bd49954afa 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([n_nationkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) @@ -21,9 +21,9 @@ SortShuffleWriterExec: partitioning=Hash([l_orderkey@2], 16) ProjectionExec: expr=[s_name@1 as s_name, s_nationkey@2 as s_nationkey, l_orderkey@3 as l_orderkey, l_suppkey@4 as l_suppkey] SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)] SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([s_suppkey@0], 16) SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_suppkey@1], 16) === Stage 6 === SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) @@ -35,17 +35,17 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@1], 16) ProjectionExec: expr=[s_name@0 as s_name, s_nationkey@1 as s_nationkey, l_orderkey@2 as l_orderkey, l_suppkey@3 as l_suppkey] SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@2, o_orderkey@0)] SortExec: expr=[l_orderkey@2 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@2], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@2], 16) SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([o_orderkey@0], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([l_orderkey@1], 16) ProjectionExec: expr=[s_name@0 as s_name, l_orderkey@2 as l_orderkey, l_suppkey@3 as l_suppkey] ProjectionExec: expr=[s_name@1 as s_name, s_nationkey@2 as s_nationkey, l_orderkey@3 as l_orderkey, l_suppkey@4 as l_suppkey, n_nationkey@0 as n_nationkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@1], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=7, partitioning: Hash([s_nationkey@1], 16) === Stage 9 === SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) @@ -63,20 +63,20 @@ SortShuffleWriterExec: partitioning=Hash([s_name@0], 16) SortMergeJoinExec: join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 SortMergeJoinExec: join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 SortExec: expr=[l_orderkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@1], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([l_orderkey@1], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([l_orderkey@0], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([l_orderkey@0], 16) === Stage 12 === ShuffleWriterExec: partitioning: None SortExec: TopK(fetch=100), expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] - UnresolvedShuffleExec: partitioning: Hash([s_name@0], 16) + UnresolvedShuffleExec: stage=11, partitioning: Hash([s_name@0], 16) === Stage 13 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], fetch=100 - UnresolvedShuffleExec: partitioning: Hash([s_name@0], 16) + UnresolvedShuffleExec: stage=12, partitioning: Hash([s_name@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt index 4dd6095874..8255afee9a 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt @@ -18,18 +18,18 @@ SortShuffleWriterExec: partitioning=Hash([cntrycode@0], 16) ProjectionExec: expr=[c_phone@1 as c_phone, c_acctbal@2 as c_acctbal, CAST(c_acctbal@2 AS Decimal128(19, 6)) as join_proj_push_down_1] SortMergeJoinExec: join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)] SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16) SortExec: expr=[o_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] - UnresolvedShuffleExec: partitioning: Hash([cntrycode@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([cntrycode@0], 16) === Stage 5 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([cntrycode@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([cntrycode@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt index a8d755ca9c..ea275d184b 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt @@ -13,9 +13,9 @@ SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) ProjectionExec: expr=[o_orderkey@1 as o_orderkey, o_orderdate@3 as o_orderdate, o_shippriority@4 as o_shippriority] SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16) SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16) === Stage 4 === SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) @@ -30,11 +30,11 @@ ShuffleWriterExec: partitioning: None ProjectionExec: expr=[o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority, l_orderkey@3 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)] SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderkey@0], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16) === Stage 6 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], fetch=10 - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt index bbb97d34bd..5e05011fef 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt @@ -14,18 +14,18 @@ SortShuffleWriterExec: partitioning=Hash([o_orderpriority@0], 16) ProjectionExec: expr=[o_orderpriority@1 as o_orderpriority] SortMergeJoinExec: join_type=LeftSemi, on=[(o_orderkey@0, l_orderkey@0)] SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([o_orderkey@0], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=2, partitioning: Hash([l_orderkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] - UnresolvedShuffleExec: partitioning: Hash([o_orderpriority@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderpriority@0], 16) === Stage 5 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([o_orderpriority@0], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([o_orderpriority@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt index c159882e44..78c140e223 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([r_regionkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -13,7 +13,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([n_nationkey@0], 16) === Stage 5 === SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) @@ -29,9 +29,9 @@ SortShuffleWriterExec: partitioning=Hash([o_orderkey@1], 16) ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@2 as o_orderkey] SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)] SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([c_custkey@0], 16) SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@1], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([o_custkey@1], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) @@ -42,9 +42,9 @@ SortShuffleWriterExec: partitioning=Hash([l_suppkey@1, c_nationkey@0], 16) ProjectionExec: expr=[c_nationkey@0 as c_nationkey, l_suppkey@3 as l_suppkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)] SortExec: expr=[o_orderkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@1], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([o_orderkey@1], 16) SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([l_orderkey@0], 16) === Stage 10 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0, s_nationkey@1], 16) @@ -55,17 +55,17 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@5 as s_nationkey] SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@1, s_suppkey@0), (c_nationkey@0, s_nationkey@1)] SortExec: expr=[l_suppkey@1 ASC, c_nationkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1, c_nationkey@0], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([l_suppkey@1, c_nationkey@0], 16) SortExec: expr=[s_suppkey@0 ASC, s_nationkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0, s_nationkey@1], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([s_suppkey@0, s_nationkey@1], 16) === Stage 12 === SortShuffleWriterExec: partitioning=Hash([n_regionkey@3], 16) ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@4 as n_name, n_regionkey@5 as n_regionkey] ProjectionExec: expr=[l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, s_nationkey@5 as s_nationkey, n_nationkey@0 as n_nationkey, n_name@1 as n_name, n_regionkey@2 as n_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + UnresolvedShuffleExec: stage=4, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=11, partitioning: Hash([s_nationkey@2], 16) === Stage 13 === SortShuffleWriterExec: partitioning=Hash([n_name@0], 16) @@ -73,17 +73,17 @@ SortShuffleWriterExec: partitioning=Hash([n_name@0], 16) ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@2 as n_name] ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, n_name@3 as n_name, n_regionkey@4 as n_regionkey, r_regionkey@0 as r_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([n_regionkey@3], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=12, partitioning: Hash([n_regionkey@3], 16) === Stage 14 === ShuffleWriterExec: partitioning: None SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - UnresolvedShuffleExec: partitioning: Hash([n_name@0], 16) + UnresolvedShuffleExec: stage=13, partitioning: Hash([n_name@0], 16) === Stage 15 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [revenue@1 DESC] - UnresolvedShuffleExec: partitioning: Hash([n_name@0], 16) + UnresolvedShuffleExec: stage=14, partitioning: Hash([n_name@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt index 77cbc87a8d..abfcad9fb2 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([n_nationkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) @@ -21,9 +21,9 @@ SortShuffleWriterExec: partitioning=Hash([l_orderkey@1], 16) ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_orderkey@2 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, l_shipdate@6 as l_shipdate] SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)] SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([s_suppkey@0], 16) SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_suppkey@1], 16) === Stage 6 === SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) @@ -34,9 +34,9 @@ SortShuffleWriterExec: partitioning=Hash([o_custkey@4], 16) ProjectionExec: expr=[s_nationkey@0 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, o_custkey@6 as o_custkey] SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)] SortExec: expr=[l_orderkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@1], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@1], 16) SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([o_orderkey@0], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) @@ -47,21 +47,21 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@0], 16) ProjectionExec: expr=[s_nationkey@0 as s_nationkey, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@6 as c_nationkey] SortMergeJoinExec: join_type=Inner, on=[(o_custkey@4, c_custkey@0)] SortExec: expr=[o_custkey@4 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@4], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([o_custkey@4], 16) SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([c_custkey@0], 16) === Stage 10 === SortShuffleWriterExec: partitioning=Hash([c_nationkey@3], 16) ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@6 as n_name] ProjectionExec: expr=[s_nationkey@2 as s_nationkey, l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, l_shipdate@5 as l_shipdate, c_nationkey@6 as c_nationkey, n_nationkey@0 as n_nationkey, n_name@1 as n_name] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@0], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=9, partitioning: Hash([s_nationkey@0], 16) === Stage 11 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([c_nationkey@3], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([c_nationkey@3], 16) === Stage 12 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -73,17 +73,17 @@ SortShuffleWriterExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2 AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] ProjectionExec: expr=[n_name@4 as supp_nation, n_name@6 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=11, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=12, partitioning: Hash([n_nationkey@0], 16) === Stage 14 === ShuffleWriterExec: partitioning: None SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - UnresolvedShuffleExec: partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) + UnresolvedShuffleExec: stage=13, partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) === Stage 15 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) + UnresolvedShuffleExec: stage=14, partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt index d70be863e9..912a825601 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt @@ -5,7 +5,7 @@ SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([r_regionkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([r_regionkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -13,7 +13,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 4 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([n_nationkey@0], 16) === Stage 5 === SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) @@ -29,9 +29,9 @@ SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16) ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_suppkey@3 as l_suppkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, l_partkey@1)] SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([p_partkey@0], 16) SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@1], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([l_partkey@1], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) @@ -42,9 +42,9 @@ SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@5 as s_nationkey] SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@1, s_suppkey@0)] SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@1], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([l_suppkey@1], 16) SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([s_suppkey@0], 16) === Stage 10 === SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) @@ -56,9 +56,9 @@ SortShuffleWriterExec: partitioning=Hash([o_custkey@3], 16) ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_custkey@5 as o_custkey, o_orderdate@6 as o_orderdate] SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)] SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([l_orderkey@0], 16) SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([o_orderkey@0], 16) === Stage 12 === SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16) @@ -69,21 +69,21 @@ SortShuffleWriterExec: partitioning=Hash([c_nationkey@4], 16) ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, s_nationkey@2 as s_nationkey, o_orderdate@4 as o_orderdate, c_nationkey@6 as c_nationkey] SortMergeJoinExec: join_type=Inner, on=[(o_custkey@3, c_custkey@0)] SortExec: expr=[o_custkey@3 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_custkey@3], 16) + UnresolvedShuffleExec: stage=11, partitioning: Hash([o_custkey@3], 16) SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([c_custkey@0], 16) + UnresolvedShuffleExec: stage=12, partitioning: Hash([c_custkey@0], 16) === Stage 14 === SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16) ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, s_nationkey@2 as s_nationkey, o_orderdate@3 as o_orderdate, n_regionkey@6 as n_regionkey] ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_orderdate@5 as o_orderdate, c_nationkey@6 as c_nationkey, n_nationkey@0 as n_nationkey, n_regionkey@1 as n_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([c_nationkey@4], 16) + UnresolvedShuffleExec: stage=4, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=13, partitioning: Hash([c_nationkey@4], 16) === Stage 15 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@2], 16) + UnresolvedShuffleExec: stage=14, partitioning: Hash([s_nationkey@2], 16) === Stage 16 === SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) @@ -93,8 +93,8 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) SortShuffleWriterExec: partitioning=Hash([n_regionkey@3], 16) ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@6 as n_name] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=15, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=16, partitioning: Hash([n_nationkey@0], 16) === Stage 18 === SortShuffleWriterExec: partitioning=Hash([o_year@0], 16) @@ -102,17 +102,17 @@ SortShuffleWriterExec: partitioning=Hash([o_year@0], 16) ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@4 as nation] ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@5 as n_name, r_regionkey@0 as r_regionkey] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([n_regionkey@3], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=17, partitioning: Hash([n_regionkey@3], 16) === Stage 19 === ShuffleWriterExec: partitioning: None SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - UnresolvedShuffleExec: partitioning: Hash([o_year@0], 16) + UnresolvedShuffleExec: stage=18, partitioning: Hash([o_year@0], 16) === Stage 20 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [o_year@0 ASC NULLS LAST] - UnresolvedShuffleExec: partitioning: Hash([o_year@0], 16) + UnresolvedShuffleExec: stage=19, partitioning: Hash([o_year@0], 16) diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt index 3d91e1dd21..0f0fa2d637 100644 --- a/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt +++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt @@ -4,7 +4,7 @@ SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16) === Stage 2 === ShuffleWriterExec: partitioning: None - UnresolvedShuffleExec: partitioning: Hash([n_nationkey@0], 16) + UnresolvedShuffleExec: stage=1, partitioning: Hash([n_nationkey@0], 16) === Stage 3 === SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16) @@ -20,9 +20,9 @@ SortShuffleWriterExec: partitioning=Hash([l_suppkey@2], 16) ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount] SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, l_partkey@1)] SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([p_partkey@0], 16) + UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@0], 16) SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_partkey@1], 16) + UnresolvedShuffleExec: stage=4, partitioning: Hash([l_partkey@1], 16) === Stage 6 === SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16) @@ -33,9 +33,9 @@ SortShuffleWriterExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 16) ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_partkey@1 as l_partkey, l_suppkey@2 as l_suppkey, l_quantity@3 as l_quantity, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, s_nationkey@7 as s_nationkey] SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@2, s_suppkey@0)] SortExec: expr=[l_suppkey@2 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@2], 16) + UnresolvedShuffleExec: stage=5, partitioning: Hash([l_suppkey@2], 16) SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([s_suppkey@0], 16) + UnresolvedShuffleExec: stage=6, partitioning: Hash([s_suppkey@0], 16) === Stage 8 === SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 16) @@ -46,9 +46,9 @@ SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16) ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_quantity@3 as l_quantity, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, s_nationkey@6 as s_nationkey, ps_supplycost@9 as ps_supplycost] SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)] SortExec: expr=[l_suppkey@2 ASC, l_partkey@1 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_suppkey@2, l_partkey@1], 16) + UnresolvedShuffleExec: stage=7, partitioning: Hash([l_suppkey@2, l_partkey@1], 16) SortExec: expr=[ps_suppkey@1 ASC, ps_partkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([ps_suppkey@1, ps_partkey@0], 16) + UnresolvedShuffleExec: stage=8, partitioning: Hash([ps_suppkey@1, ps_partkey@0], 16) === Stage 10 === SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16) @@ -59,9 +59,9 @@ SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16) ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@7 as o_orderdate] SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)] SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 16) + UnresolvedShuffleExec: stage=9, partitioning: Hash([l_orderkey@0], 16) SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true] - UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 16) + UnresolvedShuffleExec: stage=10, partitioning: Hash([o_orderkey@0], 16) === Stage 12 === SortShuffleWriterExec: partitioning=Hash([nation@0, o_year@1], 16) @@ -69,17 +69,17 @@ SortShuffleWriterExec: partitioning=Hash([nation@0, o_year@1], 16) ProjectionExec: expr=[n_name@7 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) - ps_supplycost@4 * l_quantity@0 as amount] ProjectionExec: expr=[l_quantity@2 as l_quantity, l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, s_nationkey@5 as s_nationkey, ps_supplycost@6 as ps_supplycost, o_orderdate@7 as o_orderdate, n_nationkey@0 as n_nationkey, n_name@1 as n_name] HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)] - UnresolvedShuffleExec: broadcast=true, upstream_partitions: 16 - UnresolvedShuffleExec: partitioning: Hash([s_nationkey@3], 16) + UnresolvedShuffleExec: stage=2, broadcast=true, upstream_partitions: 16 + UnresolvedShuffleExec: stage=11, partitioning: Hash([s_nationkey@3], 16) === Stage 13 === ShuffleWriterExec: partitioning: None SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - UnresolvedShuffleExec: partitioning: Hash([nation@0, o_year@1], 16) + UnresolvedShuffleExec: stage=12, partitioning: Hash([nation@0, o_year@1], 16) === Stage 14 === ShuffleWriterExec: partitioning: None SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] - UnresolvedShuffleExec: partitioning: Hash([nation@0, o_year@1], 16) + UnresolvedShuffleExec: stage=13, partitioning: Hash([nation@0, o_year@1], 16) From ce768eb35f199d3e02b7f621dd3c97a8fbdb11d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:52:08 -0600 Subject: [PATCH 035/107] chore(deps): bump aws-config from 1.8.18 to 1.9.0 (#1976) Bumps [aws-config](https://github.com/smithy-lang/smithy-rs) from 1.8.18 to 1.9.0. - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) --- updated-dependencies: - dependency-name: aws-config dependency-version: 1.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 86 +++++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d0b4e1d48..6148df5d24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -557,9 +557,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.18" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" dependencies = [ "aws-credential-types", "aws-runtime", @@ -588,9 +588,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -622,9 +622,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.4" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ed8e8c52d2dc2390ad9f15647fe663f71e9780b4262c190fbb823a32721566" +checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -647,9 +647,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.101.0" +version = "1.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b647baea49ff551960b904f905681e9b4765a6c4ea08631e89dc52d8bd3f5896" +checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401" dependencies = [ "arc-swap", "aws-credential-types", @@ -660,6 +660,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -672,9 +673,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.103.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae401c65ff288aa7873117fe535cd32b7b1bb0bc43751d28901a1d5f20636b9" +checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a" dependencies = [ "arc-swap", "aws-credential-types", @@ -685,6 +686,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -697,9 +699,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.106.0" +version = "1.108.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c80de7bb7d03e9ca8c9fd7b489f20f3948d3f3be91a7953591347d238115408" +checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f" dependencies = [ "arc-swap", "aws-credential-types", @@ -711,6 +713,7 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -723,9 +726,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.5" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -745,9 +748,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -756,9 +759,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.63.6" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -777,9 +780,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.13" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -801,9 +804,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.7" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -812,18 +815,18 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77" dependencies = [ "aws-smithy-types", "urlencoding", @@ -831,9 +834,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.11.3" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" +checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -857,9 +860,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.12.3" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" +checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -875,9 +878,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", @@ -886,9 +889,9 @@ dependencies = [ [[package]] name = "aws-smithy-schema" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -897,9 +900,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", @@ -920,18 +923,21 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.16" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86" dependencies = [ "aws-credential-types", "aws-smithy-async", From 006af4b87263b3eccf8e65a23be62eeec9ac7951 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:52:26 -0600 Subject: [PATCH 036/107] chore(deps): bump aws-credential-types from 1.2.14 to 1.3.0 (#1975) Bumps [aws-credential-types](https://github.com/smithy-lang/smithy-rs) from 1.2.14 to 1.3.0. - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) --- updated-dependencies: - dependency-name: aws-credential-types dependency-version: 1.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From b93bcc0df799cbf1e699156a421cd0083b5631ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:52:48 -0600 Subject: [PATCH 037/107] chore(deps): bump astral-sh/setup-uv from 8.3.0 to 8.3.2 (#1970) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 8.3.0 to 8.3.2. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/d31148d669074a8d0a63714ba94f3201e7020bc3...11f9893b081a58869d3b5fccaea48c9e9e46f990) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 8.3.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 79018b07e9..91a47671af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true @@ -77,7 +77,7 @@ jobs: with: python-version: "3.12" - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true @@ -100,7 +100,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7.0.0 - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true - name: Generate license file @@ -148,7 +148,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true @@ -194,7 +194,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true @@ -236,7 +236,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true @@ -278,7 +278,7 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true From 8a4ef0bd36ad3bc13da890ed433d900c5e5f07f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:53:05 -0600 Subject: [PATCH 038/107] chore(deps): bump actions/labeler from 6.1.0 to 6.2.0 (#1972) Bumps [actions/labeler](https://github.com/actions/labeler) from 6.1.0 to 6.2.0. - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/v6.1.0...v6.2.0) --- updated-dependencies: - dependency-name: actions/labeler dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 106463a206..7b96019abc 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -38,7 +38,7 @@ jobs: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize') - uses: actions/labeler@v6.1.0 + uses: actions/labeler@v6.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/workflows/dev_pr/labeler.yml From fd30f37754009977d0fba47e3c91ac6da72c4cd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:53:24 -0600 Subject: [PATCH 039/107] chore(deps): bump taiki-e/install-action from 2.82.7 to 2.82.11 (#1973) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.82.7 to 2.82.11. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/16b05812d776ae1dfaabc8277e421fb6d2506419...5ebac0d9522d786674368e47e92963ba13f2c376) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.82.11 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 076eb711c8..6742b850e5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 + uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 2bed39bef4..2116e44e17 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 + uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 2290f62c19..b6a25217b2 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,12 +64,12 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 + uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 with: tool: trunk@0.21.14 - name: Install cargo-get - uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7 + uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 with: tool: cargo-get@1.4.0 From 647cefe4d8dc1d0b4d682e0021d2cffd9393e8a7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 9 Jul 2026 08:22:02 -0600 Subject: [PATCH 040/107] docs: document TPC-H plan-stability golden files and regeneration (#1979) Add a section to the contributor development guide explaining the tpch_plan_stability test suite: what the approved golden plans capture, that they are planned deterministically against a synthetic stats table (no data or cluster required), and how to regenerate them with dev/update-tpch-plan-stability.sh when a change intentionally alters distributed planning. --- docs/source/contributors-guide/development.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/source/contributors-guide/development.md b/docs/source/contributors-guide/development.md index a361c897fd..bf404d3186 100644 --- a/docs/source/contributors-guide/development.md +++ b/docs/source/contributors-guide/development.md @@ -59,6 +59,31 @@ cargo build --release cargo test ``` +### TPC-H plan-stability golden files + +The `tpch_plan_stability` test suite in the `ballista-scheduler` crate guards the shape of the +distributed physical plans that Ballista produces for the 22 TPC-H queries. For each query it plans +the SQL and compares the staged plan text — the stage boundaries (`=== Stage N ===` banners), +`ShuffleWriterExec` / `UnresolvedShuffleExec` nodes, and any exchange reuse — against an approved +"golden" file. The queries are planned against a synthetic statistics-only table, so the suite is +deterministic and runs without TPC-H data or a running cluster. + +The golden files live in +[`ballista/scheduler/tests/tpch_plan_stability/approved/`](https://github.com/apache/datafusion-ballista/tree/main/ballista/scheduler/tests/tpch_plan_stability/approved) +(`q1.txt` through `q22.txt`). When a change alters distributed planning, the suite fails with a +plan-drift diff. Review the diff: if the change is unintended, fix the regression; if the new plans +are correct, regenerate the golden files: + +```shell +./dev/update-tpch-plan-stability.sh +``` + +This runs the suite in generate mode +(`BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability`) and +rewrites the approved files. Review the resulting changes under +`ballista/scheduler/tests/tpch_plan_stability/approved/` and commit them together with the code +change that caused them. + ## Running the examples ```shell From ec79000d4dc3c9b887dc0cf504dc7acf7a88e2ef Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 9 Jul 2026 08:41:04 -0600 Subject: [PATCH 041/107] fix: satisfy clippy 1.97 useless_borrows_in_formatting (#1978) --- .../src/tui/ui/main/jobs/stage_tasks_popup.rs | 2 +- ballista/client/src/extension.rs | 2 +- ballista/core/src/object_store.rs | 2 +- ballista/executor/src/executor_process.rs | 2 +- ballista/executor/src/executor_server.rs | 14 +++++++------- ballista/scheduler/src/scheduler_server/grpc.rs | 4 ++-- .../src/scheduler_server/query_stage_scheduler.rs | 2 +- benchmarks/src/bin/tpch.rs | 8 ++++---- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs b/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs index dfa3401604..76f9d769c1 100644 --- a/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs +++ b/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs @@ -77,7 +77,7 @@ pub(crate) fn render_stage_tasks_popup(f: &mut Frame, app: &App) { Block::default() .title(format!( " Tasks for stage '{}' of job '{}' ", - stage.id, &popup.job_id + stage.id, popup.job_id )) .borders(Borders::ALL) .border_style(app.theme.popup_border_alt) diff --git a/ballista/client/src/extension.rs b/ballista/client/src/extension.rs index 54b2123442..521b64bac8 100644 --- a/ballista/client/src/extension.rs +++ b/ballista/client/src/extension.rs @@ -169,7 +169,7 @@ impl Extension { "hostname should be provided".to_string(), ))?; let port = url.port().unwrap_or(DEFAULT_SCHEDULER_PORT); - let scheduler_url = format!("http://{}:{}", &host, port); + let scheduler_url = format!("http://{}:{}", host, port); Ok(scheduler_url) } diff --git a/ballista/core/src/object_store.rs b/ballista/core/src/object_store.rs index b26e0f193e..ce0ea01fa0 100644 --- a/ballista/core/src/object_store.rs +++ b/ballista/core/src/object_store.rs @@ -145,7 +145,7 @@ impl ObjectStoreRegistry for CustomObjectStoreRegistry { fn get_store(&self, url: &Url) -> Result> { let scheme = url.scheme(); - log::trace!("get_store: {:?}", &self.s3options.config.read()); + log::trace!("get_store: {:?}", self.s3options.config.read()); match scheme { "" | "file" => Ok(self.local.clone()), "http" | "https" => Ok(Arc::new( diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 6050649ab9..22f526edca 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -728,7 +728,7 @@ async fn clean_all_shuffle_data(work_dir: &str) -> ballista_core::error::Result< } } - info!("The work_dir {:?} will be deleted", &to_deleted); + info!("The work_dir {:?} will be deleted", to_deleted); for del in to_deleted { if let Err(e) = fs::remove_dir_all(&del).await { error!("Fail to remove the directory {del:?} due to {e}"); diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs index 3900a0f384..c7c564fe35 100644 --- a/ballista/executor/src/executor_server.rs +++ b/ballista/executor/src/executor_server.rs @@ -778,14 +778,14 @@ impl TaskRunnerPool SchedulerGrpc .map_err(|e| { let msg = format!( "Fail to update tasks status from executor {:?} due to {:?}", - &executor_id, e + executor_id, e ); error!("{msg}"); Status::internal(msg) @@ -326,7 +326,7 @@ impl SchedulerGrpc .map_err(|e| { let msg = format!( "Fail to update tasks status from executor {:?} due to {:?}", - &executor_id, e + executor_id, e ); error!("{msg}"); Status::internal(msg) diff --git a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs index 0d84950bc7..9bf33307c9 100644 --- a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs +++ b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs @@ -146,7 +146,7 @@ impl } } - error!("{}", &fail_message); + error!("{}", fail_message); QueryStageSchedulerEvent::JobPlanningFailed { job_id, fail_message, diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index 0947fe5790..c0d7ba1aae 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -614,7 +614,7 @@ async fn loadtest_ballista(opt: BallistaLoadtestOpt) -> Result<()> { .split(',') .map(|s| s.parse().unwrap()) .collect(); - println!("query list: {:?} ", &query_list); + println!("query list: {:?} ", query_list); let total = Instant::now(); let mut futures = vec![]; @@ -636,7 +636,7 @@ async fn loadtest_ballista(opt: BallistaLoadtestOpt) -> Result<()> { .unwrap(); println!( "Client {} Round {} Query {} started", - &client_id, &i, query_id + client_id, i, query_id ); let start = Instant::now(); let df = client @@ -652,7 +652,7 @@ async fn loadtest_ballista(opt: BallistaLoadtestOpt) -> Result<()> { let elapsed = start.elapsed().as_secs_f64() * 1000.0; println!( "Client {} Round {} Query {} took {:.1} ms ", - &client_id, &i, query_id, elapsed + client_id, i, query_id, elapsed ); if opt.debug && !batches.is_empty() { pretty::print_batches(&batches).unwrap(); @@ -945,7 +945,7 @@ async fn convert_tbl(opt: ConvertOpt) -> Result<()> { println!( "Converting '{}' to {} files in directory '{}'", - &input_path, &opt.file_format, &output_path + input_path, opt.file_format, output_path ); match opt.file_format.as_str() { "csv" => ctx.write_csv(csv, output_path).await?, From 2959cb7f9cd6a6515bb14cdde9f67302cacea812 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 9 Jul 2026 21:18:58 -0600 Subject: [PATCH 042/107] perf: avoid redundant TaskStatus clones in scheduler update_task_status (#1983) --- ballista/scheduler/src/state/aqe/mod.rs | 19 ++++++++-------- .../scheduler/src/state/execution_graph.rs | 22 +++++++++---------- .../scheduler/src/state/execution_stage.rs | 10 ++++----- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index 74cedb1f40..7a43d1d887 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -656,7 +656,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph { ); continue; } - let partition_id = task_status.clone().partition_id as usize; + let partition_id = task_status.partition_id as usize; let task_identity = format!( "TID {} {}/{}.{}/{}", task_status.task_id, @@ -665,19 +665,20 @@ impl ExecutionGraph for AdaptiveExecutionGraph { task_stage_attempt_num, partition_id ); - let operator_metrics = task_status.metrics.clone(); - if !running_stage - .update_task_info(partition_id, task_status.clone()) - { + if !running_stage.update_task_info(partition_id, &task_status) { continue; } + + let TaskStatus { + status, + metrics: operator_metrics, + .. + } = task_status; // // handle task failure // - if let Some(task_status::Status::Failed(failed_task)) = - task_status.status - { + if let Some(task_status::Status::Failed(failed_task)) = status { let failed_reason = failed_task.failed_reason; match failed_reason { @@ -786,7 +787,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph { // else if let Some(task_status::Status::Successful( successful_task, - )) = task_status.status + )) = status { // update task metrics for successfu task running_stage diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 79cbf9c70b..879a172964 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -791,7 +791,7 @@ impl ExecutionGraph for StaticExecutionGraph { ); continue; } - let partition_id = task_status.clone().partition_id as usize; + let partition_id = task_status.partition_id as usize; let task_identity = format!( "TID {} {}/{}.{}/{}", task_status.task_id, @@ -800,17 +800,17 @@ impl ExecutionGraph for StaticExecutionGraph { task_stage_attempt_num, partition_id ); - let operator_metrics = task_status.metrics.clone(); - - if !running_stage - .update_task_info(partition_id, task_status.clone()) - { + if !running_stage.update_task_info(partition_id, &task_status) { continue; } - if let Some(task_status::Status::Failed(failed_task)) = - task_status.status - { + let TaskStatus { + status, + metrics: operator_metrics, + .. + } = task_status; + + if let Some(task_status::Status::Failed(failed_task)) = status { let failed_reason = failed_task.failed_reason; match failed_reason { @@ -915,7 +915,7 @@ impl ExecutionGraph for StaticExecutionGraph { } } else if let Some(task_status::Status::Successful( successful_task, - )) = task_status.status + )) = status { // update task metrics for successfu task running_stage @@ -966,7 +966,7 @@ impl ExecutionGraph for StaticExecutionGraph { for task_status in stage_task_statuses.into_iter() { let task_stage_attempt_num = task_status.stage_attempt_num as usize; - let partition_id = task_status.clone().partition_id as usize; + let partition_id = task_status.partition_id as usize; let task_identity = format!( "TID {} {}/{}.{}/{}", task_status.task_id, diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index 698d951828..10440f357a 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -669,7 +669,7 @@ impl RunningStage { } /// Update the TaskInfo for task partition - pub fn update_task_info(&mut self, partition_id: usize, status: TaskStatus) -> bool { + pub fn update_task_info(&mut self, partition_id: usize, status: &TaskStatus) -> bool { debug!("Updating TaskInfo for partition {partition_id}"); let Some(task_info) = self.task_infos[partition_id].as_ref() else { warn!( @@ -686,7 +686,7 @@ impl RunningStage { return false; } let scheduled_time = task_info.scheduled_time; - let task_status = status.status.unwrap(); + let task_status = status.status.as_ref().unwrap(); let updated_task_info = TaskInfo { task_id, scheduled_time, @@ -1178,7 +1178,7 @@ mod tests { // Simulates receiving a status update for a task that was already // reset (e.g., executor heartbeat timed out). let status = make_task_status(0, 0); - let result = stage.update_task_info(0, status); + let result = stage.update_task_info(0, &status); // Should return false (update rejected), not panic. assert!(!result); @@ -1203,7 +1203,7 @@ mod tests { }); let status = make_task_status(0, 0); - let result = stage.update_task_info(0, status); + let result = stage.update_task_info(0, &status); assert!(result); assert!(matches!( @@ -1240,7 +1240,7 @@ mod tests { // Executor sends a late status update for partition 0. let status = make_task_status(0, 0); - let result = stage.update_task_info(0, status); + let result = stage.update_task_info(0, &status); // Should gracefully reject the update, not panic. assert!(!result); From 434012f2ee3d88c12a6ff2030faf4573aa0a13de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:59:43 +0300 Subject: [PATCH 043/107] chore(deps): bump regex from 1.12.4 to 1.13.0 (#1985) Bumps [regex](https://github.com/rust-lang/regex) from 1.12.4 to 1.13.0. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.12.4...1.13.0) --- updated-dependencies: - dependency-name: regex dependency-version: 1.13.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6148df5d24..1c136dbe1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5930,9 +5930,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", From 4f5e7c71e28ef85124398f61199387f838890d4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:00:03 +0300 Subject: [PATCH 044/107] chore(deps): bump taiki-e/install-action from 2.82.11 to 2.83.0 (#1984) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.82.11 to 2.83.0. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/5ebac0d9522d786674368e47e92963ba13f2c376...c7eb1735f09259a5035e8e5d44b1406b1cddc0fb) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.83.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6742b850e5..cb0a3a48ef 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 + uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 2116e44e17..c5bc57159d 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 + uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index b6a25217b2..4bca1c4117 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,12 +64,12 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 + uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 with: tool: trunk@0.21.14 - name: Install cargo-get - uses: taiki-e/install-action@5ebac0d9522d786674368e47e92963ba13f2c376 # v2.82.11 + uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 with: tool: cargo-get@1.4.0 From aad83528816c8a4ff85543c7bf5ed205ead1ff36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:00:14 +0300 Subject: [PATCH 045/107] chore(deps): bump pyarrow from 23.0.0 to 23.0.1 in /python (#1961) Bumps [pyarrow](https://github.com/apache/arrow) from 23.0.0 to 23.0.1. - [Release notes](https://github.com/apache/arrow/releases) - [Commits](https://github.com/apache/arrow/compare/apache-arrow-23.0.0...apache-arrow-23.0.1) --- updated-dependencies: - dependency-name: pyarrow dependency-version: 23.0.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/pyproject.toml | 2 +- python/uv.lock | 108 +++++++++++++++++++++--------------------- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index b3a24c9cfc..b3818cf374 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -40,7 +40,7 @@ classifiers = [ "Programming Language :: Rust", ] dependencies = [ - "pyarrow>=21.0.0", + "pyarrow>=23.0.1", "datafusion==54", "typing-extensions;python_version<'3.13'", ] diff --git a/python/uv.lock b/python/uv.lock index 5935482cf4..9e5c92d547 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -54,7 +54,7 @@ dev = [ requires-dist = [ { name = "datafusion", specifier = "==54" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=8.0.0" }, - { name = "pyarrow", specifier = ">=21.0.0" }, + { name = "pyarrow", specifier = ">=23.0.1" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] provides-extras = ["jupyter"] @@ -598,59 +598,59 @@ wheels = [ [[package]] name = "pyarrow" -version = "23.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/33/ffd9c3eb087fa41dd79c3cf20c4c0ae3cdb877c4f8e1107a446006344924/pyarrow-23.0.0.tar.gz", hash = "sha256:180e3150e7edfcd182d3d9afba72f7cf19839a497cc76555a8dce998a8f67615", size = 1167185, upload-time = "2026-01-18T16:19:42.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/2f/23e042a5aa99bcb15e794e14030e8d065e00827e846e53a66faec73c7cd6/pyarrow-23.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:cbdc2bf5947aa4d462adcf8453cf04aee2f7932653cb67a27acd96e5e8528a67", size = 34281861, upload-time = "2026-01-18T16:13:34.332Z" }, - { url = "https://files.pythonhosted.org/packages/8b/65/1651933f504b335ec9cd8f99463718421eb08d883ed84f0abd2835a16cad/pyarrow-23.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:4d38c836930ce15cd31dce20114b21ba082da231c884bdc0a7b53e1477fe7f07", size = 35825067, upload-time = "2026-01-18T16:13:42.549Z" }, - { url = "https://files.pythonhosted.org/packages/84/ec/d6fceaec050c893f4e35c0556b77d4cc9973fcc24b0a358a5781b1234582/pyarrow-23.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4222ff8f76919ecf6c716175a0e5fddb5599faeed4c56d9ea41a2c42be4998b2", size = 44458539, upload-time = "2026-01-18T16:13:52.975Z" }, - { url = "https://files.pythonhosted.org/packages/fd/d9/369f134d652b21db62fe3ec1c5c2357e695f79eb67394b8a93f3a2b2cffa/pyarrow-23.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:87f06159cbe38125852657716889296c83c37b4d09a5e58f3d10245fd1f69795", size = 47535889, upload-time = "2026-01-18T16:14:03.693Z" }, - { url = "https://files.pythonhosted.org/packages/a3/95/f37b6a252fdbf247a67a78fb3f61a529fe0600e304c4d07741763d3522b1/pyarrow-23.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1675c374570d8b91ea6d4edd4608fa55951acd44e0c31bd146e091b4005de24f", size = 48157777, upload-time = "2026-01-18T16:14:12.483Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ab/fb94923108c9c6415dab677cf1f066d3307798eafc03f9a65ab4abc61056/pyarrow-23.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:247374428fde4f668f138b04031a7e7077ba5fa0b5b1722fdf89a017bf0b7ee0", size = 50580441, upload-time = "2026-01-18T16:14:20.187Z" }, - { url = "https://files.pythonhosted.org/packages/ae/78/897ba6337b517fc8e914891e1bd918da1c4eb8e936a553e95862e67b80f6/pyarrow-23.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:de53b1bd3b88a2ee93c9af412c903e57e738c083be4f6392288294513cd8b2c1", size = 27530028, upload-time = "2026-01-18T16:14:27.353Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c0/57fe251102ca834fee0ef69a84ad33cc0ff9d5dfc50f50b466846356ecd7/pyarrow-23.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5574d541923efcbfdf1294a2746ae3b8c2498a2dc6cd477882f6f4e7b1ac08d3", size = 34276762, upload-time = "2026-01-18T16:14:34.128Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4e/24130286548a5bc250cbed0b6bbf289a2775378a6e0e6f086ae8c68fc098/pyarrow-23.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:2ef0075c2488932e9d3c2eb3482f9459c4be629aa673b725d5e3cf18f777f8e4", size = 35821420, upload-time = "2026-01-18T16:14:40.699Z" }, - { url = "https://files.pythonhosted.org/packages/ee/55/a869e8529d487aa2e842d6c8865eb1e2c9ec33ce2786eb91104d2c3e3f10/pyarrow-23.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:65666fc269669af1ef1c14478c52222a2aa5c907f28b68fb50a203c777e4f60c", size = 44457412, upload-time = "2026-01-18T16:14:49.051Z" }, - { url = "https://files.pythonhosted.org/packages/36/81/1de4f0edfa9a483bbdf0082a05790bd6a20ed2169ea12a65039753be3a01/pyarrow-23.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4d85cb6177198f3812db4788e394b757223f60d9a9f5ad6634b3e32be1525803", size = 47534285, upload-time = "2026-01-18T16:14:56.748Z" }, - { url = "https://files.pythonhosted.org/packages/f2/04/464a052d673b5ece074518f27377861662449f3c1fdb39ce740d646fd098/pyarrow-23.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1a9ff6fa4141c24a03a1a434c63c8fa97ce70f8f36bccabc18ebba905ddf0f17", size = 48157913, upload-time = "2026-01-18T16:15:05.114Z" }, - { url = "https://files.pythonhosted.org/packages/f4/1b/32a4de9856ee6688c670ca2def588382e573cce45241a965af04c2f61687/pyarrow-23.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:84839d060a54ae734eb60a756aeacb62885244aaa282f3c968f5972ecc7b1ecc", size = 50582529, upload-time = "2026-01-18T16:15:12.846Z" }, - { url = "https://files.pythonhosted.org/packages/db/c7/d6581f03e9b9e44ea60b52d1750ee1a7678c484c06f939f45365a45f7eef/pyarrow-23.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a149a647dbfe928ce8830a713612aa0b16e22c64feac9d1761529778e4d4eaa5", size = 27542646, upload-time = "2026-01-18T16:15:18.89Z" }, - { url = "https://files.pythonhosted.org/packages/3d/bd/c861d020831ee57609b73ea721a617985ece817684dc82415b0bc3e03ac3/pyarrow-23.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5961a9f646c232697c24f54d3419e69b4261ba8a8b66b0ac54a1851faffcbab8", size = 34189116, upload-time = "2026-01-18T16:15:28.054Z" }, - { url = "https://files.pythonhosted.org/packages/8c/23/7725ad6cdcbaf6346221391e7b3eecd113684c805b0a95f32014e6fa0736/pyarrow-23.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:632b3e7c3d232f41d64e1a4a043fb82d44f8a349f339a1188c6a0dd9d2d47d8a", size = 35803831, upload-time = "2026-01-18T16:15:33.798Z" }, - { url = "https://files.pythonhosted.org/packages/57/06/684a421543455cdc2944d6a0c2cc3425b028a4c6b90e34b35580c4899743/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:76242c846db1411f1d6c2cc3823be6b86b40567ee24493344f8226ba34a81333", size = 44436452, upload-time = "2026-01-18T16:15:41.598Z" }, - { url = "https://files.pythonhosted.org/packages/c6/6f/8f9eb40c2328d66e8b097777ddcf38494115ff9f1b5bc9754ba46991191e/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b73519f8b52ae28127000986bf228fda781e81d3095cd2d3ece76eb5cf760e1b", size = 47557396, upload-time = "2026-01-18T16:15:51.252Z" }, - { url = "https://files.pythonhosted.org/packages/10/6e/f08075f1472e5159553501fde2cc7bc6700944bdabe49a03f8a035ee6ccd/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:068701f6823449b1b6469120f399a1239766b117d211c5d2519d4ed5861f75de", size = 48147129, upload-time = "2026-01-18T16:16:00.299Z" }, - { url = "https://files.pythonhosted.org/packages/7d/82/d5a680cd507deed62d141cc7f07f7944a6766fc51019f7f118e4d8ad0fb8/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1801ba947015d10e23bca9dd6ef5d0e9064a81569a89b6e9a63b59224fd060df", size = 50596642, upload-time = "2026-01-18T16:16:08.502Z" }, - { url = "https://files.pythonhosted.org/packages/a9/26/4f29c61b3dce9fa7780303b86895ec6a0917c9af927101daaaf118fbe462/pyarrow-23.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:52265266201ec25b6839bf6bd4ea918ca6d50f31d13e1cf200b4261cd11dc25c", size = 27660628, upload-time = "2026-01-18T16:16:15.28Z" }, - { url = "https://files.pythonhosted.org/packages/66/34/564db447d083ec7ff93e0a883a597d2f214e552823bfc178a2d0b1f2c257/pyarrow-23.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:ad96a597547af7827342ffb3c503c8316e5043bb09b47a84885ce39394c96e00", size = 34184630, upload-time = "2026-01-18T16:16:22.141Z" }, - { url = "https://files.pythonhosted.org/packages/aa/3a/3999daebcb5e6119690c92a621c4d78eef2ffba7a0a1b56386d2875fcd77/pyarrow-23.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:b9edf990df77c2901e79608f08c13fbde60202334a4fcadb15c1f57bf7afee43", size = 35796820, upload-time = "2026-01-18T16:16:29.441Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/39195233056c6a8d0976d7d1ac1cd4fe21fb0ec534eca76bc23ef3f60e11/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:36d1b5bc6ddcaff0083ceec7e2561ed61a51f49cce8be079ee8ed406acb6fdef", size = 44438735, upload-time = "2026-01-18T16:16:38.79Z" }, - { url = "https://files.pythonhosted.org/packages/2c/41/6a7328ee493527e7afc0c88d105ecca69a3580e29f2faaeac29308369fd7/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4292b889cd224f403304ddda8b63a36e60f92911f89927ec8d98021845ea21be", size = 47557263, upload-time = "2026-01-18T16:16:46.248Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ee/34e95b21ee84db494eae60083ddb4383477b31fb1fd19fd866d794881696/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dfd9e133e60eaa847fd80530a1b89a052f09f695d0b9c34c235ea6b2e0924cf7", size = 48153529, upload-time = "2026-01-18T16:16:53.412Z" }, - { url = "https://files.pythonhosted.org/packages/52/88/8a8d83cea30f4563efa1b7bf51d241331ee5cd1b185a7e063f5634eca415/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832141cc09fac6aab1cd3719951d23301396968de87080c57c9a7634e0ecd068", size = 50598851, upload-time = "2026-01-18T16:17:01.133Z" }, - { url = "https://files.pythonhosted.org/packages/c6/4c/2929c4be88723ba025e7b3453047dc67e491c9422965c141d24bab6b5962/pyarrow-23.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:7a7d067c9a88faca655c71bcc30ee2782038d59c802d57950826a07f60d83c4c", size = 27577747, upload-time = "2026-01-18T16:18:02.413Z" }, - { url = "https://files.pythonhosted.org/packages/64/52/564a61b0b82d72bd68ec3aef1adda1e3eba776f89134b9ebcb5af4b13cb6/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:ce9486e0535a843cf85d990e2ec5820a47918235183a5c7b8b97ed7e92c2d47d", size = 34446038, upload-time = "2026-01-18T16:17:07.861Z" }, - { url = "https://files.pythonhosted.org/packages/cc/c9/232d4f9855fd1de0067c8a7808a363230d223c83aeee75e0fe6eab851ba9/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:075c29aeaa685fd1182992a9ed2499c66f084ee54eea47da3eb76e125e06064c", size = 35921142, upload-time = "2026-01-18T16:17:15.401Z" }, - { url = "https://files.pythonhosted.org/packages/96/f2/60af606a3748367b906bb82d41f0032e059f075444445d47e32a7ff1df62/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:799965a5379589510d888be3094c2296efd186a17ca1cef5b77703d4d5121f53", size = 44490374, upload-time = "2026-01-18T16:17:23.93Z" }, - { url = "https://files.pythonhosted.org/packages/ff/2d/7731543050a678ea3a413955a2d5d80d2a642f270aa57a3cb7d5a86e3f46/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ef7cac8fe6fccd8b9e7617bfac785b0371a7fe26af59463074e4882747145d40", size = 47527896, upload-time = "2026-01-18T16:17:33.393Z" }, - { url = "https://files.pythonhosted.org/packages/5a/90/f3342553b7ac9879413aed46500f1637296f3c8222107523a43a1c08b42a/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15a414f710dc927132dd67c361f78c194447479555af57317066ee5116b90e9e", size = 48210401, upload-time = "2026-01-18T16:17:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/f3/da/9862ade205ecc46c172b6ce5038a74b5151c7401e36255f15975a45878b2/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e0d2e6915eca7d786be6a77bf227fbc06d825a75b5b5fe9bcbef121dec32685", size = 50579677, upload-time = "2026-01-18T16:17:50.241Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4c/f11f371f5d4740a5dafc2e11c76bcf42d03dfdb2d68696da97de420b6963/pyarrow-23.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4b317ea6e800b5704e5e5929acb6e2dc13e9276b708ea97a39eb8b345aa2658b", size = 27631889, upload-time = "2026-01-18T16:17:56.55Z" }, - { url = "https://files.pythonhosted.org/packages/97/bb/15aec78bcf43a0c004067bd33eb5352836a29a49db8581fc56f2b6ca88b7/pyarrow-23.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:20b187ed9550d233a872074159f765f52f9d92973191cd4b93f293a19efbe377", size = 34213265, upload-time = "2026-01-18T16:18:07.904Z" }, - { url = "https://files.pythonhosted.org/packages/f6/6c/deb2c594bbba41c37c5d9aa82f510376998352aa69dfcb886cb4b18ad80f/pyarrow-23.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:18ec84e839b493c3886b9b5e06861962ab4adfaeb79b81c76afbd8d84c7d5fda", size = 35819211, upload-time = "2026-01-18T16:18:13.94Z" }, - { url = "https://files.pythonhosted.org/packages/e0/e5/ee82af693cb7b5b2b74f6524cdfede0e6ace779d7720ebca24d68b57c36b/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e438dd3f33894e34fd02b26bd12a32d30d006f5852315f611aa4add6c7fab4bc", size = 44502313, upload-time = "2026-01-18T16:18:20.367Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/95c61ad82236495f3c31987e85135926ba3ec7f3819296b70a68d8066b49/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a244279f240c81f135631be91146d7fa0e9e840e1dfed2aba8483eba25cd98e6", size = 47585886, upload-time = "2026-01-18T16:18:27.544Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6e/a72d901f305201802f016d015de1e05def7706fff68a1dedefef5dc7eff7/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c4692e83e42438dba512a570c6eaa42be2f8b6c0f492aea27dec54bdc495103a", size = 48207055, upload-time = "2026-01-18T16:18:35.425Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e5/5de029c537630ca18828db45c30e2a78da03675a70ac6c3528203c416fe3/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae7f30f898dfe44ea69654a35c93e8da4cef6606dc4c72394068fd95f8e9f54a", size = 50619812, upload-time = "2026-01-18T16:18:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/59/8d/2af846cd2412e67a087f5bda4a8e23dfd4ebd570f777db2e8686615dafc1/pyarrow-23.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:5b86bb649e4112fb0614294b7d0a175c7513738876b89655605ebb87c804f861", size = 28263851, upload-time = "2026-01-18T16:19:38.567Z" }, - { url = "https://files.pythonhosted.org/packages/7b/7f/caab863e587041156f6786c52e64151b7386742c8c27140f637176e9230e/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ebc017d765d71d80a3f8584ca0566b53e40464586585ac64176115baa0ada7d3", size = 34463240, upload-time = "2026-01-18T16:18:49.755Z" }, - { url = "https://files.pythonhosted.org/packages/c9/fa/3a5b8c86c958e83622b40865e11af0857c48ec763c11d472c87cd518283d/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:0800cc58a6d17d159df823f87ad66cefebf105b982493d4bad03ee7fab84b993", size = 35935712, upload-time = "2026-01-18T16:18:55.626Z" }, - { url = "https://files.pythonhosted.org/packages/c5/08/17a62078fc1a53decb34a9aa79cf9009efc74d63d2422e5ade9fed2f99e3/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3a7c68c722da9bb5b0f8c10e3eae71d9825a4b429b40b32709df5d1fa55beb3d", size = 44503523, upload-time = "2026-01-18T16:19:03.958Z" }, - { url = "https://files.pythonhosted.org/packages/cc/70/84d45c74341e798aae0323d33b7c39194e23b1abc439ceaf60a68a7a969a/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:bd5556c24622df90551063ea41f559b714aa63ca953db884cfb958559087a14e", size = 47542490, upload-time = "2026-01-18T16:19:11.208Z" }, - { url = "https://files.pythonhosted.org/packages/61/d9/d1274b0e6f19e235de17441e53224f4716574b2ca837022d55702f24d71d/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54810f6e6afc4ffee7c2e0051b61722fbea9a4961b46192dcfae8ea12fa09059", size = 48233605, upload-time = "2026-01-18T16:19:19.544Z" }, - { url = "https://files.pythonhosted.org/packages/39/07/e4e2d568cb57543d84482f61e510732820cddb0f47c4bb7df629abfed852/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:14de7d48052cf4b0ed174533eafa3cfe0711b8076ad70bede32cf59f744f0d7c", size = 50603979, upload-time = "2026-01-18T16:19:26.717Z" }, - { url = "https://files.pythonhosted.org/packages/72/9c/47693463894b610f8439b2e970b82ef81e9599c757bf2049365e40ff963c/pyarrow-23.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:427deac1f535830a744a4f04a6ac183a64fcac4341b3f618e693c41b7b98d2b0", size = 28338905, upload-time = "2026-01-18T16:19:32.93Z" }, +version = "23.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390, upload-time = "2026-02-16T10:08:08.654Z" }, + { url = "https://files.pythonhosted.org/packages/bc/8e/4be5617b4aaae0287f621ad31c6036e5f63118cfca0dc57d42121ff49b51/pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c", size = 35853761, upload-time = "2026-02-16T10:08:17.811Z" }, + { url = "https://files.pythonhosted.org/packages/2e/08/3e56a18819462210432ae37d10f5c8eed3828be1d6c751b6e6a2e93c286a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258", size = 44493116, upload-time = "2026-02-16T10:08:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/f8/82/c40b68001dbec8a3faa4c08cd8c200798ac732d2854537c5449dc859f55a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2", size = 47564532, upload-time = "2026-02-16T10:08:34.27Z" }, + { url = "https://files.pythonhosted.org/packages/20/bc/73f611989116b6f53347581b02177f9f620efdf3cd3f405d0e83cdf53a83/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5", size = 48183685, upload-time = "2026-02-16T10:08:42.889Z" }, + { url = "https://files.pythonhosted.org/packages/b0/cc/6c6b3ecdae2a8c3aced99956187e8302fc954cc2cca2a37cf2111dad16ce/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222", size = 50605582, upload-time = "2026-02-16T10:08:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/8d/94/d359e708672878d7638a04a0448edf7c707f9e5606cee11e15aaa5c7535a/pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d", size = 27521148, upload-time = "2026-02-16T10:08:58.077Z" }, + { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" }, + { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" }, + { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" }, + { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" }, + { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, + { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, + { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, + { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, + { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" }, + { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" }, + { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" }, + { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" }, + { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" }, + { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" }, + { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" }, + { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" }, + { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] [[package]] From bf9d9b44910a9134ded4f81d17f1983ba1027705 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:00:34 +0300 Subject: [PATCH 046/107] chore(deps): bump uuid from 1.23.1 to 1.23.4 in /python (#1936) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.23.1 to 1.23.4. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.23.1...v1.23.4) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.23.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index f4e556ce28..caa5189616 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -3929,7 +3929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4334,9 +4334,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "getrandom 0.4.2", "js-sys", From 2555985e4aba717159938c78dfd15395b09c5f0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:00:44 +0300 Subject: [PATCH 047/107] chore(deps): bump insta from 1.47.2 to 1.48.0 in /python (#1935) Bumps [insta](https://github.com/mitsuhiko/insta) from 1.47.2 to 1.48.0. - [Release notes](https://github.com/mitsuhiko/insta/releases) - [Changelog](https://github.com/mitsuhiko/insta/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/insta/compare/1.47.2...1.48.0) --- updated-dependencies: - dependency-name: insta dependency-version: 1.48.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index caa5189616..d9d9e262db 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -1847,7 +1847,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2411,9 +2411,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.47.2" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console", "once_cell", @@ -3495,7 +3495,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3775,7 +3775,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3850,7 +3850,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3932,7 +3932,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4536,7 +4536,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 2ed13c7b0be149eb8e00b2e6f508f0f523dccaf9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:00:55 +0300 Subject: [PATCH 048/107] chore(deps): bump pyo3-log from 0.13.3 to 0.13.4 in /python (#1934) Bumps [pyo3-log](https://github.com/vorner/pyo3-log) from 0.13.3 to 0.13.4. - [Changelog](https://github.com/vorner/pyo3-log/blob/main/CHANGELOG.md) - [Commits](https://github.com/vorner/pyo3-log/compare/v0.13.3...v0.13.4) --- updated-dependencies: - dependency-name: pyo3-log dependency-version: 0.13.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index d9d9e262db..8481b21c28 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -3181,9 +3181,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c2ec80932c5c3b2d4fbc578c9b56b2d4502098587edb8bef5b6bfcad43682e" +checksum = "f64083bd3a16a353d9d62335808e8e13d0552d2a2b83fdb084496192dcfa9fcd" dependencies = [ "arc-swap", "log", From 4bd3d0e0b9a7bbe5ec50a7a7d48b6c8a2f8b118f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:01:05 +0300 Subject: [PATCH 049/107] chore(deps): bump rand from 0.10.1 to 0.10.2 (#1933) Bumps [rand](https://github.com/rust-random/rand) from 0.10.1 to 0.10.2. - [Release notes](https://github.com/rust-random/rand/releases) - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.10.1...0.10.2) --- updated-dependencies: - dependency-name: rand dependency-version: 0.10.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c136dbe1f..fa3e259c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1034,7 +1034,7 @@ dependencies = [ "env_logger", "futures", "mimalloc", - "rand 0.10.1", + "rand 0.10.2", "serde", "serde_json", "structopt", @@ -1103,7 +1103,7 @@ dependencies = [ "parking_lot", "prost", "prost-types", - "rand 0.10.1", + "rand 0.10.2", "rustc_version", "serde", "tempfile", @@ -1200,7 +1200,7 @@ dependencies = [ "prometheus", "prost", "prost-types", - "rand 0.10.1", + "rand 0.10.2", "regex", "rstest", "serde", @@ -3298,7 +3298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" dependencies = [ "portable-atomic", - "rand 0.10.1", + "rand 0.10.2", "web-time", ] @@ -4874,7 +4874,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.1", + "rand 0.10.2", "reqwest 0.12.28", "ring", "rustls-pki-types", @@ -5678,9 +5678,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.2", @@ -6928,7 +6928,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", From 4365c19f73b88a8cbf28e10be57ef652a0478321 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:01:15 +0300 Subject: [PATCH 050/107] chore(deps): bump log from 0.4.29 to 0.4.33 in /python (#1932) Bumps [log](https://github.com/rust-lang/log) from 0.4.29 to 0.4.33. - [Release notes](https://github.com/rust-lang/log/releases) - [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/log/compare/0.4.29...0.4.33) --- updated-dependencies: - dependency-name: log dependency-version: 0.4.33 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 8481b21c28..c75d8a0571 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -2613,9 +2613,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" From 1684060902877a423894859d9da80b12a2884109 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:01:25 +0300 Subject: [PATCH 051/107] chore(deps): bump pyo3-build-config from 0.28.3 to 0.29.0 in /python (#1931) Bumps [pyo3-build-config](https://github.com/pyo3/pyo3) from 0.28.3 to 0.29.0. - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/compare/v0.28.3...v0.29.0) --- updated-dependencies: - dependency-name: pyo3-build-config dependency-version: 0.29.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 19 ++++++++++++++----- python/Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index c75d8a0571..a2e441b2c8 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -1704,7 +1704,7 @@ dependencies = [ "prost-types", "pyo3", "pyo3-async-runtimes", - "pyo3-build-config", + "pyo3-build-config 0.28.3", "pyo3-log", "serde_json", "tokio", @@ -3125,7 +3125,7 @@ dependencies = [ "datafusion-python", "log", "pyo3", - "pyo3-build-config", + "pyo3-build-config 0.29.0", "pyo3-log", "tokio", "tonic", @@ -3140,7 +3140,7 @@ dependencies = [ "libc", "once_cell", "portable-atomic", - "pyo3-build-config", + "pyo3-build-config 0.28.3", "pyo3-ffi", "pyo3-macros", ] @@ -3169,6 +3169,15 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "pyo3-build-config" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +dependencies = [ + "target-lexicon", +] + [[package]] name = "pyo3-ffi" version = "0.28.3" @@ -3176,7 +3185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", - "pyo3-build-config", + "pyo3-build-config 0.28.3", ] [[package]] @@ -3210,7 +3219,7 @@ checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", + "pyo3-build-config 0.28.3", "quote", "syn", ] diff --git a/python/Cargo.toml b/python/Cargo.toml index 5b0b14cbbf..47b3f9d39d 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -49,4 +49,4 @@ crate-type = ["cdylib"] name = "ballista" [build-dependencies] -pyo3-build-config = "0.28" +pyo3-build-config = "0.29" From 4b2a8d4a2f809889be4169e4e1e110ddf7cbdd27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:01:47 +0300 Subject: [PATCH 052/107] chore(deps): bump rand from 0.10.1 to 0.10.2 in /python (#1928) Bumps [rand](https://github.com/rust-random/rand) from 0.10.1 to 0.10.2. - [Release notes](https://github.com/rust-random/rand/releases) - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.10.1...0.10.2) --- updated-dependencies: - dependency-name: rand dependency-version: 0.10.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index a2e441b2c8..717a38cd06 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -520,7 +520,7 @@ dependencies = [ "parking_lot", "prost", "prost-types", - "rand 0.10.1", + "rand 0.10.2", "rustc_version", "serde", "tokio", @@ -580,7 +580,7 @@ dependencies = [ "parking_lot", "prost", "prost-types", - "rand 0.10.1", + "rand 0.10.2", "serde", "tokio", "tokio-stream", @@ -2806,7 +2806,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.1", + "rand 0.10.2", "reqwest", "ring", "rustls-pki-types", @@ -3331,9 +3331,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.2", From 36b661ad79a74e8bacf43fbe9b759899b6f8b301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:30:02 +0300 Subject: [PATCH 053/107] chore(deps): bump prost from 0.14.3 to 0.14.4 in /python (#1929) Bumps [prost](https://github.com/tokio-rs/prost) from 0.14.3 to 0.14.4. - [Release notes](https://github.com/tokio-rs/prost/releases) - [Changelog](https://github.com/tokio-rs/prost/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/prost/compare/v0.14.3...v0.14.4) --- updated-dependencies: - dependency-name: prost dependency-version: 0.14.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Martin Grigorov --- python/Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 717a38cd06..ca63ef385e 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -3030,9 +3030,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -3061,9 +3061,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools 0.14.0", From 551089d618f0f8f1934e8645423ed2be46007dbf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:23:34 +0300 Subject: [PATCH 054/107] chore(deps): bump github/codeql-action/init from 4.36.2 to 4.37.0 (#1974) * chore(deps): bump github/codeql-action/init from 4.36.2 to 4.37.0 Bumps [github/codeql-action/init](https://github.com/github/codeql-action) from 4.36.2 to 4.37.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/8aad20d150bbac5944a9f9d289da16a4b0d87c1e...99df26d4f13ea111d4ec1a7dddef6063f76b97e9) --- updated-dependencies: - dependency-name: github/codeql-action/init dependency-version: 4.37.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Sync the versions of init and analyze CodeQL actions --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Martin Grigorov --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 639690037e..444605088d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -48,11 +48,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 + uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4 + uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4 with: category: "/language:actions" From 8dad3d50daf7b716c7df6876db38dbf15b877f8e Mon Sep 17 00:00:00 2001 From: Bhargava Vadlamani <11091419+coderfender@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:16:55 -0700 Subject: [PATCH 055/107] docs: update companies using ballista (#1992) --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index b95dd5256e..fe0c204851 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,15 @@ but still there is a gap between DataFusion and Ballista which we want to bridge Refer to the [DataFusion SQL Reference](https://datafusion.apache.org/user-guide/sql/index.html) for more information on supported SQL. +## Who uses Ballista + +The following organizations use Ballista. To add yours, open a pull request. + +| Organization | | +| ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------- | +| Spice AI | [Spice AI](https://spice.ai/blog/apache-ballista-at-spice-ai) | +| Coralogix | [Coralogix](https://coralogix.com/) | + ## Contribution Guide Please see the [Contribution Guide](CONTRIBUTING.md) for information about contributing to Ballista. From 9d75f11488fc7c655a1a29898acdaaaf8b5d2e7e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 11 Jul 2026 07:01:29 -0600 Subject: [PATCH 056/107] chore: add script and CI check to sync vendored DataFusion protos (#1991) * chore: add script and CI check to sync vendored DataFusion protos The vendored `ballista/core/proto/datafusion.proto` and `datafusion_common.proto` are build-time stubs so protoc can resolve `ballista.proto`'s imports; no Rust is generated from them (build.rs maps their packages to the datafusion-proto crates via extern_path). They must stay compatible with the pinned DataFusion version but were previously synced by hand, which drifts. Add `dev/update_datafusion_proto.py` to copy the `.proto` files out of the crate source resolved by `cargo metadata --locked` (rewriting the one differing import), with a `--check` mode wired into CI via `ci/scripts/rust_proto_check.sh`. Releases that do not ship a `.proto` (datafusion-proto-common before v54) are skipped rather than failing. Also remind the datafusion version-bump script to re-run the sync. * chore: sync vendored DataFusion protos to 54.0.0 Run dev/update_datafusion_proto.py to bring the vendored proto stubs in line with the pinned datafusion-proto / datafusion-proto-common 54.0.0 crates. The crates were upgraded to 54 without re-syncing these files; no Rust is generated from them so the drift was benign, but the new CI check now enforces parity. --- .github/workflows/rust.yml | 15 ++ ballista/core/proto/datafusion.proto | 94 +++++++++++-- ballista/core/proto/datafusion_common.proto | 50 ++++++- ci/scripts/rust_proto_check.sh | 24 ++++ dev/update_datafusion_proto.py | 143 ++++++++++++++++++++ dev/update_datafusion_versions.py | 8 ++ 6 files changed, 324 insertions(+), 10 deletions(-) create mode 100755 ci/scripts/rust_proto_check.sh create mode 100755 dev/update_datafusion_proto.py diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cb0a3a48ef..9818290539 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -251,6 +251,21 @@ jobs: exit 1 fi + datafusion-proto-sync-check: + name: Check vendored DataFusion proto is in sync + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + with: + fetch-depth: 1 + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - name: Check vendored DataFusion proto matches pinned crate + run: ci/scripts/rust_proto_check.sh + # Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it # coverage: diff --git a/ballista/core/proto/datafusion.proto b/ballista/core/proto/datafusion.proto index fe81be0d4a..69c003acb8 100644 --- a/ballista/core/proto/datafusion.proto +++ b/ballista/core/proto/datafusion.proto @@ -62,6 +62,7 @@ message LogicalPlanNode { RecursiveQueryNode recursive_query = 31; CteWorkTableScanNode cte_work_table_scan = 32; DmlNode dml = 33; + EmptyTableScanNode empty_table_scan = 34; } } @@ -228,6 +229,7 @@ message AnalyzeNode { message ExplainNode { LogicalPlanNode input = 1; bool verbose = 2; + datafusion_common.ExplainFormat format = 3; } message AggregateNode { @@ -270,6 +272,25 @@ message CopyToNode { repeated string partition_by = 7; } +// Identifies a built-in file format supported by DataFusion. +// Used by DefaultLogicalExtensionCodec to serialize/deserialize +// FileFormatFactory instances (e.g. in CopyTo plans). +enum FileFormatKind { + FILE_FORMAT_KIND_UNSPECIFIED = 0; + FILE_FORMAT_KIND_CSV = 1; + FILE_FORMAT_KIND_JSON = 2; + FILE_FORMAT_KIND_PARQUET = 3; + FILE_FORMAT_KIND_ARROW = 4; + FILE_FORMAT_KIND_AVRO = 5; +} + +// Wraps a serialized FileFormatFactory with its format kind tag, +// so the decoder can dispatch to the correct format-specific codec. +message FileFormatProto { + FileFormatKind kind = 1; + bytes encoded_file_format = 2; +} + message DmlNode{ enum Type { UPDATE = 0; @@ -407,6 +428,8 @@ message LogicalExprNode { Unnest unnest = 35; + // Subquery expressions + ScalarSubqueryExprNode scalar_subquery_expr = 36; } } @@ -414,6 +437,15 @@ message Wildcard { TableReference qualifier = 1; } +message SubqueryNode { + LogicalPlanNode subquery = 1; + repeated LogicalExprNode outer_ref_columns = 2; +} + +message ScalarSubqueryExprNode { + SubqueryNode subquery = 1; +} + message PlaceholderNode { string id = 1; // We serialize the data type, metadata, and nullability separately to maintain @@ -594,11 +626,15 @@ message WhenThen { message CastNode { LogicalExprNode expr = 1; datafusion_common.ArrowType arrow_type = 2; + map metadata = 3; + optional bool nullable = 4; } message TryCastNode { LogicalExprNode expr = 1; datafusion_common.ArrowType arrow_type = 2; + map metadata = 3; + optional bool nullable = 4; } message SortExprNode { @@ -752,6 +788,7 @@ message PhysicalPlanNode { AsyncFuncExecNode async_func = 36; BufferExecNode buffer = 37; ArrowScanExecNode arrow_scan = 38; + ScalarSubqueryExecNode scalar_subquery = 39; } } @@ -853,11 +890,9 @@ message PhysicalExprNode { reserved 17; // Unique identifier for this expression to do deduplication during deserialization. - // When serializing, this is set to a unique identifier for each combination of - // expression, process and serialization run. - // When deserializing, if this ID has been seen before, the cached Arc is returned - // instead of creating a new one, enabling reconstruction of referential integrity - // across serde roundtrips. + // When serializing, this is set via `PhysicalExpr::expression_id`. When deserializing, + // this id is used by the `DeduplicatingProtoConverter` to preserve referential + // integrity across serde roundtrips for different expressions with the same id. optional uint64 expr_id = 30; oneof ExprType { @@ -897,9 +932,21 @@ message PhysicalExprNode { UnknownColumn unknown_column = 20; PhysicalHashExprNode hash_expr = 21; + + PhysicalScalarSubqueryExprNode scalar_subquery = 22; + + PhysicalDynamicFilterNode dynamic_filter = 23; } } +message PhysicalDynamicFilterNode { + repeated PhysicalExprNode children = 1; + repeated PhysicalExprNode remapped_children = 2; + uint64 generation = 3; + PhysicalExprNode inner_expr = 4; + bool is_complete = 5; +} + message PhysicalScalarUdfNode { string name = 1; repeated PhysicalExprNode args = 2; @@ -958,6 +1005,9 @@ message PhysicalBinaryExprNode { PhysicalExprNode l = 1; PhysicalExprNode r = 2; string op = 3; + // Linearized operands for chains of the same operator (e.g. a AND b AND c). + // When present, `l` and `r` are ignored and `operands` holds the flattened list. + repeated PhysicalExprNode operands = 4; } message PhysicalDateTimeIntervalExprNode { @@ -1018,9 +1068,6 @@ message PhysicalExtensionExprNode { message PhysicalHashExprNode { repeated PhysicalExprNode on_columns = 1; uint64 seed0 = 2; - uint64 seed1 = 3; - uint64 seed2 = 4; - uint64 seed3 = 5; string description = 6; } @@ -1141,6 +1188,8 @@ message HashJoinExecNode { JoinFilter filter = 8; repeated uint32 projection = 9; bool null_aware = 10; + // Optional dynamic filter expression for pushing down to the probe side. + PhysicalExprNode dynamic_filter = 11; } enum StreamPartitionMode { @@ -1179,6 +1228,10 @@ message AnalyzeExecNode { bool show_statistics = 2; PhysicalPlanNode input = 3; datafusion_common.Schema schema = 4; + // Optional metric category filter. + // Empty means "plan only". Absent (has_metric_categories=false) means "all". + bool has_metric_categories = 5; + repeated string metric_categories = 6; } message CrossJoinExecNode { @@ -1268,6 +1321,8 @@ message AggregateExecNode { repeated MaybeFilter filter_expr = 10; AggLimit limit = 11; bool has_grouping_set = 12; + // Optional dynamic filter expression for pushing down to the child. + PhysicalExprNode dynamic_filter = 13; } message GlobalLimitExecNode { @@ -1289,6 +1344,8 @@ message SortExecNode { // Maximum number of highest/lowest rows to fetch; negative means no limit int64 fetch = 3; bool preserve_partitioning = 4; + // Optional dynamic filter expression for TopK pushdown. + PhysicalExprNode dynamic_filter = 5; } message SortPreservingMergeExecNode { @@ -1330,6 +1387,7 @@ message RepartitionExecNode{ // uint64 unknown = 4; // } Partitioning partitioning = 5; + bool preserve_order = 6; } message Partitioning { @@ -1384,6 +1442,13 @@ message CteWorkTableScanNode { datafusion_common.Schema schema = 2; } +message EmptyTableScanNode { + TableReference table_name = 1; + datafusion_common.Schema schema = 2; + ProjectionColumns projection = 3; + repeated LogicalExprNode filters = 4; +} + enum GenerateSeriesName { GS_GENERATE_SERIES = 0; GS_RANGE = 1; @@ -1449,4 +1514,15 @@ message AsyncFuncExecNode { message BufferExecNode { PhysicalPlanNode input = 1; uint64 capacity = 2; -} \ No newline at end of file +} + +message ScalarSubqueryExecNode { + PhysicalPlanNode input = 1; + repeated PhysicalPlanNode subqueries = 2; +} + +message PhysicalScalarSubqueryExprNode { + datafusion_common.ArrowType data_type = 1; + bool nullable = 2; + uint32 index = 3; +} diff --git a/ballista/core/proto/datafusion_common.proto b/ballista/core/proto/datafusion_common.proto index 5848455cba..dd3588d134 100644 --- a/ballista/core/proto/datafusion_common.proto +++ b/ballista/core/proto/datafusion_common.proto @@ -199,7 +199,7 @@ message Union{ repeated int32 type_ids = 3; } -// Used for List/FixedSizeList/LargeList/Struct/Map +// Used for List/FixedSizeList/LargeList/ListView/LargeListView/Struct/Map message ScalarNestedValue { message Dictionary { bytes ipc_message = 1; @@ -306,6 +306,8 @@ message ScalarValue{ ScalarNestedValue large_list_value = 16; ScalarNestedValue list_value = 17; ScalarNestedValue fixed_size_list_value = 18; + ScalarNestedValue list_view_value = 46; + ScalarNestedValue large_list_view_value = 47; ScalarNestedValue struct_value = 32; ScalarNestedValue map_value = 41; @@ -398,6 +400,8 @@ message ArrowType{ List LIST = 25; List LARGE_LIST = 26; FixedSizeList FIXED_SIZE_LIST = 27; + List LIST_VIEW = 43; + List LARGE_LIST_VIEW = 44; Struct STRUCT = 28; Union UNION = 29; Dictionary DICTIONARY = 30; @@ -430,6 +434,13 @@ message JsonWriterOptions { } +enum CsvQuoteStyle { + NECESSARY = 0; + ALWAYS = 1; + NON_NUMERIC = 2; + NEVER = 3; +} + message CsvWriterOptions { // Compression type CompressionTypeVariant compression = 1; @@ -453,6 +464,12 @@ message CsvWriterOptions { string escape = 10; // Optional flag whether to double quotes, instead of escaping. Defaults to `true` bool double_quote = 11; + // Quote style for CSV writing + CsvQuoteStyle quote_style = 12; + // Whether to ignore leading whitespace in string values + bool ignore_leading_whitespace = 13; + // Whether to ignore trailing whitespace in string values + bool ignore_trailing_whitespace = 14; } // Options controlling CSV format @@ -476,6 +493,12 @@ message CsvOptions { bytes terminator = 17; // Optional terminator character as a byte bytes truncated_rows = 18; // Indicates if truncated rows are allowed optional uint32 compression_level = 19; // Optional compression level + // Quote style for CSV writing + CsvQuoteStyle quote_style = 20; + // Whether to ignore leading whitespace in string values + bytes ignore_leading_whitespace = 21; + // Whether to ignore trailing whitespace in string values + bytes ignore_trailing_whitespace = 22; } // Options controlling CSV format @@ -603,6 +626,24 @@ message ParquetOptions { oneof max_predicate_cache_size_opt { uint64 max_predicate_cache_size = 33; } + + ParquetCdcOptions content_defined_chunking = 35; + + // Optional timezone applied to INT96-coerced timestamps when `coerce_int96` + // is set. When `Some`, INT96 columns coerce to + // `Timestamp(, Some())` instead of the default + // `Timestamp(, None)`. No effect when `coerce_int96` is unset. + oneof coerce_int96_tz_opt { + string coerce_int96_tz = 36; + } +} + +// Content-defined chunking (CDC) options for writing parquet files. +message ParquetCdcOptions { + bool enabled = 1; + uint64 min_chunk_size = 2; + uint64 max_chunk_size = 3; + int32 norm_level = 4; } enum JoinSide { @@ -637,4 +678,11 @@ message ColumnStats { Precision null_count = 3; Precision distinct_count = 4; Precision byte_size = 6; +} + +enum ExplainFormat { + EXPLAIN_FORMAT_INDENT = 0; + EXPLAIN_FORMAT_TREE = 1; + EXPLAIN_FORMAT_PGJSON = 2; + EXPLAIN_FORMAT_GRAPHVIZ = 3; } \ No newline at end of file diff --git a/ci/scripts/rust_proto_check.sh b/ci/scripts/rust_proto_check.sh new file mode 100755 index 0000000000..4347126871 --- /dev/null +++ b/ci/scripts/rust_proto_check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fail if the vendored DataFusion proto files drift from the datafusion-proto / +# datafusion-proto-common versions pinned in Cargo.lock. See the script header of +# dev/update_datafusion_proto.py for why these files are vendored. +set -ex +python3 dev/update_datafusion_proto.py --check diff --git a/dev/update_datafusion_proto.py b/dev/update_datafusion_proto.py new file mode 100755 index 0000000000..bc627b5b8c --- /dev/null +++ b/dev/update_datafusion_proto.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Sync the vendored DataFusion proto files with the version pinned in Cargo.lock. +# +# `ballista/core/proto/ballista.proto` imports `datafusion.proto` and +# `datafusion_common.proto`. Those two files are NOT compiled into Rust here -- +# `ballista/core/build.rs` maps their packages to the real `datafusion-proto` / +# `datafusion-proto-common` crates via `extern_path`. They exist only so that +# `protoc` can resolve `ballista.proto`'s imports at build time. Because of that +# they must stay compatible with the DataFusion version Ballista depends on. +# +# The `datafusion-proto` / `datafusion-proto-common` crates ship their `.proto` +# files, but expose no `links`/`DEP_*` path for a build script to consume, so we +# vendor a copy. This script keeps that copy honest: it reads the exact crate +# source resolved by `cargo metadata` and copies the `.proto` files in, rewriting +# the one import path that differs from our flat layout. +# +# Usage: +# dev/update_datafusion_proto.py # update the vendored files in place +# dev/update_datafusion_proto.py --check # fail if vendored files are stale (CI) + +import argparse +import difflib +import json +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent.absolute() +VENDOR_DIR = REPO_ROOT / "ballista" / "core" / "proto" + +# vendored file name -> (crate name, file name within the crate's `proto/` dir) +FILES = { + "datafusion.proto": ("datafusion-proto", "datafusion.proto"), + "datafusion_common.proto": ("datafusion-proto-common", "datafusion_common.proto"), +} + + +def crate_source_dirs(): + """Map crate name -> source directory, as resolved in Cargo.lock.""" + out = subprocess.run( + ["cargo", "metadata", "--format-version", "1", "--locked"], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + metadata = json.loads(out.stdout) + dirs = {} + for pkg in metadata["packages"]: + if pkg["name"] in ("datafusion-proto", "datafusion-proto-common"): + # manifest_path points at the crate's Cargo.toml; protos sit alongside it. + dirs[pkg["name"]] = (Path(pkg["manifest_path"]).parent, pkg["version"]) + missing = {"datafusion-proto", "datafusion-proto-common"} - dirs.keys() + if missing: + sys.exit(f"could not resolve crate(s) from cargo metadata: {sorted(missing)}") + return dirs + + +def rewrite(content: str) -> str: + """Rewrite crate-relative imports to our flat vendored layout. + + Upstream imports the common proto by its repo path, e.g. + `import "datafusion/proto-common/proto/datafusion_common.proto";` + but we vendor it flat next to `datafusion.proto`. + """ + return re.sub( + r'import\s+"[^"]*datafusion_common\.proto";', + 'import "datafusion_common.proto";', + content, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="verify the vendored files are up to date; do not write. Exits 1 on drift.", + ) + args = parser.parse_args() + + dirs = crate_source_dirs() + stale = [] + for vendor_name, (crate, src_name) in FILES.items(): + crate_dir, version = dirs[crate] + src = crate_dir / "proto" / src_name + if not src.exists(): + # Some releases don't publish the `.proto` file (e.g. datafusion-proto-common + # only started shipping it in v54). Nothing to sync against, so leave the + # vendored copy as-is rather than failing. + print( + f"note: {crate}@{version} does not ship proto/{src_name}; " + f"leaving vendored {vendor_name} unchanged", + file=sys.stderr, + ) + continue + expected = rewrite(src.read_text()) + dest = VENDOR_DIR / vendor_name + + if args.check: + current = dest.read_text() if dest.exists() else "" + if current != expected: + stale.append(vendor_name) + diff = difflib.unified_diff( + current.splitlines(keepends=True), + expected.splitlines(keepends=True), + fromfile=f"vendored/{vendor_name}", + tofile=f"{crate}@{version}/proto/{src_name}", + ) + sys.stderr.writelines(diff) + else: + dest.write_text(expected) + print(f"updated {dest.relative_to(REPO_ROOT)} from {crate}@{version}") + + if args.check and stale: + sys.exit( + "\nVendored DataFusion proto file(s) out of date: " + + ", ".join(stale) + + "\nRun `dev/update_datafusion_proto.py` and commit the result." + ) + + +if __name__ == "__main__": + main() diff --git a/dev/update_datafusion_versions.py b/dev/update_datafusion_versions.py index aed2cee3a2..d21096ea13 100755 --- a/dev/update_datafusion_versions.py +++ b/dev/update_datafusion_versions.py @@ -133,6 +133,14 @@ def main(): update_docs("README.md", new_version) update_pyproject_toml(os.path.join(repo_root, "python", "pyproject.toml"), new_version) + print( + '\nNext steps:\n' + ' 1. Run `cargo update` (or `cargo build`) so Cargo.lock resolves the new ' + 'datafusion crates.\n' + ' 2. Run `dev/update_datafusion_proto.py` to re-sync the vendored ' + 'DataFusion proto files, then commit the result.' + ) + if __name__ == "__main__": main() From 1cd1affc40e1cb9b2f84e0397ff4f1f2003947c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Sat, 11 Jul 2026 15:02:13 +0200 Subject: [PATCH 057/107] feat: improve state creation (#1963) --- ballista/scheduler/src/state/aqe/planner.rs | 47 +++++++++------------ 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/planner.rs b/ballista/scheduler/src/state/aqe/planner.rs index 4981912271..67d303761a 100644 --- a/ballista/scheduler/src/state/aqe/planner.rs +++ b/ballista/scheduler/src/state/aqe/planner.rs @@ -30,14 +30,14 @@ use ballista_core::serde::scheduler::PartitionLocation; use datafusion::common; use datafusion::common::{HashMap, exec_err}; use datafusion::error::DataFusionError; +#[cfg(test)] +use datafusion::execution::config::SessionConfig; use datafusion::execution::context::SessionContext; -use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::logical_expr::LogicalPlan; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable}; use datafusion::physical_planner::DefaultPhysicalPlanner; -use datafusion::prelude::SessionConfig; use log::debug; use std::collections::HashSet; use std::fmt::{Debug, Formatter}; @@ -86,9 +86,7 @@ impl AdaptivePlanner { /// Creates a new `AdaptivePlanner` with the specified physical optimizer rules. /// /// # Arguments: - /// - /// * `session_config` - The session configuration for the job. - /// * `runtime_env` - runtime environment + /// * `state_builder` - Session state builder, /// * `plan` - The physical execution plan for the job. /// * `job_name` - The name of the job. /// * `physical_optimizer_rules` - A list of physical optimizer rules to apply. @@ -96,17 +94,13 @@ impl AdaptivePlanner { /// # Returns /// A new instance of `AdaptivePlanner` or an error if the initialization fails. pub fn try_new_with_optimizers( - session_config: &SessionConfig, - runtime_env: Arc, + state_builder: SessionStateBuilder, plan: Arc, job_name: String, physical_optimizer_rules: Vec, ) -> common::Result { - let session_state = Self::create_session_state( - session_config, - runtime_env, - physical_optimizer_rules, - ); + let session_state = + Self::create_session_state(state_builder, physical_optimizer_rules); let planner = DefaultPhysicalPlanner::default(); let plan = planner.optimize_physical_plan(plan, &session_state, |_, _| {})?; @@ -137,9 +131,10 @@ impl AdaptivePlanner { job_name: String, ) -> common::Result { let plan_id_generator = Arc::new(AtomicUsize::new(0)); + let state_builder = SessionStateBuilder::new_with_default_features() + .with_config(session_config.clone()); Self::try_new_with_optimizers( - session_config, - RuntimeEnv::default().into(), + state_builder, plan, job_name, Self::default_optimizers(plan_id_generator), @@ -166,14 +161,14 @@ impl AdaptivePlanner { // running standard set of optimizers, which will // after each stage. let plan_id_generator = Arc::new(AtomicUsize::new(0)); - let runtime_env = ctx.runtime_env(); - let plan_preparation_stage = Self::create_session_state( - ctx.state().config(), - ctx.runtime_env(), + + let plan_preparation_state_builder = SessionStateBuilder::from(ctx.state()); + let plan_preparation_state = Self::create_session_state( + plan_preparation_state_builder, Self::plan_preparation_optimizers(plan_id_generator.clone()), ); - let plan = plan_preparation_stage + let plan = plan_preparation_state .create_physical_plan(logical_plan) .await?; @@ -184,9 +179,9 @@ impl AdaptivePlanner { .await .map_err(|e| DataFusionError::Execution(e.to_string()))?; + let state_builder = SessionStateBuilder::from(ctx.state()); Self::try_new_with_optimizers( - ctx.state().config(), - runtime_env, + state_builder, plan, job_name, Self::default_optimizers(plan_id_generator), @@ -551,22 +546,20 @@ impl AdaptivePlanner { /// Creates a session state with the given configuration and optimizer rules. /// /// # Arguments - /// * `session_config` - The session configuration. + /// * `session_builder` - The session builder. /// * `physical_optimizers` - A list of physical optimizer rules. /// /// # Returns /// A new `SessionState` instance. fn create_session_state( - session_config: &SessionConfig, - runtime_env: Arc, + builder: SessionStateBuilder, physical_optimizers: Vec, ) -> SessionState { - SessionStateBuilder::new_with_default_features() + builder .with_physical_optimizer_rules(physical_optimizers) - .with_config(session_config.clone()) - .with_runtime_env(runtime_env) .build() } + /// Recursively finds runnable exchanges in the execution plan. /// /// # Arguments From ac41a22245b563ec7bfba53e2d4b0783aa5c5318 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 11 Jul 2026 07:48:02 -0600 Subject: [PATCH 058/107] docs: link AQE limitations to tracking issues and add AQE contributor guide section (#1990) --- .../source/contributors-guide/architecture.md | 62 +++++++++++++++++++ docs/source/user-guide/tuning-guide.md | 12 ++-- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/docs/source/contributors-guide/architecture.md b/docs/source/contributors-guide/architecture.md index b25321d040..8a7ff10636 100644 --- a/docs/source/contributors-guide/architecture.md +++ b/docs/source/contributors-guide/architecture.md @@ -199,3 +199,65 @@ and [ShuffleReaderExec] operators. [shufflewriterexec]: https://github.com/apache/datafusion-ballista/blob/main/ballista/core/src/execution_plans/shuffle_writer.rs [shufflereaderexec]: https://github.com/apache/datafusion-ballista/blob/main/ballista/core/src/execution_plans/shuffle_reader.rs + +## Adaptive Query Execution (AQE) + +The scheduling described above is _static_: the full set of query stages is computed once, at job +submission time, from the plan-time cost estimates. Adaptive Query Execution (AQE) is an experimental +alternative in which the stage DAG is built _incrementally_, re-optimizing the remaining plan using the +exact row counts and byte sizes observed as each shuffle stage completes. It is disabled by default and +enabled with the `ballista.planner.adaptive.enabled` configuration setting. + +### Static vs adaptive execution graphs + +Job scheduling revolves around an `ExecutionGraph`, which translates a physical plan into a set of stages. +There are two implementations, selected by configuration when a job is submitted: + +- **`StaticExecutionGraph`** — the default. All stages are planned up front by the `DistributedPlanner`, + which returns a static list of stages. +- **`AdaptiveExecutionGraph`** — the adaptive counterpart. It is driven by the `AdaptivePlanner`, which runs + a set of pluggable physical optimizer rules after each stage completes and returns only the stages that are + currently runnable. Because decisions are deferred, stages come back already resolved, so the + `UnResolved` stage state used by the static path is not needed here. + +Keeping the two graphs side by side lets the adaptive implementation mature without destabilizing the static +path. + +### How re-optimization works + +As each shuffle stage finishes, its exact output statistics are attached to the remaining plan and the +optimizer rules are re-run. This is the key difference from DataFusion's normal single-shot physical +optimization, where each rule runs once per plan. Running the full rule set repeatedly means the rules must be +**idempotent** — otherwise they would keep inserting redundant exec nodes on each pass. + +Two adaptive optimizations are implemented today: + +- **Join reordering** — runtime row counts are used so the smaller side drives the join. +- **Empty stage elimination** — when a completed stage produces zero rows, its downstream exchange is + replaced with an empty execution node and emptiness is propagated up the plan so dependent stages are + skipped entirely. + +### Code layout + +The adaptive scheduler code lives under [`ballista/scheduler/src/state/aqe`], with the +`AdaptiveExecutionGraph` in `mod.rs`, the planner in `planner.rs`, and the optimizer rules in +`optimizer_rule.rs`. + +### Current limitations + +The adaptive path currently covers the happy path only. Known gaps, each with a tracking issue: + +- Executor failure handling on the AQE path ([#1986]) +- Dynamic coalescing of shuffle partitions ([#1987]) +- Switching from hash join to sort-merge join based on runtime statistics ([#1988]) +- Switching from streaming aggregation to hash aggregation based on runtime statistics ([#1989]) + +Design and progress are tracked in the AQE epic ([#1359]). See also the +[AQE section of the tuning guide](../user-guide/tuning-guide.md) for the user-facing description. + +[`ballista/scheduler/src/state/aqe`]: https://github.com/apache/datafusion-ballista/tree/main/ballista/scheduler/src/state/aqe +[#1359]: https://github.com/apache/datafusion-ballista/issues/1359 +[#1986]: https://github.com/apache/datafusion-ballista/issues/1986 +[#1987]: https://github.com/apache/datafusion-ballista/issues/1987 +[#1988]: https://github.com/apache/datafusion-ballista/issues/1988 +[#1989]: https://github.com/apache/datafusion-ballista/issues/1989 diff --git a/docs/source/user-guide/tuning-guide.md b/docs/source/user-guide/tuning-guide.md index 7b6f38130f..b7906b166c 100644 --- a/docs/source/user-guide/tuning-guide.md +++ b/docs/source/user-guide/tuning-guide.md @@ -218,14 +218,14 @@ implemented: The implementation covers the happy path only. The following are known to be missing or incomplete: -- Executor failure handling on the AQE path -- Dynamic coalescing of shuffle partitions -- Switching from hash join to sort-merge join based on runtime statistics -- Switching from streaming aggregation to hash aggregation based on runtime statistics +- Executor failure handling on the AQE path ([#1986](https://github.com/apache/datafusion-ballista/issues/1986)) +- Dynamic coalescing of shuffle partitions ([#1987](https://github.com/apache/datafusion-ballista/issues/1987)) +- Switching from hash join to sort-merge join based on runtime statistics ([#1988](https://github.com/apache/datafusion-ballista/issues/1988)) +- Switching from streaming aggregation to hash aggregation based on runtime statistics ([#1989](https://github.com/apache/datafusion-ballista/issues/1989)) Until these gaps are closed, AQE should be used for testing and experimentation -rather than production workloads. See [issue #387](https://github.com/apache/datafusion-ballista/issues/387) -for the tracking issue and ongoing work. +rather than production workloads. See [issue #1359](https://github.com/apache/datafusion-ballista/issues/1359) +for the tracking epic and ongoing work. ## Push-based vs Pull-based Task Scheduling From efaeda5ba689e14dc2b7368faf5438450dc1cfa4 Mon Sep 17 00:00:00 2001 From: jgrim Date: Sat, 11 Jul 2026 15:51:18 +0200 Subject: [PATCH 059/107] feat(tui + rest): surface failed tasks (#1949) * feat(rest): emit tasks with failure reason for Failed stages * feat(tui): show task failure reason in the stage task view * address review comments --- ballista-cli/src/tui/domain/jobs/stages.rs | 14 +++- .../src/tui/ui/main/jobs/stage_tasks_popup.rs | 22 +++--- ballista/scheduler/src/api/handlers.rs | 73 ++++++++++++++++++- 3 files changed, 91 insertions(+), 18 deletions(-) diff --git a/ballista-cli/src/tui/domain/jobs/stages.rs b/ballista-cli/src/tui/domain/jobs/stages.rs index 3297aca433..5b81d04109 100644 --- a/ballista-cli/src/tui/domain/jobs/stages.rs +++ b/ballista-cli/src/tui/domain/jobs/stages.rs @@ -43,11 +43,19 @@ pub struct JobStageResponse { pub tasks: Vec>, } +// TaskStatus +#[derive(Deserialize, Clone, Debug)] +pub enum StageTaskStatus { + Running, + Successful, + Failed { reason: String }, +} + // TaskSummary #[derive(Deserialize, Clone, Debug)] pub struct StageTaskResponse { pub id: usize, - pub status: String, + pub status: StageTaskStatus, pub partition_id: u32, pub input_rows: usize, pub output_rows: usize, @@ -364,7 +372,7 @@ impl StagesGraph { mod tests { use super::{ JobStageResponse, JobStagesPopup, JobStagesResponse, StageTaskResponse, - StagesGraph, TaskPercentiles, + StageTaskStatus, StagesGraph, TaskPercentiles, }; fn make_percentiles() -> TaskPercentiles { @@ -608,7 +616,7 @@ mod tests { fn make_task(id: usize) -> StageTaskResponse { StageTaskResponse { id, - status: "Completed".to_string(), + status: StageTaskStatus::Successful, partition_id: id as u32, input_rows: 0, output_rows: 0, diff --git a/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs b/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs index 76f9d769c1..8c89a699bd 100644 --- a/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs +++ b/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs @@ -16,7 +16,7 @@ // under the License. use crate::tui::app::App; -use crate::tui::domain::jobs::stages::StageTaskResponse; +use crate::tui::domain::jobs::stages::{StageTaskResponse, StageTaskStatus}; use crate::tui::ui::components::clear_area::clear_area; use crate::tui::ui::vertical_scrollbar; use ratatui::Frame; @@ -101,21 +101,19 @@ fn build_stage_task_row(i: usize, task: &StageTaskResponse, app: &App) -> Row<'s app.theme.row_odd }; - let status_style = match task.status.as_str() { - "Running" => app.theme.status_running, - "Queued" => app.theme.status_queued, - "Successful" | "Completed" => app.theme.status_completed, - "Failed" => app.theme.status_failed, - _ => app.theme.status_unknown, + let status_text = match &task.status { + StageTaskStatus::Running => Text::from("Running").style(app.theme.status_running), + StageTaskStatus::Successful => { + Text::from("Successful").style(app.theme.status_completed) + } + StageTaskStatus::Failed { reason } => { + Text::from(reason.clone()).style(app.theme.status_failed) + } }; Row::new(vec![ Cell::from(Text::from(task.id.to_string()).right_aligned()), - Cell::from( - Text::from(task.status.clone()) - .style(status_style) - .centered(), - ), + Cell::from(status_text.centered()), Cell::from(Text::from(app.format_count(task.input_rows)).right_aligned()), Cell::from(Text::from(app.format_count(task.output_rows)).right_aligned()), Cell::from(Text::from(task.partition_id.to_string()).right_aligned()), diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index ff886a69f8..8da872253a 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -22,9 +22,12 @@ use axum::{ extract::{Path, State}, response::{IntoResponse, Response}, }; +use ballista_core::serde::protobuf::failed_task::FailedReason::{ + ExecutionError, ExecutorLost, FetchPartitionError, IoError, ResultLost, TaskKilled, +}; use ballista_core::serde::protobuf::job_status::Status; use ballista_core::serde::protobuf::{ - ExecutorMetric, executor_metric::Metric, task_status, + ExecutorMetric, FailedTask, executor_metric::Metric, task_status, }; use ballista_core::serde::scheduler::{ ExecutorOperatingSystemSpecification, ExecutorSpecification, @@ -165,14 +168,17 @@ pub struct TaskSummary { pub enum TaskStatus { Running, Successful, - Failed, + Failed { reason: String, error: String }, } impl From<&task_status::Status> for TaskStatus { fn from(value: &task_status::Status) -> Self { match value { task_status::Status::Running(_) => TaskStatus::Running, - task_status::Status::Failed(_) => TaskStatus::Failed, + task_status::Status::Failed(failed) => TaskStatus::Failed { + reason: failed_reason(failed), + error: failed.error.clone(), + }, task_status::Status::Successful(_) => TaskStatus::Successful, } } @@ -637,6 +643,54 @@ pub async fn get_query_stages< }) .collect(); } + ExecutionStage::Failed(failed_stage) => { + let metrics = failed_stage.stage_metrics.as_deref().unwrap_or(&[]); + summary.stage_plan = Some(match plan_format { + PlanFormat::Default => displayable(failed_stage.plan.as_ref()).indent(false).to_string(), + PlanFormat::Tree => displayable(failed_stage.plan.as_ref()).tree_render().to_string(), + PlanFormat::Metrics => format_stage_metrics(failed_stage.plan.as_ref(), metrics), + }); + summary.input_rows = get_combined_count(metrics, "input_rows"); + summary.output_rows = get_combined_count(metrics, "output_rows"); + summary.elapsed_compute = get_finished_stage_time( + &failed_stage + .task_infos + .iter() + .flatten() + .cloned() + .collect::>(), + ); + + summary.tasks = failed_stage + .task_infos + .iter() + .enumerate() + .map(|(partition_id, task_info)| { + task_info.as_ref().map(|info| { + let (input_rows, output_rows) = + get_partition_counts(metrics, partition_id); + + let start_exec_time = info.start_exec_time as u64; + let end_exec_time = info.end_exec_time as u64; + let task_status: TaskStatus = (&info.task_status).into(); + + TaskSummary { + id: info.task_id, + partition_id: partition_id as u32, + scheduled_time: info.scheduled_time as u64, + launch_time: info.launch_time as u64, + start_exec_time, + end_exec_time, + exec_duration: end_exec_time.saturating_sub(start_exec_time), + finish_time: info.finish_time as u64, + input_rows, + output_rows, + status: task_status, + } + }) + }) + .collect(); + } _ => {} } summary.task_duration_percentiles = task_duration_percentiles(&summary.tasks); @@ -761,6 +815,19 @@ fn get_running_stage_time( } } +fn failed_reason(failed: &FailedTask) -> String { + match &failed.failed_reason { + Some(ExecutionError(_)) => "ExecutionError", + Some(FetchPartitionError(_)) => "FetchPartitionError", + Some(IoError(_)) => "IoError", + Some(ExecutorLost(_)) => "ExecutorLost", + Some(ResultLost(_)) => "ResultLost", + Some(TaskKilled(_)) => "TaskKilled", + None => "Failed", + } + .to_string() +} + fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option { let min_start = task_infos .iter() From b491d3cf7d50830d380e718a891335a1af877d20 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 11 Jul 2026 08:07:47 -0600 Subject: [PATCH 060/107] Add shuffle-read fetch metrics and surface per-operator metrics in plan display (#1968) * feat: add ShuffleReadMetrics with local/remote split and local read timing * feat: record remote shuffle fetch time, bytes, requests, retries and governor permit wait * feat: render inner-operator metrics in ShuffleWriterExec plan display * test: harden shuffle-writer display test and apply fmt * test: cover remote fetch metrics on error path; refine fetch_requests semantics * test: update EXPLAIN ANALYZE golden for new shuffle-read metrics --- ballista/client/tests/context_checks.rs | 2 +- .../src/execution_plans/shuffle_reader.rs | 319 ++++++++++++++++-- .../src/execution_plans/shuffle_writer.rs | 37 +- 3 files changed, 326 insertions(+), 32 deletions(-) diff --git a/ballista/client/tests/context_checks.rs b/ballista/client/tests/context_checks.rs index 272abc94bd..37f242f94d 100644 --- a/ballista/client/tests/context_checks.rs +++ b/ballista/client/tests/context_checks.rs @@ -1203,7 +1203,7 @@ mod supported { "| | ShuffleWriterExec: partitioning: None, metrics=[output_rows=..., input_rows=..., repart_time=..., write_time=...] |", "| | ProjectionExec: expr=[count(Int64(1))@1 as count(*), id@0 as id], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=..., expr_1_eval_time=...] |", "| | AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[count(Int64(1))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., spill_count=..., spilled_bytes=..., spilled_rows=..., peak_mem_used=..., aggregate_arguments_time=..., aggregation_time=..., emitting_time=..., time_calculating_group_ids=...] |", - "| | ShuffleReaderExec: upstream_stage: 1, partitioning: Hash([id@0], 16), metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=...] |", + "| | ShuffleReaderExec: upstream_stage: 1, partitioning: Hash([id@0], 16), metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., decoded_bytes=..., fetch_requests=..., fetch_retries=..., local_partitions=..., remote_partitions=..., fetch_time=..., local_read_time=..., permit_wait_time=...] |", "+-------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", ]; diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index 89ae76f73f..5513283f7a 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -34,7 +34,7 @@ use datafusion::error::{DataFusionError, Result}; use datafusion::execution::context::TaskContext; use datafusion::physical_plan::coalesce::{LimitedBatchCoalescer, PushBatchStatus}; use datafusion::physical_plan::metrics::{ - BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, + self, BaselineMetrics, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet, }; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ @@ -398,11 +398,14 @@ impl ExecutionPlan for ShuffleReaderExec { "ShuffleReader work dir should have been set by executor".to_owned(), ))?; + let read_metrics = ShuffleReadMetrics::new(partition, &self.metrics); + let response_receiver = send_fetch_partitions( work_dir, partition_locations, config, self.client_pool.clone(), + read_metrics, ); let input_stream = Box::pin(RecordBatchStreamAdapter::new( @@ -703,11 +706,65 @@ fn local_remote_read_split( } } +/// Fetch-side metrics for `ShuffleReaderExec`, recorded per output partition. +/// +/// NOTE: the reader's `BaselineMetrics::elapsed_compute` measures poll time of +/// the consuming stream, which overlaps with background fetching. `fetch_time` +/// here is *additive* wall-time spent inside the fetch tasks — do not sum the +/// two. `decoded_bytes` is the in-memory Arrow footprint of fetched batches, +/// not compressed wire bytes. `fetch_time` and `permit_wait_time` are each +/// summed across every concurrent remote fetch task, so their totals can +/// exceed the operator's wall-clock elapsed time — read them as aggregate +/// cost, not wall-clock. +#[derive(Debug, Clone)] +struct ShuffleReadMetrics { + /// Wall-time fetching remote partitions (Arrow-Flight fetch + buffering). + fetch_time: metrics::Time, + /// Wall-time opening node-local shuffle files. + local_read_time: metrics::Time, + /// Wall-time blocked acquiring the reduce-side in-flight governor permits + /// (request + per-address + byte semaphores, #1951). + permit_wait_time: metrics::Time, + /// Decoded (in-memory Arrow) bytes of fetched remote partitions. + decoded_bytes: metrics::Count, + /// Number of remote fetch attempts issued, including retries. + fetch_requests: metrics::Count, + /// Extra fetch attempts taken by the reduce-side retry loop. + fetch_retries: metrics::Count, + /// Partitions served from node-local shuffle files. + local_partitions: metrics::Count, + /// Partitions fetched from a remote executor. + remote_partitions: metrics::Count, +} + +impl ShuffleReadMetrics { + fn new(partition: usize, metrics: &ExecutionPlanMetricsSet) -> Self { + Self { + fetch_time: MetricBuilder::new(metrics).subset_time("fetch_time", partition), + local_read_time: MetricBuilder::new(metrics) + .subset_time("local_read_time", partition), + permit_wait_time: MetricBuilder::new(metrics) + .subset_time("permit_wait_time", partition), + decoded_bytes: MetricBuilder::new(metrics) + .counter("decoded_bytes", partition), + fetch_requests: MetricBuilder::new(metrics) + .counter("fetch_requests", partition), + fetch_retries: MetricBuilder::new(metrics) + .counter("fetch_retries", partition), + local_partitions: MetricBuilder::new(metrics) + .counter("local_partitions", partition), + remote_partitions: MetricBuilder::new(metrics) + .counter("remote_partitions", partition), + } + } +} + fn send_fetch_partitions( work_dir: &str, partition_locations: Vec, config: &SessionConfig, client_pool: Option>, + read_metrics: ShuffleReadMetrics, ) -> AbortableReceiverStream { let ballista_config = config.ballista_config(); let max_reqs = config.ballista_shuffle_reader_maximum_concurrent_requests(); @@ -747,13 +804,20 @@ fn send_fetch_partitions( remote_locations.len() ); + read_metrics.local_partitions.add(local_locations.len()); + read_metrics.remote_partitions.add(remote_locations.len()); + // keep local shuffle files reading in serial order for memory control. let response_sender_c = response_sender.clone(); let work_dir = work_dir.to_string(); + let local_read_time = read_metrics.local_read_time.clone(); spawned_tasks.push(SpawnedTask::spawn_blocking({ move || { for p in local_locations { - let r = fetch_partition_local(&work_dir, &p, sort_shuffle_enabled); + let r = { + let _timer = local_read_time.timer(); + fetch_partition_local(&work_dir, &p, sort_shuffle_enabled) + }; if let Err(e) = response_sender_c.blocking_send(r) { error!("Fail to send response event to the channel due to {e}"); } @@ -784,41 +848,55 @@ fn send_fetch_partitions( let customize_endpoint = customize_endpoint.clone(); let client_grpc_config = client_grpc_config.clone(); let client_pool = client_pool.clone(); + let read_metrics = read_metrics.clone(); spawned_tasks.push(SpawnedTask::spawn(async move { let addr = p.executor_meta.id.clone(); let size = block_size(&p, default_block_size); - // Acquire the cheap count/address gates first, then the byte budget - // last, so a fetch does not hold scarce byte permits while blocked - // on a per-address slot. All acquires are on Arcs that - // are never closed, so `unwrap()` cannot panic. - let req_permit = req_sem.acquire_owned().await.unwrap(); - let addr_sem = { - let mut map = addr_sems.lock().unwrap(); - map.entry(addr.clone()) - .or_insert_with(|| { - Arc::new(Semaphore::new(max_blocks_per_addr.max(1))) - }) - .clone() + // Time spent blocked acquiring the three governor permits (#1951). + let (req_permit, addr_permit, byte_permit) = { + let _permit_timer = read_metrics.permit_wait_time.timer(); + let req_permit = req_sem.acquire_owned().await.unwrap(); + let addr_sem = { + let mut map = addr_sems.lock().unwrap(); + map.entry(addr.clone()) + .or_insert_with(|| { + Arc::new(Semaphore::new(max_blocks_per_addr.max(1))) + }) + .clone() + }; + let addr_permit = addr_sem.acquire_owned().await.unwrap(); + let byte_permit = byte_sem + .acquire_many_owned(byte_permits_for(size, max_bytes)) + .await + .unwrap(); + (req_permit, addr_permit, byte_permit) }; - let addr_permit = addr_sem.acquire_owned().await.unwrap(); - let byte_permit = byte_sem - .acquire_many_owned(byte_permits_for(size, max_bytes)) - .await - .unwrap(); - let r = with_retry(outer_retries, io_wait, is_retriable_fetch_error, || { - fetch_partition_buffered( - &p, - client_grpc_config.clone(), - prefer_flight, - customize_endpoint.clone(), - client_pool.clone(), - ) - }) - .await + let mut attempts = 0usize; + // Cloned (rather than used by reference) to avoid a partial move of + // `read_metrics`, which is still needed below for + // `fetch_requests`/`fetch_retries`. + let decoded_bytes = read_metrics.decoded_bytes.clone(); + let r = { + let _fetch_timer = read_metrics.fetch_time.timer(); + with_retry(outer_retries, io_wait, is_retriable_fetch_error, || { + attempts += 1; + fetch_partition_buffered( + &p, + client_grpc_config.clone(), + prefer_flight, + customize_endpoint.clone(), + client_pool.clone(), + ) + }) + .await + } .map(|(schema, batches)| { + let bytes: usize = + batches.iter().map(|b| b.get_array_memory_size()).sum(); + decoded_bytes.add(bytes); Box::pin(GovernedStream::new( buffered_stream(schema, batches), byte_permit, @@ -826,6 +904,10 @@ fn send_fetch_partitions( addr_permit, )) as SendableRecordBatchStream }); + // Total wire attempts (initial + retries), recorded only after + // `with_retry` has finished retrying. + read_metrics.fetch_requests.add(attempts); + read_metrics.fetch_retries.add(attempts.saturating_sub(1)); if let Err(e) = response_sender.send(r).await { error!("Fail to send response event to the channel due to {e}"); @@ -1550,6 +1632,99 @@ mod tests { Ok(()) } + #[tokio::test] + async fn fetch_partitions_error_path_records_metrics() -> Result<()> { + // A single remote partition location pointing at a host with no Flight + // server listening, so every wire attempt fails with a + // `GrpcConnectionError` (remapped to `FetchFailed` by + // `fetch_partition_remote`), which `is_retriable_fetch_error` treats + // as retriable. Unlike `test_fetch_partitions_error_mapping` (which + // fans out 4 upstream locations into 4 concurrent remote tasks), this + // test uses exactly one location so there is a single fetch task and + // the attempt/retry counters are deterministic: with several + // concurrent tasks racing to error out first, the stream can return + // as soon as the fastest task fails, aborting the others mid-retry + // and making their counters nondeterministic. + let retries: usize = 2; + let config = SessionConfig::new_with_ballista() + .set_usize(crate::config::BALLISTA_CLIENT_IO_RETRIES_TIMES, retries) + .set_usize(crate::config::BALLISTA_CLIENT_IO_RETRY_WAIT_TIME_MS, 0); + + let session_ctx = SessionContext::new_with_config(config); + let task_ctx = session_ctx.task_ctx(); + + let schema = Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + Field::new("c", DataType::Int32, false), + ]); + + let job_id = "test_job_metrics"; + let input_stage_id = 2; + let partition = PartitionLocation { + map_partition_id: 0, + partition_id: PartitionId { + job_id: job_id.into(), + stage_id: input_stage_id, + partition_id: 0, + }, + executor_meta: ExecutorMetadata { + id: "executor_1".to_string(), + host: "executor_1".to_string(), + port: 7070, + grpc_port: 8080, + specification: ExecutorSpecification::default().with_task_slots(1), + os_info: ExecutorOperatingSystemSpecification::default(), + }, + partition_stats: Default::default(), + file_id: None, + is_sort_shuffle: false, + }; + let work_dir = TempDir::new().unwrap(); + let work_dir = work_dir.path().to_str().unwrap().to_owned(); + + let shuffle_reader_exec = ShuffleReaderExec::try_new( + input_stage_id, + vec![vec![partition]], + Arc::new(schema), + Partitioning::UnknownPartitioning(1), + )? + .with_work_dir(work_dir); + + let mut stream = shuffle_reader_exec.execute(0, task_ctx)?; + let batches = utils::collect_stream(&mut stream).await; + assert!(batches.is_err()); + let ballista_error = batches.unwrap_err(); + assert!(matches!( + ballista_error, + BallistaError::FetchFailed(_, _, _, _) + )); + + // The injected error is retriable (see `is_retriable_fetch_error`), so + // `with_retry` runs the initial attempt plus `retries` retries before + // giving up: total wire attempts = 1 + retries. + let expected_attempts = retries + 1; + let expected_retries = retries; + + let metrics = shuffle_reader_exec + .metrics() + .expect("ShuffleReaderExec should report metrics"); + let count = |name: &str| metrics.sum_by_name(name).map(|v| v.as_usize()); + + assert_eq!(count("fetch_requests"), Some(expected_attempts)); + assert_eq!(count("fetch_retries"), Some(expected_retries)); + assert!( + metrics.sum_by_name("fetch_time").is_some(), + "fetch_time metric should be registered" + ); + assert!( + metrics.sum_by_name("permit_wait_time").is_some(), + "permit_wait_time metric should be registered" + ); + + Ok(()) + } + #[tokio::test] async fn test_send_fetch_partitions_1() { test_send_fetch_partitions(1, 10).await; @@ -1675,11 +1850,13 @@ mod tests { let config = SessionConfig::new_with_ballista() .with_ballista_shuffle_reader_maximum_concurrent_requests(max_request_num); + let metrics_set = ExecutionPlanMetricsSet::new(); let response_receiver = send_fetch_partitions( &work_dir.to_string_lossy(), partition_locations, &config, None, + ShuffleReadMetrics::new(0, &metrics_set), ); let stream = RecordBatchStreamAdapter::new( @@ -1691,6 +1868,90 @@ mod tests { assert_eq!(partition_num, result.len()); } + #[tokio::test] + async fn send_fetch_partitions_records_local_metrics() { + let schema = get_test_partition_schema(); + let data_array = Int32Array::from(vec![1]); + let batch = + RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(data_array)]) + .unwrap(); + let tmp_dir = tempdir().unwrap(); + let work_dir = tmp_dir.path(); + let partition_num = 3usize; + for p in 0..partition_num { + let file_path = + create_shuffle_path(work_dir, &"job".into(), 1, p, None, false).unwrap(); + std::fs::create_dir_all(file_path.parent().unwrap()).unwrap(); + let file = File::create(&file_path).unwrap(); + let mut writer = StreamWriter::try_new(file, &schema).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + let partition_locations = get_test_partition_locations(partition_num, None); + let config = SessionConfig::new_with_ballista(); + let metrics_set = ExecutionPlanMetricsSet::new(); + let read_metrics = ShuffleReadMetrics::new(0, &metrics_set); + + let response_receiver = send_fetch_partitions( + &work_dir.to_string_lossy(), + partition_locations, + &config, + None, + read_metrics, + ); + let stream = RecordBatchStreamAdapter::new( + Arc::new(schema), + response_receiver.try_flatten(), + ); + let result = common::collect(Box::pin(stream)).await.unwrap(); + assert_eq!(partition_num, result.len()); + + let metrics = metrics_set.clone_inner(); + let count = + |name: &str| metrics.sum_by_name(name).map(|v| v.as_usize()).unwrap_or(0); + assert_eq!(count("local_partitions"), partition_num); + assert_eq!(count("remote_partitions"), 0); + assert!( + metrics.sum_by_name("local_read_time").is_some(), + "local_read_time metric should be registered" + ); + } + + #[tokio::test] + async fn send_fetch_partitions_counts_remote_split() { + let tmp_dir = tempdir().unwrap(); + let work_dir = tmp_dir.path(); + // No files on disk => all partitions are treated as remote. + let partition_locations = get_test_partition_locations(2, None); + let config = SessionConfig::new_with_ballista(); + let metrics_set = ExecutionPlanMetricsSet::new(); + let read_metrics = ShuffleReadMetrics::new(0, &metrics_set); + + // Drop the receiver without polling; the split counts are recorded + // synchronously inside send_fetch_partitions. + let _rx = send_fetch_partitions( + &work_dir.to_string_lossy(), + partition_locations, + &config, + None, + read_metrics, + ); + + let metrics = metrics_set.clone_inner(); + assert_eq!( + metrics + .sum_by_name("remote_partitions") + .map(|v| v.as_usize()), + Some(2) + ); + assert_eq!( + metrics + .sum_by_name("local_partitions") + .map(|v| v.as_usize()), + Some(0) + ); + } + fn get_test_partition_locations( n: usize, file_id: Option, diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index 3398be2275..4a99197061 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -53,9 +53,10 @@ use datafusion::physical_plan::metrics::{ self, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet, }; +use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, - SendableRecordBatchStream, Statistics, displayable, + SendableRecordBatchStream, Statistics, }; use futures::{StreamExt, TryFutureExt, TryStreamExt}; @@ -95,7 +96,7 @@ pub struct ShuffleWriterExec { impl std::fmt::Display for ShuffleWriterExec { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let printable_plan = displayable(self.plan.as_ref()) + let printable_plan = DisplayableExecutionPlan::with_metrics(self.plan.as_ref()) .set_show_statistics(true) .indent(false); write!( @@ -665,6 +666,38 @@ mod tests { Ok(()) } + #[tokio::test] + #[cfg(not(feature = "force_hash_collisions"))] + async fn display_renders_child_operator_metrics() -> Result<()> { + let session_ctx = SessionContext::new(); + let task_ctx = session_ctx.task_ctx(); + + let input_plan = Arc::new(CoalescePartitionsExec::new(create_input_plan()?)); + let work_dir = TempDir::new()?; + let query_stage = ShuffleWriterExec::try_new( + JobId::new("jobDisplay"), + 1, + input_plan, + work_dir.path().to_str().unwrap().to_owned(), + Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)), + )?; + let mut stream = query_stage.execute(0, task_ctx)?; + let _ = utils::collect_stream(&mut stream) + .await + .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; + + let rendered = format!("{query_stage}"); + assert!( + rendered.contains("metrics="), + "expected child-operator metrics in rendered plan:\n{rendered}" + ); + assert!( + rendered.contains("elapsed_compute"), + "expected populated elapsed_compute metric in rendered plan:\n{rendered}" + ); + Ok(()) + } + #[tokio::test] // number of rows in each partition is a function of the hash output, so don't test here #[cfg(not(feature = "force_hash_collisions"))] From e1e9514ba1fd89b59074b60a3b1e1ded334fb0a9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 11 Jul 2026 09:01:50 -0600 Subject: [PATCH 061/107] feat: delete intermediate shuffle files on job success (#1982) --- ballista/core/proto/ballista.proto | 6 ++ ballista/core/src/serde/generated/ballista.rs | 8 ++ ballista/executor/src/execution_loop.rs | 9 ++- ballista/executor/src/executor_process.rs | 74 +++++++++++++++++-- ballista/executor/src/executor_server.rs | 7 +- .../scheduler/src/scheduler_server/grpc.rs | 3 +- .../scheduler_server/query_stage_scheduler.rs | 16 +++- .../scheduler/src/state/execution_graph.rs | 37 ++++++++++ .../scheduler/src/state/execution_stage.rs | 12 +++ .../scheduler/src/state/executor_manager.rs | 60 +++++++++++++-- ballista/scheduler/src/state/mod.rs | 12 ++- ballista/scheduler/src/state/task_manager.rs | 13 ++-- 12 files changed, 224 insertions(+), 33 deletions(-) diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto index 50a3e25e70..41c946365f 100644 --- a/ballista/core/proto/ballista.proto +++ b/ballista/core/proto/ballista.proto @@ -783,6 +783,9 @@ message CancelJobResult { message CleanJobDataParams { string job_id = 1; + // Specific stage ids to remove within the job dir. + // Empty means remove the whole job dir (legacy behavior). + repeated uint32 remove_stage_ids = 2; } message CleanJobDataResult { @@ -820,6 +823,9 @@ message CancelTasksResult { message RemoveJobDataParams { string job_id = 1; + // Specific stage ids to remove within the job dir. + // Empty means remove the whole job dir (legacy behavior). + repeated uint32 remove_stage_ids = 2; } message RemoveJobDataResult { diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs index d071546664..07dfa8a42f 100644 --- a/ballista/core/src/serde/generated/ballista.rs +++ b/ballista/core/src/serde/generated/ballista.rs @@ -1188,6 +1188,10 @@ pub struct CancelJobResult { pub struct CleanJobDataParams { #[prost(string, tag = "1")] pub job_id: ::prost::alloc::string::String, + /// Specific stage ids to remove within the job dir. + /// Empty means remove the whole job dir (legacy behavior). + #[prost(uint32, repeated, tag = "2")] + pub remove_stage_ids: ::prost::alloc::vec::Vec, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CleanJobDataResult {} @@ -1233,6 +1237,10 @@ pub struct CancelTasksResult { pub struct RemoveJobDataParams { #[prost(string, tag = "1")] pub job_id: ::prost::alloc::string::String, + /// Specific stage ids to remove within the job dir. + /// Empty means remove the whole job dir (legacy behavior). + #[prost(uint32, repeated, tag = "2")] + pub remove_stage_ids: ::prost::alloc::vec::Vec, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct RemoveJobDataResult {} diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index a8b08efd7e..77cd4c7c5e 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -23,7 +23,7 @@ use crate::cpu_bound_executor::DedicatedExecutor; use crate::executor::Executor; -use crate::executor_process::remove_job_dir; +use crate::executor_process::remove_job_data; use crate::{TaskExecutionTimes, as_task_status}; use ballista_core::JobId; use ballista_core::error::BallistaError; @@ -119,12 +119,15 @@ where for cleanup in jobs_to_clean { let job_id = cleanup.job_id.clone().into(); let work_dir = executor.work_dir.clone(); + let remove_stage_ids = cleanup.remove_stage_ids.clone(); // In poll-based cleanup, removing job data is fire-and-forget. // Failures here do not affect task execution and are only logged. tokio::spawn(async move { - if let Err(e) = remove_job_dir(&work_dir, &job_id).await { - error!("failed to remove job dir {job_id}: {e}"); + if let Err(e) = + remove_job_data(&work_dir, &job_id, &remove_stage_ids).await + { + error!("failed to remove job data {job_id}: {e}"); } }); } diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 22f526edca..4e37687e81 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -737,11 +737,16 @@ async fn clean_all_shuffle_data(work_dir: &str) -> ballista_core::error::Result< Ok(()) } -/// Remove a job directory under work_dir. +/// Remove job data under work_dir. /// Used by both push-based (gRPC handler) and pull-based (poll loop) cleanup. -pub(crate) async fn remove_job_dir( +/// +/// `remove_stage_ids` empty ⇒ remove the whole `work_dir/{job_id}` dir (legacy +/// behavior). Non-empty ⇒ remove only the listed `work_dir/{job_id}/{stage_id}` +/// subdirs, retaining the rest of the job dir (e.g. the final-stage output). +pub(crate) async fn remove_job_data( work_dir: &str, job_id: &JobId, + remove_stage_ids: &[u32], ) -> ballista_core::error::Result<()> { let work_path = PathBuf::from(&work_dir); let job_path = work_path.join(job_id.as_str()); @@ -761,11 +766,30 @@ pub(crate) async fn remove_job_dir( ))); } - info!("Remove data for job {:?}", job_id); + // Empty stage list ⇒ remove the whole job dir (legacy behavior). + if remove_stage_ids.is_empty() { + info!("Remove data for job {:?}", job_id); + return tokio::fs::remove_dir_all(&job_path).await.map_err(|e| { + BallistaError::General(format!("Failed to remove {job_path:?} due to {e}")) + }); + } - tokio::fs::remove_dir_all(&job_path).await.map_err(|e| { - BallistaError::General(format!("Failed to remove {job_path:?} due to {e}")) - })?; + // Otherwise remove only the given (intermediate) stage subdirs. + for stage_id in remove_stage_ids { + let stage_path = job_path.join(stage_id.to_string()); + if !tokio::fs::try_exists(&stage_path).await.unwrap_or(false) { + continue; + } + if !is_subdirectory(stage_path.as_path(), job_path.as_path()) { + return Err(BallistaError::General(format!( + "Path {stage_path:?} is not a subdirectory of {job_path:?}" + ))); + } + info!("Remove intermediate data for job {job_id:?} stage {stage_id}"); + tokio::fs::remove_dir_all(&stage_path).await.map_err(|e| { + BallistaError::General(format!("Failed to remove {stage_path:?} due to {e}")) + })?; + } Ok(()) } @@ -886,6 +910,7 @@ mod tests { use std::path::{Path, PathBuf}; use super::clean_shuffle_data_loop; + use super::remove_job_data; use ballista_core::JobId; use std::fs; use std::fs::File; @@ -984,6 +1009,43 @@ mod tests { assert!(!is_subdirectory(&job_path, base_dir)); } } + #[tokio::test] + async fn test_remove_intermediate_stage_data() { + let work_dir = TempDir::new().unwrap(); + let work = work_dir.path(); + let job_id: JobId = "job".into(); + + // Create job/1, job/2, job/3, each with a data file. + for stage in [1u32, 2, 3] { + let stage_dir = work.join("job").join(stage.to_string()); + fs::create_dir_all(&stage_dir).unwrap(); + File::create(stage_dir.join("data.arrow")) + .unwrap() + .write_all(b"x") + .unwrap(); + } + + // Remove intermediate stages 1 and 2; stage 3 (final) is retained. + remove_job_data(work.to_str().unwrap(), &job_id, &[1, 2]) + .await + .unwrap(); + assert!(!work.join("job").join("1").exists()); + assert!(!work.join("job").join("2").exists()); + assert!(work.join("job").join("3").exists()); + + // Removing a missing stage id is a no-op (idempotent). + remove_job_data(work.to_str().unwrap(), &job_id, &[99]) + .await + .unwrap(); + assert!(work.join("job").join("3").exists()); + + // Empty list removes the whole job dir. + remove_job_data(work.to_str().unwrap(), &job_id, &[]) + .await + .unwrap(); + assert!(!work.join("job").exists()); + } + fn prepare_testing_job_directory(base_dir: &Path, job_id: &JobId) -> PathBuf { let mut path = base_dir.to_path_buf(); path.push(job_id.as_str()); diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs index c7c564fe35..8e7b023549 100644 --- a/ballista/executor/src/executor_server.rs +++ b/ballista/executor/src/executor_server.rs @@ -63,7 +63,7 @@ use tokio::task::JoinHandle; use crate::cpu_bound_executor::DedicatedExecutor; use crate::executor::Executor; -use crate::executor_process::{ExecutorProcessConfig, remove_job_dir}; +use crate::executor_process::{ExecutorProcessConfig, remove_job_data}; use crate::metrics::ExecutorMetricCollectionPolicy; use crate::shutdown::ShutdownNotifier; use crate::{TaskExecutionTimes, as_task_status}; @@ -933,9 +933,10 @@ impl ExecutorGrpc &self, request: Request, ) -> Result, Status> { - let job_id = request.into_inner().job_id.into(); + let params = request.into_inner(); + let job_id = params.job_id.into(); - remove_job_dir(&self.executor.work_dir, &job_id) + remove_job_data(&self.executor.work_dir, &job_id, ¶ms.remove_stage_ids) .await .map_err(|e| Status::invalid_argument(e.to_string()))?; diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs index ef1a93136a..5c21fa806d 100644 --- a/ballista/scheduler/src/scheduler_server/grpc.rs +++ b/ballista/scheduler/src/scheduler_server/grpc.rs @@ -161,8 +161,9 @@ impl SchedulerGrpc .executor_manager .drain_pending_cleanup_jobs(&executor_id) .into_iter() - .map(|job_id| CleanJobDataParams { + .map(|(job_id, remove_stage_ids)| CleanJobDataParams { job_id: job_id.into_inner(), + remove_stage_ids, }) .collect(); Ok(Response::new(PollWorkResult { diff --git a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs index 9bf33307c9..cd8e21db05 100644 --- a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs +++ b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs @@ -211,10 +211,18 @@ impl .record_completed(&job_id, queued_at, completed_at); info!("Job finished successfully: [{job_id}]"); - if let Err(e) = self.state.task_manager.succeed_job(&job_id).await { - error!("Fail to invoke succeed_job for job {job_id} due to {e:?}"); - } - self.state.clean_up_successful_job(job_id); + let intermediate_stage_ids = + match self.state.task_manager.succeed_job(&job_id).await { + Ok(ids) => ids, + Err(e) => { + error!( + "Fail to invoke succeed_job for job {job_id} due to {e:?}" + ); + vec![] + } + }; + self.state + .clean_up_successful_job(job_id, intermediate_stage_ids); } QueryStageSchedulerEvent::JobRunningFailed { job_id, diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 879a172964..68ab2f4a8d 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -240,6 +240,16 @@ pub trait ExecutionGraph: Debug { /// Exposes executions stages and stage id's fn stages(&self) -> &HashMap; + /// Stage ids of all non-final (intermediate) stages — those whose + /// `output_links` is non-empty. The final stage(s) are excluded. + fn intermediate_stage_ids(&self) -> Vec { + self.stages() + .iter() + .filter(|(_, stage)| !stage.output_links().is_empty()) + .map(|(stage_id, _)| *stage_id as u32) + .collect() + } + /// returns next task to run /// (used for testing only) #[cfg(test)] @@ -1786,6 +1796,33 @@ mod test { test_union_all_plan, test_union_plan, }; + #[tokio::test] + async fn test_intermediate_stage_ids() { + // A simple aggregation produces a 2-stage graph: one intermediate + // stage (non-empty output_links) feeding one final stage (empty + // output_links). + let graph = test_aggregation_plan(4).await; + + assert_eq!(graph.stages().len(), 2); + + // Exactly one final stage. + let final_count = graph + .stages() + .values() + .filter(|s| s.output_links().is_empty()) + .count(); + assert_eq!(final_count, 1); + + // Intermediate = all - final = exactly one stage, and none of the + // returned ids is a final stage. + let intermediate = graph.intermediate_stage_ids(); + assert_eq!(intermediate.len(), 1); + for id in &intermediate { + let stage = graph.stages().get(&(*id as usize)).unwrap(); + assert!(!stage.output_links().is_empty()); + } + } + #[tokio::test] async fn test_fail_job_sets_end_time_and_failed_metadata() -> Result<()> { let mut graph = test_aggregation_plan(4).await; diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index 10440f357a..eafc13af9a 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -107,6 +107,18 @@ impl ExecutionStage { ExecutionStage::Failed(stage) => stage.plan.as_ref(), } } + + /// Get the output links for this stage. An empty slice means this is a + /// final stage in the `ExecutionGraph`. + pub fn output_links(&self) -> &[usize] { + match self { + ExecutionStage::UnResolved(stage) => &stage.output_links, + ExecutionStage::Resolved(stage) => &stage.output_links, + ExecutionStage::Running(stage) => &stage.output_links, + ExecutionStage::Successful(stage) => &stage.output_links, + ExecutionStage::Failed(stage) => &stage.output_links, + } + } } /// For a stage whose input stages are not all completed, we say it's a unresolved stage diff --git a/ballista/scheduler/src/state/executor_manager.rs b/ballista/scheduler/src/state/executor_manager.rs index 1871514a7b..febb4084df 100644 --- a/ballista/scheduler/src/state/executor_manager.rs +++ b/ballista/scheduler/src/state/executor_manager.rs @@ -66,8 +66,9 @@ pub struct ExecutorManager { config: Arc, /// Cached gRPC clients for communicating with executors. clients: ExecutorClients, - /// Jobs pending cleanup on each executor. - pending_cleanup_jobs: Arc>>, + /// Per-executor pending cleanups: job id -> stage ids to remove + /// (empty stage ids ⇒ remove the whole job dir). + pending_cleanup_jobs: Arc>>>, /// Configuration for gRPC client connections. grpc_client_config: GrpcClientConfig, } @@ -191,7 +192,9 @@ impl ExecutorManager { let executor_manager = self.clone(); tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(clean_up_interval)).await; - executor_manager.clean_up_job_data_inner(job_id).await; + executor_manager + .clean_up_job_data_inner(job_id, vec![]) + .await; }); } @@ -199,13 +202,49 @@ impl ExecutorManager { pub fn clean_up_job_data(&self, job_id: JobId) { let executor_manager = self.clone(); tokio::spawn(async move { - executor_manager.clean_up_job_data_inner(job_id).await; + executor_manager + .clean_up_job_data_inner(job_id, vec![]) + .await; + }); + } + + /// Immediately reclaim intermediate-stage shuffle data for a successful job, + /// retaining the rest of the job dir (e.g. the final-stage output) for the + /// existing delayed whole-job cleanup. No-op if `remove_stage_ids` is empty. + /// + /// Contract: call this AT MOST ONCE per job, passing the complete set of + /// intermediate stage ids. It is not designed for per-stage invocation. + /// In poll-based scheduling the pending-cleanup value is a `Vec` keyed + /// by job, and `HashMap::insert` REPLACES on key collision (it does not + /// merge). One intermediate call followed by the delayed whole-job cleanup + /// (empty `remove_stage_ids`) is correct: the empty/whole-job entry + /// supersedes and removes the entire job dir. But two DISTINCT partial calls + /// for the same job before the executor polls would drop the earlier stage + /// ids. Do not "fix" this by merging/extending: an empty vec means "remove + /// the whole job dir", so extending could never represent whole-job + /// supersession. + pub(crate) fn clean_up_intermediate_job_data( + &self, + job_id: JobId, + remove_stage_ids: Vec, + ) { + if remove_stage_ids.is_empty() { + return; + } + let executor_manager = self.clone(); + tokio::spawn(async move { + executor_manager + .clean_up_job_data_inner(job_id, remove_stage_ids) + .await; }); } /// 1. Push strategy: Send rpc to Executors to clean up the job data /// 2. Poll strategy: Save cleanup job ids and send them to executors - async fn clean_up_job_data_inner(&self, job_id: JobId) { + /// + /// `remove_stage_ids` empty ⇒ remove the whole job dir; non-empty ⇒ remove + /// only those stage subdirs. + async fn clean_up_job_data_inner(&self, job_id: JobId, remove_stage_ids: Vec) { let alive_executors = self.get_alive_executors(); for executor in alive_executors { @@ -215,10 +254,12 @@ impl ExecutorManager { if let Ok(mut client) = self.get_client(&executor, &self.grpc_client_config).await { + let remove_stage_ids = remove_stage_ids.clone(); tokio::spawn(async move { if let Err(err) = client .remove_job_data(RemoveJobDataParams { job_id: job_id_clone, + remove_stage_ids, }) .await { @@ -234,7 +275,7 @@ impl ExecutorManager { self.pending_cleanup_jobs .entry(executor) .or_default() - .insert(job_id.clone()); + .insert(job_id.clone(), remove_stage_ids.clone()); } } } @@ -387,10 +428,13 @@ impl ExecutorManager { Ok(()) } - pub(crate) fn drain_pending_cleanup_jobs(&self, executor_id: &str) -> HashSet { + pub(crate) fn drain_pending_cleanup_jobs( + &self, + executor_id: &str, + ) -> Vec<(JobId, Vec)> { self.pending_cleanup_jobs .remove(executor_id) - .map(|(_, jobs)| jobs) + .map(|(_, jobs)| jobs.into_iter().collect()) .unwrap_or_default() } diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index ce007330e7..4b8b4385d5 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -382,8 +382,16 @@ impl SchedulerState, + ) { + self.executor_manager + .clean_up_intermediate_job_data(job_id.clone(), intermediate_stage_ids); self.executor_manager.clean_up_job_data_delayed( job_id.clone(), self.config.finished_job_data_clean_up_interval_seconds, diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index 7f18d9581b..5c4d907898 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -585,24 +585,25 @@ impl TaskManager Ok(events) } - /// Mark a job to success. This will create a key under the CompletedJobs keyspace - /// and remove the job from ActiveJobs - pub(crate) async fn succeed_job(&self, job_id: &JobId) -> Result<()> { + /// Move a job from Active to Success and return the ids of its intermediate + /// (non-final) stages so their shuffle data can be reclaimed immediately. + /// Returns an empty vec if the job is not found or not successful. + pub(crate) async fn succeed_job(&self, job_id: &JobId) -> Result> { debug!("Moving job {job_id} from Active to Success"); if let Some(graph) = self.remove_active_execution_graph(job_id) { let graph = graph.read().await; if graph.is_successful() { self.state.save_job(job_id, &graph).await?; + Ok(graph.intermediate_stage_ids()) } else { error!("Job {job_id} has not finished and cannot be completed"); - return Ok(()); + Ok(vec![]) } } else { warn!("Fail to find job {job_id} in the cache"); + Ok(vec![]) } - - Ok(()) } /// Cancel the job and return a Vec of running tasks need to cancel From daebb3c928aa784933f2221b45e231b2994639b9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 11 Jul 2026 10:38:20 -0600 Subject: [PATCH 062/107] feat(executor): log task execution duration in finished-task message (#1999) Include the elapsed execution time in the "Finished task" log at both task-dispatch sites (the executor_server push path and the execution_loop pull path), so operators can read per-task durations directly from executor logs without cross-referencing task-status reports. --- ballista/executor/src/execution_loop.rs | 8 ++++++-- ballista/executor/src/executor_server.rs | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index 77cd4c7c5e..8a759f501c 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -44,7 +44,7 @@ use std::any::Any; use std::convert::TryInto; use std::error::Error; use std::sync::mpsc::{Receiver, Sender, TryRecvError}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::{sync::Arc, time::Duration}; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tonic::codegen::{Body, Bytes, StdError}; @@ -297,6 +297,7 @@ async fn run_received_task ExecutorServer ExecutorServer Date: Sat, 11 Jul 2026 16:54:50 -0700 Subject: [PATCH 063/107] perf: prune irrelevant partitions task (#1911) --- ballista/scheduler/src/state/task_manager.rs | 238 ++++++++++++++++++- 1 file changed, 236 insertions(+), 2 deletions(-) diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index 5c4d907898..57bbd797e9 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -27,6 +27,8 @@ use crate::state::execution_graph::{ use crate::state::executor_manager::ExecutorManager; use ballista_core::error::BallistaError; use ballista_core::error::Result; +#[cfg(feature = "disable-stage-plan-cache")] +use ballista_core::execution_plans::ShuffleReaderExec; use ballista_core::extension::{SessionConfigExt, SessionConfigHelperExt}; use ballista_core::serde::BallistaCodec; use ballista_core::serde::protobuf::{ @@ -35,10 +37,14 @@ use ballista_core::serde::protobuf::{ use ballista_core::serde::scheduler::ExecutorMetadata; use ballista_core::{JobId, JobStatusSubscriber}; use dashmap::DashMap; +#[cfg(feature = "disable-stage-plan-cache")] +use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::execution::config::SessionConfig; use datafusion::execution::context::SessionContext; use datafusion::logical_expr::LogicalPlan; use datafusion::physical_plan::ExecutionPlan; +#[cfg(feature = "disable-stage-plan-cache")] +use datafusion::physical_plan::ExecutionPlanProperties; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; use datafusion_proto::protobuf::PhysicalPlanNode; @@ -163,12 +169,75 @@ impl JobInfoCache { encoded_stage_plans: HashMap::new(), } } + + #[cfg(feature = "disable-stage-plan-cache")] + fn partition_prune_helper( + partition_ids: &[usize], + plan: &Arc, + ) -> Result> { + let n = plan.output_partitioning().partition_count(); + let wanted: HashSet = partition_ids.iter().copied().collect(); + Ok(plan + .clone() + .transform_up(|node| { + let Some(r) = node.downcast_ref::() else { + return Ok(Transformed::no(node)); + }; + // Skip broadcast readers (serve partition[0] for every index) and readers + // whose partition count differs from the stage output `n`, since pruning by + // index is only valid when reader partition `i` feeds output partition `i`. + if r.broadcast || r.partition.len() != n { + return Ok(Transformed::no(node)); + } + // Nothing to prune when this task consumes every partition. + if wanted.len() == r.partition.len() { + return Ok(Transformed::no(node)); + } + + // Every requested id must index a real reader partition, else we'd + // silently prune away locations the task needs. + debug_assert!(wanted.iter().all(|&p| p < r.partition.len())); + + let partition = r + .partition + .iter() + .enumerate() + .map(|(i, loc)| { + if wanted.contains(&i) { + loc.clone() + } else { + vec![] + } + }) + .collect(); + + let reader = match r.coalesce.clone() { + Some(c) => ShuffleReaderExec::try_new_coalesced( + r.stage_id, + partition, + c, + r.schema(), + r.properties().output_partitioning().clone(), + )?, + None => ShuffleReaderExec::try_new( + r.stage_id, + partition, + r.schema(), + r.properties().output_partitioning().clone(), + )?, + }; + Ok(Transformed::yes(Arc::new(reader) as Arc)) + })? + .data) + } + #[cfg(not(feature = "disable-stage-plan-cache"))] fn encode_stage_plan( &mut self, stage_id: usize, plan: &Arc, codec: &dyn PhysicalExtensionCodec, + _partition_ids: &[usize], ) -> Result> { if let Some(plan) = self.encoded_stage_plans.get(&stage_id) { Ok(plan.clone()) @@ -177,7 +246,6 @@ impl JobInfoCache { let plan_proto = U::try_from_physical_plan(plan.clone(), codec)?; plan_proto.try_encode(&mut plan_buf)?; self.encoded_stage_plans.insert(stage_id, plan_buf.clone()); - Ok(plan_buf) } } @@ -188,9 +256,11 @@ impl JobInfoCache { _stage_id: usize, plan: &Arc, codec: &dyn PhysicalExtensionCodec, + partition_ids: &[usize], ) -> Result> { let mut plan_buf: Vec = vec![]; - let plan_proto = U::try_from_physical_plan(plan.clone(), codec)?; + let pruned_plan = Self::partition_prune_helper(partition_ids, plan)?; + let plan_proto = U::try_from_physical_plan(pruned_plan, codec)?; plan_proto.try_encode(&mut plan_buf)?; Ok(plan_buf) @@ -737,6 +807,7 @@ impl TaskManager stage_id, &task.plan, self.codec.physical_extension_codec(), + &[task.partition.partition_id], )?; let task_definition = TaskDefinition { @@ -792,6 +863,10 @@ impl TaskManager &self, tasks: Vec, ) -> Result> { + let partition_ids: Vec = tasks + .iter() + .map(|task| task.partition.partition_id) + .collect(); if let Some(task) = tasks.first() { let session_id = task.session_id.clone(); let job_id = task.partition.job_id.clone(); @@ -814,6 +889,7 @@ impl TaskManager stage_id, &task.plan, self.codec.physical_extension_codec(), + &partition_ids, )?; let launch_time = SystemTime::now() @@ -939,3 +1015,161 @@ impl From<&ExecutionGraphBox> for JobOverview { } } } + +#[cfg(all(test, feature = "disable-stage-plan-cache"))] +mod prune_partition_tests { + use crate::state::task_manager::JobInfoCache; + use ballista_core::JobId; + use ballista_core::execution_plans::{ + CoalescePlan, PartitionGroup, ShuffleReaderExec, + }; + use ballista_core::serde::scheduler::{ + ExecutorMetadata, PartitionId, PartitionLocation, + }; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::physical_expr::Partitioning; + use datafusion::physical_plan::ExecutionPlan; + use std::sync::Arc; + + fn create_partition(partition: usize) -> PartitionLocation { + PartitionLocation { + map_partition_id: 0, + partition_id: PartitionId { + job_id: JobId::new("demo".to_string()), + stage_id: 0, + partition_id: partition, + }, + executor_meta: ExecutorMetadata { + id: "1".to_string(), + host: "1.1.1.1".to_string(), + port: 0, + grpc_port: 0, + specification: Default::default(), + os_info: Default::default(), + }, + partition_stats: Default::default(), + file_id: None, + is_sort_shuffle: false, + } + } + + #[test] + fn check_prune_unwanted_partitions() { + let partitions = (0..4).map(|i| vec![create_partition(i)]).collect(); + let reader = ShuffleReaderExec::try_new( + 1, + partitions, + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + Partitioning::UnknownPartitioning(4), + ) + .unwrap(); + let plan: Arc = Arc::new(reader); + let pruned = JobInfoCache::partition_prune_helper(&[1], &plan).unwrap(); + let r = pruned + .downcast_ref::() + .expect("expected a ShuffleReaderExec"); + assert!(r.partition[0].is_empty()); + assert_eq!(r.partition[1].len(), 1); + assert_eq!(r.partition[1][0].partition_id.partition_id, 1); + assert!(r.partition[2].is_empty()); + assert!(r.partition[3].is_empty()); + } + + #[test] + fn check_keeps_multiple_wanted_partitions() { + let partitions = (0..4).map(|i| vec![create_partition(i)]).collect(); + let reader = ShuffleReaderExec::try_new( + 1, + partitions, + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + Partitioning::UnknownPartitioning(4), + ) + .unwrap(); + let plan: Arc = Arc::new(reader); + let pruned = JobInfoCache::partition_prune_helper(&[0, 3], &plan).unwrap(); + let r = pruned + .downcast_ref::() + .expect("expected a ShuffleReaderExec"); + assert_eq!(r.partition[0].len(), 1); + assert!(r.partition[1].is_empty()); + assert!(r.partition[2].is_empty()); + assert_eq!(r.partition[3].len(), 1); + } + + #[test] + fn check_broadcast_reader_not_pruned() { + // Broadcast readers serve partition[0] for every index, so pruning must be a no-op. + let reader = ShuffleReaderExec::try_new_broadcast( + 1, + (0..4).map(create_partition).collect(), + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + 4, + ) + .unwrap(); + let plan: Arc = Arc::new(reader); + let pruned = JobInfoCache::partition_prune_helper(&[1], &plan).unwrap(); + let r = pruned + .downcast_ref::() + .expect("expected a ShuffleReaderExec"); + // Broadcast keeps all locations flattened into partition[0]; nothing pruned. + assert!(r.broadcast); + assert_eq!(r.partition.len(), 1); + assert_eq!(r.partition[0].len(), 4); + } + + #[test] + fn check_coalesced_reader_pruned_and_stays_coalesced() { + // K = 2 coalesce groups, each folding two upstream partitions. + let coalesce = CoalescePlan { + upstream_partition_count: 4, + groups: vec![ + PartitionGroup { + upstream_indices: vec![0, 1], + }, + PartitionGroup { + upstream_indices: vec![2, 3], + }, + ], + }; + let reader = ShuffleReaderExec::try_new_coalesced( + 1, + vec![ + vec![create_partition(0), create_partition(1)], + vec![create_partition(2), create_partition(3)], + ], + coalesce, + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + Partitioning::UnknownPartitioning(2), + ) + .unwrap(); + let plan: Arc = Arc::new(reader); + let pruned = JobInfoCache::partition_prune_helper(&[1], &plan).unwrap(); + let r = pruned + .downcast_ref::() + .expect("expected a ShuffleReaderExec"); + assert!(r.coalesce.is_some()); + assert!(r.partition[0].is_empty()); + assert_eq!(r.partition[1].len(), 2); + } + + #[test] + fn check_no_op_when_all_partitions_wanted() { + let partitions = (0..3).map(|i| vec![create_partition(i)]).collect(); + let reader = ShuffleReaderExec::try_new( + 1, + partitions, + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + Partitioning::UnknownPartitioning(3), + ) + .unwrap(); + let plan: Arc = Arc::new(reader); + let pruned = JobInfoCache::partition_prune_helper(&[0, 1, 2], &plan).unwrap(); + let r = pruned + .downcast_ref::() + .expect("expected a ShuffleReaderExec"); + // Task consumes every partition, so nothing is emptied. + assert_eq!(r.partition[0].len(), 1); + assert_eq!(r.partition[1].len(), 1); + assert_eq!(r.partition[2].len(), 1); + } +} From f67969cf6b642d49114674f10d30c520839d155e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 12 Jul 2026 07:45:01 -0600 Subject: [PATCH 064/107] feat: Share read-side runtime state across a session's tasks on the executor (#1995) --- Cargo.lock | 1 + ballista/executor/Cargo.toml | 1 + ballista/executor/src/config.rs | 10 + ballista/executor/src/execution_loop.rs | 3 +- ballista/executor/src/executor.rs | 105 +++++++++++ ballista/executor/src/executor_process.rs | 171 ++++++++++------- ballista/executor/src/executor_server.rs | 4 +- ballista/executor/src/lib.rs | 2 + ballista/executor/src/runtime_cache.rs | 218 ++++++++++++++++++++++ python/Cargo.lock | 24 ++- 10 files changed, 462 insertions(+), 77 deletions(-) create mode 100644 ballista/executor/src/runtime_cache.rs diff --git a/Cargo.lock b/Cargo.lock index fa3e259c26..b918eb711a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1159,6 +1159,7 @@ dependencies = [ "futures", "libc", "log", + "lru 0.18.0", "memory-stats", "mimalloc", "parking_lot", diff --git a/ballista/executor/Cargo.toml b/ballista/executor/Cargo.toml index c80e68b9cc..aa60cb2fd1 100644 --- a/ballista/executor/Cargo.toml +++ b/ballista/executor/Cargo.toml @@ -50,6 +50,7 @@ datafusion = { workspace = true } datafusion-proto = { workspace = true } futures = { workspace = true } log = { workspace = true } +lru = "0.18" memory-stats = { version = "1.2.0", features = ["always_use_statm"] } mimalloc = { workspace = true, optional = true } parking_lot = { workspace = true } diff --git a/ballista/executor/src/config.rs b/ballista/executor/src/config.rs index 8ddeb4ab3c..48b5a28c9c 100644 --- a/ballista/executor/src/config.rs +++ b/ballista/executor/src/config.rs @@ -171,6 +171,15 @@ pub struct Config { help = "Optional total executor memory budget (e.g. \"8GB\", \"512MiB\"). Each concurrent task receives an equal share." )] pub memory_pool_size: Option, + /// Maximum number of sessions whose shared runtime state (object-store + /// clients, Parquet footer cache) is retained on the executor (LRU). `0` + /// disables caching. + #[arg( + long, + default_value_t = 16, + help = "Max number of sessions whose shared runtime state (object-store clients, Parquet footer cache) is retained on the executor (LRU). 0 disables caching." + )] + pub session_runtime_cache_capacity: usize, /// Number of seconds established client connection should be cached if not used (0 means no cache) #[arg( long, @@ -207,6 +216,7 @@ impl TryFrom for ExecutorProcessConfig { executor_heartbeat_interval_seconds: opt.executor_heartbeat_interval_seconds, metric_collection_policy: opt.metric_collection_policy, memory_pool_size: opt.memory_pool_size, + session_runtime_cache_capacity: opt.session_runtime_cache_capacity, override_execution_engine: None, override_function_registry: None, override_config_producer: None, diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs index 8a759f501c..ff0838a7e6 100644 --- a/ballista/executor/src/execution_loop.rs +++ b/ballista/executor/src/execution_loop.rs @@ -262,7 +262,8 @@ async fn run_received_task, + + /// Optional session-keyed cache of shared base runtime envs. When set, + /// `produce_runtime_for_session` reuses read-side state across a session's + /// tasks; when `None`, each task builds a runtime from `runtime_producer`. + session_runtime_cache: Option>, } impl Executor { @@ -144,6 +150,7 @@ impl Executor { concurrent_tasks, abort_handles: Default::default(), execution_engine, + session_runtime_cache: None, } } /// Creates new Executor with default `ExecutionEngine`. @@ -167,6 +174,7 @@ impl Executor { concurrent_tasks, abort_handles: Default::default(), execution_engine: Arc::new(DefaultExecutionEngine::new()), + session_runtime_cache: None, } } } @@ -180,6 +188,30 @@ impl Executor { (self.runtime_producer)(config) } + /// Attaches (or clears) the session-keyed runtime cache. Cloning the + /// `Executor` shares the same cache via `Arc`. + pub fn with_session_runtime_cache( + mut self, + cache: Option>, + ) -> Self { + self.session_runtime_cache = cache; + self + } + + /// Produces the runtime for a task, reusing the session's shared read-side + /// state when a session cache is attached; otherwise builds a runtime per + /// task via the `runtime_producer`. + pub fn produce_runtime_for_session( + &self, + session_id: &str, + config: &SessionConfig, + ) -> datafusion::error::Result> { + match &self.session_runtime_cache { + Some(cache) => cache.produce_runtime(session_id, config), + None => (self.runtime_producer)(config), + } + } + /// Creates a default [`SessionConfig`] using the configured config producer. pub fn produce_config(&self) -> SessionConfig { (self.config_producer)() @@ -269,6 +301,9 @@ impl Executor { mod test { use crate::execution_engine::{DefaultQueryStageExec, ShuffleWriterVariant}; use crate::executor::Executor; + use crate::runtime_cache::{ + DefaultSessionRuntimeCache, MemoryPoolPolicy, SessionRuntimeCache, + }; use ballista_core::RuntimeProducer; use ballista_core::execution_plans::ShuffleWriterExec; use ballista_core::serde::protobuf::ExecutorRegistration; @@ -278,11 +313,13 @@ mod test { use datafusion::arrow::record_batch::RecordBatch; use datafusion::error::{DataFusionError, Result}; use datafusion::execution::context::TaskContext; + use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; + use datafusion::prelude::SessionConfig; use datafusion::prelude::SessionContext; use futures::Stream; use std::pin::Pin; @@ -459,4 +496,72 @@ mod test { let inner_result = result.unwrap().unwrap(); assert!(inner_result.is_err()); } + + #[test] + fn produce_runtime_for_session_shares_read_side_state() { + let executor_registration = ExecutorRegistration { + id: "executor".to_string(), + port: 0, + grpc_port: 0, + specification: None, + host: None, + os_info: None, + }; + let config_producer = Arc::new(default_config_producer); + + // A base producer that builds a fresh env each call, plus an identity + // pool policy, so shared read-side state is observable via ptr equality. + let base_producer: RuntimeProducer = + Arc::new(|_| Ok(Arc::new(RuntimeEnv::default()))); + let identity: MemoryPoolPolicy = Arc::new(|base, _| Ok(base)); + let cache: Arc = Arc::new( + DefaultSessionRuntimeCache::new(base_producer.clone(), identity, 4), + ); + + let executor = Executor::new_basic( + executor_registration, + "/tmp", + base_producer, + config_producer, + 2, + ) + .with_session_runtime_cache(Some(cache)); + + let cfg = SessionConfig::new(); + let e1 = executor.produce_runtime_for_session("s1", &cfg).unwrap(); + let e2 = executor.produce_runtime_for_session("s1", &cfg).unwrap(); + let e3 = executor.produce_runtime_for_session("s2", &cfg).unwrap(); + + assert!(Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager)); + assert!(!Arc::ptr_eq(&e1.cache_manager, &e3.cache_manager)); + } + + #[test] + fn produce_runtime_for_session_falls_back_without_cache() { + let executor_registration = ExecutorRegistration { + id: "executor".to_string(), + port: 0, + grpc_port: 0, + specification: None, + host: None, + os_info: None, + }; + let config_producer = Arc::new(default_config_producer); + let base_producer: RuntimeProducer = + Arc::new(|_| Ok(Arc::new(RuntimeEnv::default()))); + + // No cache attached: each call builds a fresh env. + let executor = Executor::new_basic( + executor_registration, + "/tmp", + base_producer, + config_producer, + 2, + ); + + let cfg = SessionConfig::new(); + let e1 = executor.produce_runtime_for_session("s1", &cfg).unwrap(); + let e2 = executor.produce_runtime_for_session("s1", &cfg).unwrap(); + assert!(!Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager)); + } } diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 4e37687e81..360a894913 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -42,7 +42,7 @@ use tokio::{fs, time}; use uuid::Uuid; use datafusion::execution::memory_pool::{FairSpillPool, MemoryPool}; -use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder}; use datafusion::prelude::SessionConfig; use ballista_core::config::{LogRotationPolicy, TaskSchedulingPolicy}; @@ -70,36 +70,46 @@ use crate::executor_server::TERMINATING; use crate::flight_service::BallistaFlightService; use crate::metrics::ExecutorMetricCollectionPolicy; use crate::metrics::LoggingMetricsCollector; +use crate::runtime_cache::{ + DefaultSessionRuntimeCache, MemoryPoolPolicy, SessionRuntimeCache, +}; use crate::shutdown::Shutdown; use crate::shutdown::ShutdownNotifier; use crate::{ArrowFlightServerProvider, terminate}; use crate::{execution_loop, executor_server}; -/// Wrap a [`RuntimeProducer`] so that every produced -/// [`RuntimeEnv`](datafusion::execution::runtime_env::RuntimeEnv) carries a -/// fresh [`FairSpillPool`] of size `total_bytes / concurrent_tasks`. +/// Builds a per-task memory-pool policy: each task's runtime is rebuilt from the +/// shared base env with a fresh [`FairSpillPool`] of size +/// `total_bytes / concurrent_tasks`. The base env's disk manager, cache manager, +/// and object-store registry are preserved via +/// [`RuntimeEnvBuilder::from_runtime_env`]. /// /// Returns an error if the per-task share would be zero (i.e. `total_bytes < -/// concurrent_tasks`). The inner env's disk manager, cache manager, and -/// object store registry are preserved via [`RuntimeEnvBuilder::from_runtime_env`]. -fn wrap_runtime_producer_with_memory_pool( - inner: RuntimeProducer, +/// concurrent_tasks`). +fn memory_pool_policy( total_bytes: u64, concurrent_tasks: usize, -) -> Result { +) -> Result { let per_task = (total_bytes / concurrent_tasks as u64) as usize; if per_task == 0 { return Err(BallistaError::Configuration(format!( "memory_pool_size ({total_bytes} bytes) is smaller than concurrent_tasks ({concurrent_tasks})" ))); } - Ok(Arc::new(move |session_config: &SessionConfig| { - let inner_env = inner(session_config)?; - let pool: Arc = Arc::new(FairSpillPool::new(per_task)); - RuntimeEnvBuilder::from_runtime_env(&inner_env) - .with_memory_pool(pool) - .build_arc() - })) + Ok(Arc::new( + move |base: Arc, _config: &SessionConfig| { + let pool: Arc = Arc::new(FairSpillPool::new(per_task)); + RuntimeEnvBuilder::from_runtime_env(&base) + .with_memory_pool(pool) + .build_arc() + }, + )) +} + +/// A no-op policy: the task uses the shared base env unchanged (DataFusion's +/// default unbounded pool). Used when `--memory-pool-size` is unset. +fn identity_pool_policy() -> MemoryPoolPolicy { + Arc::new(|base, _config| Ok(base)) } /// Configuration for the executor process. @@ -155,6 +165,11 @@ pub struct ExecutorProcessConfig { /// `memory_pool_size / concurrent_tasks`. When `None`, no pool is /// installed and DataFusion falls back to its unbounded default. pub memory_pool_size: Option, + /// Maximum number of sessions whose shared base runtime env is retained on + /// the executor (LRU). Sharing reuses object-store clients and the Parquet + /// footer cache across a session's tasks and queries. `0` disables caching + /// and builds a fresh runtime per task. + pub session_runtime_cache_capacity: usize, /// Optional execution engine to use to execute physical plans, will default to /// DataFusion if none is provided. pub override_execution_engine: Option>, @@ -214,6 +229,7 @@ impl Default for ExecutorProcessConfig { executor_heartbeat_interval_seconds: 60, metric_collection_policy: ExecutorMetricCollectionPolicy::default(), memory_pool_size: None, + session_runtime_cache_capacity: 16, override_execution_engine: None, override_function_registry: None, override_runtime_producer: None, @@ -290,7 +306,7 @@ pub async fn start_executor_process( .unwrap_or_else(|| Arc::new(default_config_producer)); let wd = work_dir.clone(); - let runtime_producer: RuntimeProducer = + let base_runtime_producer: RuntimeProducer = opt.override_runtime_producer.clone().unwrap_or_else(|| { Arc::new(move |_| { let runtime_env = RuntimeEnvBuilder::new() @@ -300,19 +316,24 @@ pub async fn start_executor_process( }) }); - let runtime_producer = if let Some(total) = opt.memory_pool_size { - let producer = wrap_runtime_producer_with_memory_pool( - runtime_producer, - total, - concurrent_tasks, - )?; + let pool_policy: MemoryPoolPolicy = if let Some(total) = opt.memory_pool_size { + let policy = memory_pool_policy(total, concurrent_tasks)?; let per_task = total / concurrent_tasks as u64; info!( "Memory pool: total {total} bytes split into {concurrent_tasks} tasks ({per_task} bytes each)" ); - producer + policy } else { - runtime_producer + identity_pool_policy() + }; + + // Combined producer preserving the current per-task behavior: build a fresh + // base env, then apply the pool policy. Used by `Executor::produce_runtime` + // and as the fallback when session caching is disabled. + let runtime_producer: RuntimeProducer = { + let base = base_runtime_producer.clone(); + let policy = pool_policy.clone(); + Arc::new(move |config: &SessionConfig| policy(base(config)?, config)) }; let logical = opt @@ -330,26 +351,43 @@ pub async fn start_executor_process( datafusion_proto::protobuf::PhysicalPlanNode, > = BallistaCodec::new(logical, physical); - let executor = Arc::new(Executor::new( - executor_meta.clone(), - &work_dir, - runtime_producer, - config_producer, - opt.override_function_registry.clone().unwrap_or_default(), - metrics_collector, - concurrent_tasks, - opt.override_execution_engine.clone().unwrap_or_else(|| { - if opt.client_ttl > 0 { - let client_pool = - Arc::new(DefaultBallistaClientPool::with_eviction_thread( - Duration::from_secs(opt.client_ttl), - )); - Arc::new(DefaultExecutionEngine::with_client_pool(client_pool)) - } else { - Arc::new(DefaultExecutionEngine::new()) - } - }), - )); + // Session caching applies only to the default runtime producer. With an + // override producer we cannot split base + pool, so we serve per-task as + // before by leaving the cache unset. + let session_runtime_cache: Option> = + if opt.override_runtime_producer.is_none() { + Some(Arc::new(DefaultSessionRuntimeCache::new( + base_runtime_producer.clone(), + pool_policy.clone(), + opt.session_runtime_cache_capacity, + ))) + } else { + None + }; + + let executor = Arc::new( + Executor::new( + executor_meta.clone(), + &work_dir, + runtime_producer, + config_producer, + opt.override_function_registry.clone().unwrap_or_default(), + metrics_collector, + concurrent_tasks, + opt.override_execution_engine.clone().unwrap_or_else(|| { + if opt.client_ttl > 0 { + let client_pool = + Arc::new(DefaultBallistaClientPool::with_eviction_thread( + Duration::from_secs(opt.client_ttl), + )); + Arc::new(DefaultExecutionEngine::with_client_pool(client_pool)) + } else { + Arc::new(DefaultExecutionEngine::new()) + } + }), + ) + .with_session_runtime_cache(session_runtime_cache), + ); let connect_timeout = opt.scheduler_connect_timeout_seconds as u64; let session_config = (executor.config_producer)(); @@ -1064,14 +1102,9 @@ mod memory_pool_tests { use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder}; use std::sync::Arc; - fn baseline_producer() -> RuntimeProducer { - Arc::new(|_| Ok(Arc::new(RuntimeEnv::default()))) - } - #[test] fn returns_error_when_total_smaller_than_concurrent_tasks() { - let inner = baseline_producer(); - let result = wrap_runtime_producer_with_memory_pool(inner, 4, 8); + let result = memory_pool_policy(4, 8); assert!(result.is_err()); let msg = result.err().unwrap().to_string(); assert!(msg.contains("memory_pool_size")); @@ -1084,13 +1117,9 @@ mod memory_pool_tests { let concurrent = 8usize; let expected_per_task = (total / concurrent as u64) as usize; - let wrapped = wrap_runtime_producer_with_memory_pool( - baseline_producer(), - total, - concurrent, - ) - .unwrap(); - let env = wrapped(&SessionConfig::new()).unwrap(); + let policy = memory_pool_policy(total, concurrent).unwrap(); + let base = Arc::new(RuntimeEnv::default()); + let env = policy(base, &SessionConfig::new()).unwrap(); match env.memory_pool.memory_limit() { MemoryLimit::Finite(n) => assert_eq!(n, expected_per_task), @@ -1100,20 +1129,26 @@ mod memory_pool_tests { } #[test] - fn preserves_inner_object_store_registry() { + fn preserves_base_object_store_registry() { let registry: Arc = Arc::new(DefaultObjectStoreRegistry::new()); - let registry_for_inner = registry.clone(); - let inner: RuntimeProducer = Arc::new(move |_| { - let env = RuntimeEnvBuilder::new() - .with_object_store_registry(registry_for_inner.clone()) - .build()?; - Ok(Arc::new(env)) - }); + let base = Arc::new( + RuntimeEnvBuilder::new() + .with_object_store_registry(registry.clone()) + .build() + .unwrap(), + ); - let wrapped = wrap_runtime_producer_with_memory_pool(inner, 1024, 1).unwrap(); - let env = wrapped(&SessionConfig::new()).unwrap(); + let policy = memory_pool_policy(1024, 1).unwrap(); + let env = policy(base, &SessionConfig::new()).unwrap(); assert!(Arc::ptr_eq(&env.object_store_registry, ®istry)); } + + #[test] + fn identity_policy_returns_base_unchanged() { + let base = Arc::new(RuntimeEnv::default()); + let env = identity_pool_policy()(base.clone(), &SessionConfig::new()).unwrap(); + assert!(Arc::ptr_eq(&env, &base)); + } } diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs index e65dbb0e0c..a78b006e96 100644 --- a/ballista/executor/src/executor_server.rs +++ b/ballista/executor/src/executor_server.rs @@ -386,7 +386,9 @@ impl ExecutorServer { diff --git a/ballista/executor/src/lib.rs b/ballista/executor/src/lib.rs index f0d0f995ec..70234ceaa5 100644 --- a/ballista/executor/src/lib.rs +++ b/ballista/executor/src/lib.rs @@ -39,6 +39,8 @@ pub mod executor_server; pub mod flight_service; /// Metrics collection for executor runtime statistics. pub mod metrics; +/// Session-scoped cache of shared executor runtime environments. +pub mod runtime_cache; /// Graceful shutdown coordination for executor components. pub mod shutdown; /// Signal handling for process termination. diff --git a/ballista/executor/src/runtime_cache.rs b/ballista/executor/src/runtime_cache.rs new file mode 100644 index 0000000000..2cc0e05fb9 --- /dev/null +++ b/ballista/executor/src/runtime_cache.rs @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Session-scoped reuse of executor runtime state. + +use std::num::NonZeroUsize; +use std::sync::Arc; + +use ballista_core::RuntimeProducer; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::prelude::SessionConfig; +use lru::LruCache; +use parking_lot::Mutex; + +/// Derives a task's runtime from a shared base [`RuntimeEnv`] by installing a +/// fresh per-task memory pool. The base's object-store registry and the cache +/// manager's underlying file-metadata (footer) cache are preserved via +/// [`RuntimeEnvBuilder::from_runtime_env`](datafusion::execution::runtime_env::RuntimeEnvBuilder::from_runtime_env) +/// — the outer `CacheManager` is rebuilt around that same inner cache — so +/// read-side state is shared while the memory pool stays per task. +pub type MemoryPoolPolicy = Arc< + dyn Fn(Arc, &SessionConfig) -> datafusion::error::Result> + + Send + + Sync, +>; + +/// Produces a task's [`RuntimeEnv`], optionally reusing read-side state across +/// the tasks of a session. +/// +/// The executor calls [`produce_runtime`](Self::produce_runtime) for every task; +/// implementors decide whether and how to share base runtime state (object-store +/// clients, file-metadata cache) between a session's tasks. +/// [`DefaultSessionRuntimeCache`] is the built-in implementation — provide a +/// custom one to change the caching/sharing strategy. +pub trait SessionRuntimeCache: Send + Sync { + /// Returns the per-task runtime for `session_id`. + fn produce_runtime( + &self, + session_id: &str, + config: &SessionConfig, + ) -> datafusion::error::Result>; +} + +/// A bounded, session-keyed cache of shared *base* [`RuntimeEnv`]s. +/// +/// A base env carries the read-side state safe to share across all tasks of a +/// session: the object-store registry and the cache manager (whose Parquet +/// footer cache is thereby reused across the session's tasks and queries). +/// `RuntimeEnvBuilder::from_runtime_env` also carries over the disk manager +/// (rooted at the executor's `work_dir`), so a session's tasks share one +/// `DiskManager` too; this is safe because spill temp files are uniquely +/// named, matching the standard one-`RuntimeEnv`-per-`SessionContext` model. +/// Each task's real runtime is produced by applying [`MemoryPoolPolicy`] to +/// the shared base, which installs a fresh per-task memory pool — so memory +/// isolation is unchanged. +/// +/// The cache is bounded by an LRU of `capacity` sessions. A capacity of `0` +/// disables caching entirely: every call builds a fresh base env, matching the +/// behavior of building a runtime per task. +pub struct DefaultSessionRuntimeCache { + base_producer: RuntimeProducer, + pool_policy: MemoryPoolPolicy, + cache: Option>>>, +} + +impl DefaultSessionRuntimeCache { + /// Creates a new cache that produces per-task runtimes from `base_producer` + /// (invoked at most once per cached session) and `pool_policy` (invoked on + /// every call). `capacity` bounds the number of distinct sessions kept in + /// the LRU; `0` disables caching. + pub fn new( + base_producer: RuntimeProducer, + pool_policy: MemoryPoolPolicy, + capacity: usize, + ) -> Self { + let cache = NonZeroUsize::new(capacity).map(|cap| Mutex::new(LruCache::new(cap))); + Self { + base_producer, + pool_policy, + cache, + } + } +} + +impl SessionRuntimeCache for DefaultSessionRuntimeCache { + /// Returns the per-task runtime for `session_id`, reusing a cached base env + /// when present and building + caching one on miss. + fn produce_runtime( + &self, + session_id: &str, + config: &SessionConfig, + ) -> datafusion::error::Result> { + let base = match &self.cache { + None => (self.base_producer)(config)?, + Some(cache) => { + if let Some(base) = cache.lock().get(session_id) { + base.clone() + } else { + // Build the base env without holding the lock, so a miss + // never stalls other sessions' lookups. A rare concurrent + // first-miss for the same session may build twice; that is + // harmless (idempotent, cheap) — the last writer wins and + // the extra env is dropped. + let base = (self.base_producer)(config)?; + cache.lock().put(session_id.to_string(), base.clone()); + base + } + } + }; + (self.pool_policy)(base, config) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::execution::memory_pool::GreedyMemoryPool; + use datafusion::execution::runtime_env::RuntimeEnvBuilder; + + fn base_producer() -> RuntimeProducer { + Arc::new(|_| Ok(Arc::new(RuntimeEnv::default()))) + } + + /// Identity policy: the task shares the base env unchanged (no pool swap). + fn identity_policy() -> MemoryPoolPolicy { + Arc::new(|base, _| Ok(base)) + } + + /// Rebuilding policy: mimics production — a fresh per-task pool layered onto + /// the shared base, preserving the base's read-side state. + fn per_task_pool_policy() -> MemoryPoolPolicy { + Arc::new(|base, _| { + RuntimeEnvBuilder::from_runtime_env(&base) + .with_memory_pool(Arc::new(GreedyMemoryPool::new(1024))) + .build_arc() + }) + } + + #[test] + fn same_session_shares_cache_manager() { + let cache = + DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 4); + let cfg = SessionConfig::new(); + let e1 = cache.produce_runtime("s1", &cfg).unwrap(); + let e2 = cache.produce_runtime("s1", &cfg).unwrap(); + assert!(Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager)); + } + + #[test] + fn different_sessions_get_different_base() { + let cache = + DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 4); + let cfg = SessionConfig::new(); + let e1 = cache.produce_runtime("s1", &cfg).unwrap(); + let e2 = cache.produce_runtime("s2", &cfg).unwrap(); + assert!(!Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager)); + } + + #[test] + fn per_task_pool_shares_footer_cache_but_not_env() { + let cache = + DefaultSessionRuntimeCache::new(base_producer(), per_task_pool_policy(), 4); + let cfg = SessionConfig::new(); + let e1 = cache.produce_runtime("s1", &cfg).unwrap(); + let e2 = cache.produce_runtime("s1", &cfg).unwrap(); + // Different runtime envs (fresh per-task pool)... + assert!(!Arc::ptr_eq(&e1, &e2)); + // ...but the shared read-side state is reused: object-store registry + // passes through unchanged, and the inner footer (file-metadata) cache + // is the same instance even though the outer CacheManager wrapper is + // rebuilt. Do NOT compare the outer Arc. + assert!(Arc::ptr_eq( + &e1.object_store_registry, + &e2.object_store_registry + )); + assert!(Arc::ptr_eq( + &e1.cache_manager.get_file_metadata_cache(), + &e2.cache_manager.get_file_metadata_cache(), + )); + } + + #[test] + fn capacity_zero_disables_cache() { + let cache = + DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 0); + let cfg = SessionConfig::new(); + let e1 = cache.produce_runtime("s1", &cfg).unwrap(); + let e2 = cache.produce_runtime("s1", &cfg).unwrap(); + assert!(!Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager)); + } + + #[test] + fn evicts_least_recently_used() { + let cache = + DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 2); + let cfg = SessionConfig::new(); + let s1_a = cache.produce_runtime("s1", &cfg).unwrap(); + cache.produce_runtime("s2", &cfg).unwrap(); + // Inserting s3 (capacity 2) evicts the least-recently-used, s1. + cache.produce_runtime("s3", &cfg).unwrap(); + let s1_b = cache.produce_runtime("s1", &cfg).unwrap(); + assert!(!Arc::ptr_eq(&s1_a.cache_manager, &s1_b.cache_manager)); + } +} diff --git a/python/Cargo.lock b/python/Cargo.lock index ca63ef385e..e7d0ef3c93 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -548,6 +548,7 @@ dependencies = [ "futures", "libc", "log", + "lru", "memory-stats", "parking_lot", "serde", @@ -1847,7 +1848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2617,6 +2618,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -3504,7 +3514,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3784,7 +3794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3859,7 +3869,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3938,10 +3948,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4545,7 +4555,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From d54b48f360fde6666c2509cfd26809ea92ff8f20 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 12 Jul 2026 09:43:15 -0600 Subject: [PATCH 065/107] chore: bump Ballista version to 54.0.0 (#2004) * chore: bump Ballista version to 54.0.0 * chore: update python/Cargo.lock for 54.0.0 --- Cargo.lock | 14 +++++++------- ballista-cli/Cargo.toml | 4 ++-- ballista/client/Cargo.toml | 12 ++++++------ ballista/core/Cargo.toml | 2 +- ballista/executor/Cargo.toml | 4 ++-- ballista/scheduler/Cargo.toml | 4 ++-- benchmarks/Cargo.toml | 6 +++--- examples/Cargo.toml | 10 +++++----- python/Cargo.lock | 10 +++++----- python/Cargo.toml | 10 +++++----- 10 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b918eb711a..43ea105778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1002,7 +1002,7 @@ dependencies = [ [[package]] name = "ballista" -version = "53.0.0" +version = "54.0.0" dependencies = [ "async-trait", "ballista-core", @@ -1023,7 +1023,7 @@ dependencies = [ [[package]] name = "ballista-benchmarks" -version = "53.0.0" +version = "54.0.0" dependencies = [ "ballista", "ballista-core", @@ -1044,7 +1044,7 @@ dependencies = [ [[package]] name = "ballista-cli" -version = "53.0.0" +version = "54.0.0" dependencies = [ "ballista", "chrono", @@ -1083,7 +1083,7 @@ dependencies = [ [[package]] name = "ballista-core" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow-flight", "async-trait", @@ -1119,7 +1119,7 @@ dependencies = [ [[package]] name = "ballista-examples" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow-flight", "ballista", @@ -1145,7 +1145,7 @@ dependencies = [ [[package]] name = "ballista-executor" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow", "arrow-flight", @@ -1178,7 +1178,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow-flight", "async-trait", diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml index 683a3c2453..e9aaa9863e 100644 --- a/ballista-cli/Cargo.toml +++ b/ballista-cli/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "ballista-cli" description = "Command Line Client for Ballista distributed query engine." -version = "53.0.0" +version = "54.0.0" authors = ["Apache DataFusion "] edition = { workspace = true } rust-version = { workspace = true } @@ -33,7 +33,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # CLI-only deps (DataFusion/Ballista) — gated by `cli` feature -ballista = { path = "../ballista/client", version = "53.0.0", features = ["standalone"], optional = true } +ballista = { path = "../ballista/client", version = "54.0.0", features = ["standalone"], optional = true } datafusion = { workspace = true, optional = true } datafusion-cli = { workspace = true, optional = true } rustyline = { version = "18.0.0", optional = true } diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml index f3d9d2e3e8..062b3435d2 100644 --- a/ballista/client/Cargo.toml +++ b/ballista/client/Cargo.toml @@ -19,7 +19,7 @@ name = "ballista" description = "Ballista Distributed Compute" license = "Apache-2.0" -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" readme = "README.md" @@ -29,9 +29,9 @@ rust-version = { workspace = true } [dependencies] async-trait = { workspace = true } -ballista-core = { path = "../core", version = "53.0.0" } -ballista-executor = { path = "../executor", version = "53.0.0", optional = true, default-features = false, features = ["arrow-ipc-optimizations"] } -ballista-scheduler = { path = "../scheduler", version = "53.0.0", optional = true, default-features = false } +ballista-core = { path = "../core", version = "54.0.0" } +ballista-executor = { path = "../executor", version = "54.0.0", optional = true, default-features = false, features = ["arrow-ipc-optimizations"] } +ballista-scheduler = { path = "../scheduler", version = "54.0.0", optional = true, default-features = false } datafusion = { workspace = true } log = { workspace = true } @@ -39,8 +39,8 @@ tokio = { workspace = true } url = { workspace = true } [dev-dependencies] -ballista-executor = { path = "../executor", version = "53.0.0" } -ballista-scheduler = { path = "../scheduler", version = "53.0.0" } +ballista-executor = { path = "../executor", version = "54.0.0" } +ballista-scheduler = { path = "../scheduler", version = "54.0.0" } ctor = { workspace = true } datafusion-proto = { workspace = true } env_logger = { workspace = true } diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml index 914c1eed3e..7359183906 100644 --- a/ballista/core/Cargo.toml +++ b/ballista/core/Cargo.toml @@ -19,7 +19,7 @@ name = "ballista-core" description = "Ballista Distributed Compute" license = "Apache-2.0" -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" readme = "README.md" diff --git a/ballista/executor/Cargo.toml b/ballista/executor/Cargo.toml index aa60cb2fd1..caae3b4260 100644 --- a/ballista/executor/Cargo.toml +++ b/ballista/executor/Cargo.toml @@ -19,7 +19,7 @@ name = "ballista-executor" description = "Ballista Distributed Compute - Executor" license = "Apache-2.0" -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" readme = "README.md" @@ -42,7 +42,7 @@ spark-compat = ["ballista-core/spark-compat"] arrow = { workspace = true } arrow-flight = { workspace = true } async-trait = { workspace = true } -ballista-core = { path = "../core", version = "53.0.0" } +ballista-core = { path = "../core", version = "54.0.0" } bytesize = "2" clap = { workspace = true, optional = true } dashmap = { workspace = true } diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index 92339a5c50..d2012b3469 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -19,7 +19,7 @@ name = "ballista-scheduler" description = "Ballista Distributed Compute - Scheduler" license = "Apache-2.0" -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" readme = "README.md" @@ -50,7 +50,7 @@ arrow-flight = { workspace = true } async-trait = { workspace = true } axum = "0.8.9" tower-http = { version = "0.7", features = ["cors"] } -ballista-core = { path = "../core", version = "53.0.0" } +ballista-core = { path = "../core", version = "54.0.0" } clap = { workspace = true, optional = true } dashmap = { workspace = true } datafusion = { workspace = true } diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index ab969b4777..c107aaee5d 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "ballista-benchmarks" description = "Ballista Benchmarks" -version = "53.0.0" +version = "54.0.0" edition = "2024" authors = ["Apache DataFusion "] homepage = "https://datafusion.apache.org/ballista/" @@ -31,8 +31,8 @@ ci = [] default = ["mimalloc"] [dependencies] -ballista = { path = "../ballista/client", version = "53.0.0" } -ballista-core = { path = "../ballista/core", version = "53.0.0", features = [ +ballista = { path = "../ballista/client", version = "54.0.0" } +ballista-core = { path = "../ballista/core", version = "54.0.0", features = [ "build-binary", ] } clap = { workspace = true } diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 2f6e55451b..531ce36565 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "ballista-examples" description = "Ballista usage examples" -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" authors = ["Apache DataFusion "] @@ -57,10 +57,10 @@ rustls = { version = "0.23", features = ["ring"], optional = true } [dev-dependencies] arrow-flight = { workspace = true } -ballista = { path = "../ballista/client", version = "53.0.0" } -ballista-core = { path = "../ballista/core", version = "53.0.0", default-features = false, features = ["build-binary"] } -ballista-executor = { path = "../ballista/executor", version = "53.0.0", default-features = false } -ballista-scheduler = { path = "../ballista/scheduler", version = "53.0.0", default-features = false } +ballista = { path = "../ballista/client", version = "54.0.0" } +ballista-core = { path = "../ballista/core", version = "54.0.0", default-features = false, features = ["build-binary"] } +ballista-executor = { path = "../ballista/executor", version = "54.0.0", default-features = false } +ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", default-features = false } ctor = { workspace = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } diff --git a/python/Cargo.lock b/python/Cargo.lock index e7d0ef3c93..64c9566c7e 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -491,7 +491,7 @@ dependencies = [ [[package]] name = "ballista" -version = "53.0.0" +version = "54.0.0" dependencies = [ "async-trait", "ballista-core", @@ -505,7 +505,7 @@ dependencies = [ [[package]] name = "ballista-core" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow-flight", "async-trait", @@ -535,7 +535,7 @@ dependencies = [ [[package]] name = "ballista-executor" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow", "arrow-flight", @@ -563,7 +563,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow-flight", "async-trait", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "pyballista" -version = "53.0.0" +version = "54.0.0" dependencies = [ "async-trait", "ballista", diff --git a/python/Cargo.toml b/python/Cargo.toml index 47b3f9d39d..97a2bead29 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "pyballista" -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" authors = ["Apache DataFusion "] @@ -31,10 +31,10 @@ publish = false [dependencies] async-trait = "0.1.89" -ballista = { path = "../ballista/client", version = "53.0.0" } -ballista-core = { path = "../ballista/core", version = "53.0.0" } -ballista-executor = { path = "../ballista/executor", version = "53.0.0", default-features = false } -ballista-scheduler = { path = "../ballista/scheduler", version = "53.0.0", default-features = false } +ballista = { path = "../ballista/client", version = "54.0.0" } +ballista-core = { path = "../ballista/core", version = "54.0.0" } +ballista-executor = { path = "../ballista/executor", version = "54.0.0", default-features = false } +ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", default-features = false } datafusion = { version = "54", features = ["avro"] } datafusion-proto = { version = "54" } datafusion-python = { version = "54", default-features = false } From acede27396fa73fc690c5aa77fb40f82ce838871 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 12 Jul 2026 10:51:13 -0600 Subject: [PATCH 066/107] docs: add user personas as a review contract for additive changes (#2002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: add user personas as a review contract for additive changes Ballista serves several distinct audiences, each depending on a different set of guarantees: DataFusion users going multi-node who want transparent distribution with identical results, Spark users who want the stage/task execution model and AQE exposed and tunable, and library users embedding Ballista as the foundation for a specialized engine who depend on stable, composable APIs and extension points. Add a User Personas page to the Contributors Guide that names these audiences and, for each, records the capabilities it relies on and the red flags that would regress it. The registry is a review contract: changes must be additive, so reviewers can check pull requests against it to avoid silently taking functionality away from an existing audience. * docs: add "Who is Ballista for" section to the README Summarize the three audiences Ballista serves — DataFusion users going multi-node, Spark users wanting the same execution model, and library users building a specialized engine — and link to the full User Personas guide for the guarantees each relies on. --- README.md | 10 ++ .../contributors-guide/user-personas.md | 148 ++++++++++++++++++ docs/source/index.rst | 1 + 3 files changed, 159 insertions(+) create mode 100644 docs/source/contributors-guide/user-personas.md diff --git a/README.md b/README.md index fe0c204851..99f3a30332 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,16 @@ async fn main() -> datafusion::error::Result<()> { For documentation or more examples, please refer to the [Ballista User Guide][user-guide]. +## Who is Ballista for + +Ballista serves several distinct audiences: + +- **DataFusion users going multi-node** — you already use [Apache DataFusion](https://github.com/apache/datafusion) on a single machine and have outgrown it. Ballista runs the same SQL and DataFrame workloads across a cluster with minimal code changes and the same results. +- **Spark users wanting the same execution model** — you run Spark SQL or batch jobs and want a lighter, Rust-native alternative without relearning a new paradigm. Ballista keeps the familiar model: plans split into stages at shuffle boundaries, one task per partition, executors with task slots, and adaptive query execution (AQE). +- **Library users building a specialized engine** — you are building a bespoke distributed query engine and want reusable scheduler, executor, and plan-serialization building blocks with extension points, instead of writing distributed execution from scratch. + +These audiences are documented in more detail, along with the guarantees each relies on, in the [User Personas](docs/source/contributors-guide/user-personas.md) guide. + ## Architecture A Ballista cluster consists of one or more scheduler processes and one or more executor processes. These processes diff --git a/docs/source/contributors-guide/user-personas.md b/docs/source/contributors-guide/user-personas.md new file mode 100644 index 0000000000..a54d038548 --- /dev/null +++ b/docs/source/contributors-guide/user-personas.md @@ -0,0 +1,148 @@ + + +# User Personas + +Ballista serves several distinct audiences, each of which comes to the project +for a different reason and depends on a different set of capabilities. This page +names those audiences as **personas** and, for each one, records the guarantees +that audience relies on. + +## Why this page exists + +This is a review contract, not marketing copy. It exists so that changes to +Ballista stay **additive**: + +- A pull request may **add** a new persona, or **extend** an existing one with + new capabilities. +- A pull request must **not remove or silently regress** a capability that an + existing persona depends on. + +Reviewers evaluate every non-trivial pull request against this list. Each +persona lists concrete "red flags" — the kinds of change that would quietly take +functionality away from that audience. A red flag does not mean a change is +forbidden, but it does mean the change needs explicit discussion and maintainer +sign-off, and usually a migration path (for example an entry in the +[upgrade guide](../upgrading/index)), rather than landing silently. + +These personas are deliberately in tension with one another. One audience wants +distributed execution to be a transparent implementation detail; another wants +the execution model exposed and tunable; another wants Ballista's internals to +be a stable, composable library. The point of keeping them all written down is +to make sure a change that delights one audience does not quietly break another. + +## Persona 1: The DataFusion user going multi-node + +- **Who**: A data engineer or analyst already using + [DataFusion](https://datafusion.apache.org/) single-process (often through the + Python bindings) who has outgrown a single machine. +- **Coming from**: DataFusion, single process. +- **Why Ballista**: Run the same SQL and DataFrame workloads across a cluster + with minimal code change and _identical results_. +- **Depends on** (must not regress): + - Query results and semantics match single-process DataFusion for the same + input. + - A client surface that mirrors DataFusion's `SessionContext` / DataFrame / SQL + API. + - `datafusion.*` session configuration overrides are honored end-to-end, so + users tune DataFusion the way they already do. + - Standard object stores and file formats work and produce correct results + across partitions. + - Distributed execution is _transparent_: getting a correct answer never + requires the user to reason about stages, tasks, or partitioning. +- **Red flags in a pull request**: + - Results diverge from single-process DataFusion for the same input. + - `datafusion.*` overrides are dropped or ignored during scheduling/planning. + - A DataFusion feature that is correct single-node becomes silently wrong when + the plan is split across stages (for example dynamic filter pushdown, + uncorrelated scalar subqueries, or a scan reading more than its assigned + partition). + - Client API changes that break the DataFusion-mirroring surface. + +## Persona 2: The Spark user wanting the same execution model + +- **Who**: A data engineer or platform owner running Spark SQL / batch jobs who + wants a lighter, Rust-native alternative without relearning a new execution + paradigm. +- **Coming from**: Apache Spark. +- **Why Ballista**: Keep the Spark mental model — plans split into **stages** at + shuffle boundaries, one **task** per partition, shuffle files, executors with + task slots, and adaptive query execution (**AQE**) for runtime adaptivity — on + top of DataFusion and Arrow. +- **Depends on** (must not regress): + - The stage/task execution model at shuffle boundaries. + - The AQE / adaptive planner path (partition coalescing, dynamic join + selection, runtime broadcast), available and gated behind configuration. + - Broadcast joins for small build sides, to avoid shuffles and skew (the + equivalent of Spark's broadcast hash join). + - The operational surface Spark users expect: a scheduler plus executors, + task-slot concurrency, a tunable `target_partitions`, and stage/task + observability (task timings, a history server / UI) for debugging skew. + - Behavior driven by configuration and `SET`, not hard-coded planner choices. +- **Red flags in a pull request**: + - Removing or short-circuiting the stage/task boundary model, or collapsing + toward single-process execution. + - Regressing or removing the AQE / adaptive planner path — or making it the + only path in a way that breaks Persona 1's transparent static path. + - Hard-coding a join or planner strategy instead of gating it behind + configuration. + - Removing the stage/task observability that Spark users rely on to diagnose + skew. + +## Persona 3: The library user building a specialized engine + +- **Who**: An engineer using Ballista as a _framework_ — the foundation for a + specialized or bespoke distributed query engine — rather than as a turnkey + cluster. +- **Coming from**: Building on DataFusion as a library, now needing distribution + (or replacing a hand-rolled distributed layer). +- **Why Ballista**: Reusable scheduler, executor, and plan-serialization + building blocks with extension points, instead of writing distributed + execution from scratch. +- **Depends on** (must not regress): + - Stable, composable public APIs across the `scheduler`, `executor`, `client`, + and `core` crates (the execution graph and plan serialization included). + - Extension points that do not require forking: custom object stores, + physical/logical extension codecs, custom operators and functions, a + pluggable runtime producer, custom session configuration, and a custom query + planner. + - Behavior that stays _configurable rather than hard-coded_, so an embedder can + swap the planner, join strategy, partitioning, or runtime environment. + - Backward-compatible public APIs, with breaking changes signaled in the + [upgrade guide](../upgrading/index). + - No assumptions baked into the core that only fit the built-in clients or the + TPC-H benchmark. +- **Red flags in a pull request**: + - Breaking a public API signature without an upgrade-guide entry. + - Hard-coding behavior that used to be pluggable, or removing an extension + point (an extension-codec argument, the runtime-producer override, + session-config injection). + - Coupling core logic to a specific client, planner, or benchmark instead of + keeping it behind a trait or configuration. + - Wire-format or serialization changes that break externally-produced plans + with no compatibility path. + +## Adding a persona + +This registry is append-only. As Ballista attracts new kinds of users, add a +persona for each one via a pull request, using the same schema (who they are, +what they are coming from, why Ballista, what they depend on, and the red flags +that would regress them). Do not remove existing personas: an audience that +relied on Ballista does not stop existing because our priorities shifted, and +the whole value of this page is that the guarantees only accumulate. diff --git a/docs/source/index.rst b/docs/source/index.rst index e66e237528..c208c29ecc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -78,6 +78,7 @@ Table of content contributors-guide/architecture contributors-guide/code-organization + contributors-guide/user-personas contributors-guide/development Source code From ac9bc060bc1fb5e8a54e61bc7545241d6e5612e7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 12 Jul 2026 12:18:36 -0600 Subject: [PATCH 067/107] docs: include TestPyPI wheels link in release vote email (#2010) --- dev/release/create-tarball.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh index f1f0bd2972..63d9f4fe4d 100755 --- a/dev/release/create-tarball.sh +++ b/dev/release/create-tarball.sh @@ -83,6 +83,7 @@ I would like to propose a release of Apache DataFusion Ballista version ${versio This release candidate is based on commit: ${release_hash} [1] The proposed release tarball and signatures are hosted at [2]. The changelog is located at [3]. +The Python wheels built from this release candidate are published to TestPyPI at [4]. Please download, verify checksums and signatures, run the unit tests, and vote on the release. The vote will be open for at least 72 hours. @@ -101,6 +102,7 @@ Here is my vote: +1 [1]: https://github.com/apache/datafusion-ballista/tree/${release_hash} [2]: ${url} [3]: https://github.com/apache/datafusion-ballista/blob/${release_hash}/CHANGELOG.md +[4]: https://test.pypi.org/project/ballista/${version}/ MAIL echo "---------------------------------------------------------" From 51be47517dfd32bf3495c94c195e93727d3c244f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 12 Jul 2026 16:19:33 -0600 Subject: [PATCH 068/107] docs: remove outdated TPC-H benchmark results from READMEs (#2015) --- README.md | 24 ------------------ ballista/client/README.md | 24 ------------------ .../source/_static/images/tpch_allqueries.png | Bin 27455 -> 0 bytes .../_static/images/tpch_queries_compare.png | Bin 32843 -> 0 bytes .../images/tpch_queries_speedup_abs.png | Bin 37391 -> 0 bytes .../images/tpch_queries_speedup_rel.png | Bin 47169 -> 0 bytes 6 files changed, 48 deletions(-) delete mode 100644 docs/source/_static/images/tpch_allqueries.png delete mode 100644 docs/source/_static/images/tpch_queries_compare.png delete mode 100644 docs/source/_static/images/tpch_queries_speedup_abs.png delete mode 100644 docs/source/_static/images/tpch_queries_speedup_rel.png diff --git a/README.md b/README.md index 99f3a30332..a490cd4942 100644 --- a/README.md +++ b/README.md @@ -113,30 +113,6 @@ between the executor(s) and the scheduler for fetching tasks and reporting task See the [architecture guide](docs/source/contributors-guide/architecture.md) for more details. -## Performance - -We run some simple benchmarks comparing Ballista with Apache Spark to track progress with performance optimizations. -These are benchmarks derived from TPC-H and not official TPC-H benchmarks. These results are from running individual -queries at scale factor 100 (100 GB) on a single node with a single executor and 8 concurrent tasks. - -### Overall Speedup - -The overall speedup is 2.9x - -![benchmarks](docs/source/_static/images/tpch_allqueries.png) - -### Per Query Comparison - -![benchmarks](docs/source/_static/images/tpch_queries_compare.png) - -### Relative Speedup - -![benchmarks](docs/source/_static/images/tpch_queries_speedup_rel.png) - -### Absolute Speedup - -![benchmarks](docs/source/_static/images/tpch_queries_speedup_abs.png) - ## Getting Started The easiest way to get started is to run one of the standalone or distributed [examples](./examples/README.md). After diff --git a/ballista/client/README.md b/ballista/client/README.md index 5e94087341..10222a5555 100644 --- a/ballista/client/README.md +++ b/ballista/client/README.md @@ -171,27 +171,3 @@ The output should look similar to the following table. ``` More [examples](../../examples/examples/) can be found in the datafusion-ballista repository. - -## Performance - -We run some simple benchmarks comparing Ballista with Apache Spark to track progress with performance optimizations. - -These are benchmarks derived from TPC-H and not official TPC-H benchmarks. These results are from running individual queries at scale factor 100 (100 GB) on a single node with a single executor and 8 concurrent tasks. - -### Overall Speedup - -The overall speedup is 2.9x - -![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_allqueries.png?raw=true) - -### Per Query Comparison - -![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_queries_compare.png?raw=true) - -### Relative Speedup - -![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_queries_speedup_rel.png?raw=true) - -### Absolute Speedup - -![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_queries_speedup_abs.png?raw=true) diff --git a/docs/source/_static/images/tpch_allqueries.png b/docs/source/_static/images/tpch_allqueries.png deleted file mode 100644 index 5e30bde02f4f89c2a7291d71d0485063776c3ec0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27455 zcmd?RcRZK<`#ydd*=1x!c19$$h(h*gAVraxN+qKtn<8YTkakE>Atf4CgwjHa$S5w zD6vsto=1Aen0UEaH#pPQz8(ux`1ar^Q`NlQmg2s1#-lIJ9=t@8=)dM*SCkf$n5GW? z2x#BTr4tp$A8#d^Mfks3#r0G~czE~L#0TJF5jusSGqfx^8wYGc)=G zk6g;Z?pODHzV9u{d-LW^&dz6CEA}^V7^QLj`0*os`#ss?a|MqN4Gk$*yvujB z6PUbhU)J*D!}8_J<1SsAepl(*lsDHl@aM`47cN{ek*U7p^t}Gv%coCEo)Hl=j#irk6qhvzO=fUW7Vov{7Me=W@ctAw*)ykgYh%Td*9wrT(Ke~ zIa$nl$BvKP-C_Icvoh)~DmfmndcDcmm^CjiZ+z<8YxmJF@AIeCDNQpoE~)iNl(e)o zxr$kuQBP~JhPZ@8&*hV0BU%;w-p6j!d)JuS82_Wt+F)&x#ZUHs%LQt)|f z!vD;f`yDO{#g`o&9E{WoxZ)BMk7Mz;S1)*Fe2KO3q1Rwmi(SO@_4Uq-nj+V1115Ak zzvK4Zda9HCI+re8S{vpTE4;R?zd!oIg@qxMQH$*7*O(t$ye~Ia(egq^7vHPDW+*YOLPe zq0!Nb9LLJ)T<6UhEpKmHd>j~<$H&JvHB`oaWPC`hXlB~?!$VKj$0DU0Un=>CGmM(8NUE11$v+J02aj$8+EaX{H$^d@Xm`zd!232kSLQhN!-GncCmF zUb;_Dd4KMB_$P_H(YNU zn2Z-${pOAKU`wHCs@~k4yLNr5%h=4X5wJ!}OY8RG{zivNJxSMhx3=BR&)4_yQQde^ zHFV{^cXs&s@v+uG`#pOs8$TAF&dpu%;==0r6w6Qc=9_v^{J{)!1=8cUWLYnuhNdPC z#xE|5-D4)P&YX5?YU)W}z(qs@dTO}Yt2HBj@cmtVisKl=+*4J4etsc|iHQw4j`~H1 zYu~&n$KP7{jW4VFhs4Fz>>H!w2nYyp-7k&JdQj)=GAEt>#(U-X`x6|Sa*@C(v4HW9 zj5S#02H^3xE~ZgeQFSQ`m{s{M~ct)-W1(P-Z}5oyaud`2AjThzYaZi?3mlZ zgPSv&vUaH1@7pJL>eQ*Lcko`BsZS-rbY}1SoAN{B;^N%g-8o}LG+x#3-nuo>GH?Ic z@TjQxs%x7rFQvlo-&aX^I5Rz&O_|&}82JA4o9lC^9FI=Cpv!^N+bE7lhx=tm#vgeN zvN$?Ae*E&qy6pj8w(*i34@WW`e77AqkY!+O?Av{~<=y#!%f`>KPv|N4FAX_beRCKX zSg22>p{%lWT0u&K!3H*WIj zq`IDtPQi|oI@xWKcCsrot@0U$K6xynE~t6U3;g+=@5k66yRB`j-}mn$BSSyOTAvHF z$1mQjxck+@sgXCE?t65yNNu_lg&mxt9X5}Dt=gt{-qyvhBcGR)T&Zs7liwZ^AJ0v0ch05A)Km$*n1zcAU%h%&f!qAp)<&&eyVe|g zZ)W0k;K{hSWDo1ks)XT*?#u0Xsp+=G>TGOmL+!B|v=rC;`ARsa4e#Ht+q8*2CML%6 zzMGiG!Gl{GNf_vTdy{eg{Q0>3lwB{+;k~~XES4U0z&TB~$QH(j*xB0$XBSTkAl6h_ z40gRd-}5e0aEZLUy`5bU4xRnij%T&glVgL9h)d~PGX?M~CRx_24`JUv>AqC>V;cKr zu;#wIW4xTjJO%~^^09}%G{gCf-6dW7Njt?~N)zQ~awJi=vH%X=)85yzLw=WWJ-a|e4c>DOWgmXA5 z6jhZV$4E8Y-OWy2vBIiSb935Pyb=Bd3)Ur@h>D5P zxHRTIj+4-@zG^uC(3b|zXn~bJ_0=?#MV9rtw>P$IIi*UCf4TdbD!Bh;-|pREef8NA zYHEVt`|2$+uHU>#LUcQ>YPQY9gkzzQPz@5BL*;xC5%#w7v?l#nVSD76>iT;9`~%JA zz266#1*JD$2;xYo9__8qW|xqVSbtHKr>m=rEct=fqG;y%Qdia;V?ef)85!4@nVP@l zhWWbois(r0nIIav+wSdpe#7N_{A!1nyt=3sHu0@n0wJ<^X8Ms)^X=P;BRw^A*wGc( zJEoc2S6)p^OGX$uGSb7zz%CqxME&W-+2uoHWBTUi+=)wfuux>%`;7F^czJm(vA)ki zaR>;UcyoQz%|%xwoskWC>a#@zR_qBP*O8K9c%CREv?5 za+If=re3YbuX1kOxREyKc5W{B`Bg4l39?&GEe(zqUc24z`&*0h6ZdW#mmop&A!xp` z5a!|%r^17RD6tTZrhHdE>dCXy(%;_V@ZP+glf$XS6c-=Ahzdta?7VsAZg|$$jZnVAlTi=?d$B6L}uUG9IS|;A-9CDhNq>NU0hr~-!hTla0H^8 zxpC*u%lYr9?i^@qv!l`6-oq7Ru;w{iA*-!_Lwfmvu~>@q7;IdsIa=9rCl*T zI7Kg7V4{VNBKet#nb~x)$jZI(lqk-NbV7MW1xuRQ*}>YyHU}T%0qmu0x~yK?moojL zb(G6(ot4dn?7XpF$+q3EFEIn0D!H|3Cgi22riPkj#8=Jd;$pdQRQu{X;}T@p3Y;Q}P>kqcr#KmG$%Yw|8_r)*3K5 z7h9$rkOwC~Ug%B6=IdNLxY}`PfTHIXS=reN{ecs$^>1G`T^{=>-xE`d99&vkYu1R< zl92rh&yOpJEnIViBRDviz;b-Dd)JEv2|ksvB7yBaw!%0(_;^Ng8Nd`J#|pM7BwYjn z+vG}~F)t?|DJlqGPLGcjQsXU&Ja1^T%hi<~hbZ9=;797VySrqp@|FOwin|C43qO7K z>@l8%hT64jm%)Y&@?QrVA3Z!WEbidws01Yc*^7<>jB5MxWywXWmcbF_R=;BiQ8#jv zO=PZS+X^c^uc|6VF#6cj!-RKXUF0{B**-Ef6oh<%>|0&v>xEjP9nh{)`m*1t?ADMF z`bCnG$+(L-qe$DV_a8pI5#QhZhz}wCl-|=|zGB<8zN3D9Ns&L62U@+kw&D5M*kaec z<>wcw9%71!i0G+F4Kv!`*x0zI;kFo33qxD^*cJ;5J}i&EmR6}+xv>b}JsaAr&DZDQ z#2zy~v$WIqZkPA;)at{7EztnhpB^7$PN@0Wi*^|lZ7!$LztLs=(({dZ&GgZ>JGT_9Ub1A#)p}dM;W??cD;v$MEjmJ0!?+RcWuT zToFYIV?=V_GO;6I_3D?Kt6z8Hmga19k?ENuV`0~U z(Z&wdVK1LO%c#~k49G?WA^DR{9-4R0r76!ZRYzCIQ})F5O_x*rCcf780X8OMJ!QWQ z+Xy>TL5%Fe1;b;7D*IhrST8&g;o+G>nfZAuYjz|QZ&%K~WR?*^pg#_n-}kTTDv`?< ze?V0?S8&x6yu{$TI&%b4o(apg@`8IKBZtV{%61B=9-^oV>i!R_>2Kqbb1oU%vlvOS zm{k|#6gcfe8Eev}wB_d3IYSQKS8iHllfrWMX4` zg5_`P>S9PUTb#LRtoP;8^5OTpFL$25ey)44ct(%pXpP&HTA)T~3C1++Ys zJ^koKpWOP58)K?S>@?PHdHv~(ZNOs1INyn`Td2;GQCT(=`YO*`s1k%gw4>bfccks<>`Z%Sy#Ly@ zYqF_AsP$3G=o~(rHxjneRY&smL(jez=epr7_x9D3lJ(?;3%5=kxuoW`v5~!izh?2~ zYivtx3er8a^>*!&%CafEVb;~&ehj#+9Tmw0S3`c>#fx*>W?Z8d$0Q~3Gl_lm@7d+# zl#%;W6~)kMQdT4F+<)+ZXL~;K#M75A%aLi@K7C5z(spogC|b$R`l+ogC_X-3{AZM> zr)Lc+nq1dbVbrtys)u9(riS$a?N+wRbUdnxT#F;vy2r%QGAUQXb;aZc!*2*L)z>$P zEU_uDlr`S8Ngp4{te3eLm<3?5GMJ9hJd1^%fr0lyv{8brVXA`yD=X`Bi>+9EzwXmW za)TPC*y?=WHFzbTJ$_765;^0>-RV_}zdBaNkYdyw6@OjU4#7x1g$m&Q8v`bv*GfKL zrl_i_+VJ4u^@TfX7+<`6IT!o5{Kxmc9PC}cNkrMS#k6?UfS)GsjZQ^IvX+*WrLi+_ z+O%nn?B70uZo|bLy+~JqXt=!|>$3I1-R8&^B z}~Bj)e=Oz$J7vok>cPk-U{{^Q^kI_S68Ae9y9F-1rk&?xr|Q`og7D;sK;+?RC~^f+ z`th4L^LIQvv<^s<@G__#2p5Aer)zE=zhobcs)|ZUO%12us{JK6*mgh-0Emu5ZI3OB z1J-i0%&CYL3`gP{RK|xmJfc+otbE04?+obXHYHint2M0+#n7gNj7U;heuq@}uu)`4_pBpdM z9`x`i1^gg^)coew4L)4q9JHLAoDFxJSaWl8O_H@j{oDr|a_A~5Dmb*-hTAL0KVQFm z^5n@NK-dJ%lPOE--he9foB26LwpW(*gQF;L+rM@`Cr@(2B0C0XhR>(q*fBbsh@;p~ zTwu~7;J3YSK1P&VZ# z@$6h(V{zy{;%uH0eol&sc0|x%WCg&+qsNZ9j}O`3vq>Hv9)67797G7fo7qKL$g_ZR zNXV6bu~L2)Q7*RO*VZ-VxhQ#b7&tFTLb&MM=ZI4=4))?>Uti=Zm&P`1-F)FI&b%h?sLLoR8u$ z#Mi7@I503kr~{Czrq|Xlm;x&h9ue`;v3tcu>wH0A4=aEKW?NfZCD)cUIp9hYqSg1i zep__1-q6^1erw=N;hyR%!HTO^@jiU`u=Lq8LpBk0>kl}2(UoWFPHWQA(S;&}0-9u* zk9Itp@BX#J(0L+^m2YV{@+tSnThpz5iCfmuJU0M_vW^l|-q_I4zyuPd25U-t#)eZX z9M!PZiUS`kd;0oVfoMSiOP@M@n)R0Ng@qQdn46{q`*nBBMiFN$|uaRkI|NIHWo-ec<7g zj5oG^{dyggqfY=}b$omtnk&f38G~lr+|bm-DkLOizk9bd&e4`$reTN093mo@X(xR~ zznn!W&Icw(%fiCK($X><=Q9#{*9L_!!X1Km7$6k))~#E~518MwG{woa)X~+=$OYFc zCa`kvNX0j~{o5oM85uKLutRh%s(OZ8x^ziVS(y(=H$KtQ`#n}ePStB*35vXsqM{-K zqmooTZ{%u-iLD#ytKWp{G7+{9l^%%Ov#t^GUjAH9ZwCDMEd8;gV-bj!kh^#90xQG^ zjM;2$m>dH-B;2GHUNHfA4&XOFUB4VqmCr`-K$41k3JP_OA@9 z0GW57`2;GTRFKK!y;natxLZY4Ra{z{5p2-o*BUcXh;oid(zVSubzD8ZG$;WeC{0MD zm`P0D4MO_le@*Rv+0)f2RsgJcf0`BKRmtT>0#z0Ed}0t>a&Ch zx&P(M{v}p3@+wId6<_D!Rra0gbFFBOAQGyIaP#XiaYJ zPFs%7LeVCz>XOLa2bWsjre>zTu|w-I$v?1T{RI_Hko)m+Aza^wK5BpVz6Dut#NKyXv>W+;H*#N0hy^%}VNaguMDM6smuFmpf-n5Ui7GFN zrzkekw;NcF(@pI@j&y# zEw$J6p1xD zFoRsKytcMBVl{V$tr$JH#}fePSAbQiAOdU}8wDUc=-~$=-FbC%u72Kmo~{9Rjk2v% z4r1GbDjBAl$^A$#gcE;wxSvH%PL6ODAm4m%=B;vVVFN%$HAy!;JxxRkmuB1?2;;J{ zGHuz-vI(T5M^4|K_u$?cDNup)Q2W`^)6-u8O3^*A8>(DwJvIiZ z3PB_hCRIg6CH}|N~XI6!1tRK}%H(;DNABpY*`k6}w>sIjGRK^0;} zqDWWEqoC#wA7(etp4zL-rKP3jRj2_>GtAPw_A0w(@LO3m(T+7WHI;$N(vMlV4hWH^ z!1CEN&>+_9soM7qIy&d{`J^c1;`V(-M0$9t60`G6t*meYt zLDDNnLP=8v&Ovqe1OhaTrV&5nLV{jF>Dz%2g(j)@z@yWxzj0+0KH}whWqnY(R}Wvf zcu^^E+V9Z7hXp7_K*5$mU~5CtlBe4Y(fCkro#DGd8k}DGkSK8L^3aNwfQw#=S}TM@ z>r3+^i>b-MVvpmd4$#u*DNv;5jmamYrl-DbA_61S3~5|~+~1Z_Mj~)c&Oe&2Hr@D|L7U|X2lfB zk?pN##jNK3i5?#wCbi*QTgS80OmpYzg5aY{r>ThXcj>LEv$y?Qlo@#4j9o}Rj>Zb)qatxOjO3x8Hz z+J7bpWpd7eW>q42KuuC0q5S$rA)K~nJfu3{Tvx$avW7=)s}90qH6XT^(X{YdpAe)M z0TBPkf`e)HzPlBHiq>c{lp|@CivXwbN0M1Q&X8LE` zsZ(FyDUr|iXWF@e3IwVlK>}(%(PWV8ZU_AQ;gEkCl?UE}CWH*sR%O_vw^7Q(%Uf-| zad%IR_!8Tqg)}rYDqaH%qoShPIy*zb$Yg9pG$JAaUV#{taFkI`fI*Eqtcj7N#p2Em z+Do`aqVp1n>oe-==BB6@D;y1(_rN3d0uP0=GureJW04p7s38ge8k zJ49zli50*j1dJ1B03PVR+eccW98su6=b+vl27i>Y*MY;5g!>S{C|v<}@|}m*6vE!QFv`XYuTQAgvcudUku&^-mYv!_0 zvb5zqqO&BNue|#(NO5)x71U^fo2x+p&%9~VAU}YWdd5#V2 zAcX|P3!=9jXIpTTQ$o)U8Wt7Q)3b*Px_9qhl{*-{+9Ld?<+g1ZUlhRAfQ(2}&8`Az z;Nfwn3bL@yNOGJ!&6l?~`5@_^&i2FY_W&B?G*-QQsf!AQ)E6WJklGW$nv;XW#Ox#Q z9z3*(Y(nj>-Mg>9n(U!p-;y-K&mcW z?V)2d4p`zo@L@IEf@Sm+D7oXnC*WxfPxOJ62Kh7#e7{&g`StPfDL;{b2kY@qeMdip zDq*j)7f6CVAMPt5wS*xXd<4Tq>RymotZn6c+Cc^o%^U~qaZ8Kv=Ia}EC%$&pWNenE z6!h3oI?C-(eFd%^(j_6)TWHqfaYR-khb=aoE3l#jCCNNwJZxH4Kmkjz$5qy)s60hi zuH+%QpM!%qQO>{>YojQ*ao79XF5)>sp$MYoD0GQL1zQ_|>-bQeMS`ImTJeWDL~dZ& zWD<)@c}#Z#a^ZWL}w)t1X(s9{*6IScCqK21^Wii$;9^&6xExkpisb zL~(j%#HTcbnH+fRF?ST`e?pyP5aJLC8$o=pKkm43IoprJxF< zf{^~p>g)N)GZEBu$I1B5M@3Tt&>9f_sUQffDZnfgg=&Qqp?^L}B8W)IkcCK83keG= zYibgtI1nKK@1FB8;CRJVoOYpigQ^TChqkr#LP&uTfP^&E(j5;&5bB=(^Hc4oHCt#d z(`%Xgj4$F4_p$Qh3O}bnWBcgv+?9Llg5a^i(cQYI8Dw+Da&8iZe~8?Z3PQxxL!<_s zW%B5-3+kDIHEl^_+o5txOY2`uCP=QGNL4NIt& zZ1-tR^3FRXt!R&+#(MCO+*yhiUU}Z08m+7>^38OPJ6UwZ*VEpYU95*UKipFzL6s0{ z5gYv3g4yNPJ1tLxb0`hyY6qXg^D_!4Bk8D<@$n%bxSsufC*PO0fp@xPl|K*csF^q3 zJ)t=NynD$YIIH%cmMQW`GkYc-LKgCwK^}cIf<(S+>nn?Z-nzvwiT#`En86az5kiI- zS%6K5T!^!&bl`&u4!F*r9rL_o{kav)fD3M3Uiv8hKswF?mqC^Zh31)mGiO}I&6RQKf8Bc#_-Se7wdu|<1Idk{b&+C4DSsx*ufCL0AJES9z&~>=R)nRTe zCQ|{Su#k|pM}tLWcpeC}kJ1evSs?H+$N)f4JonZS5pUdsY5f#`CV*vHqVefm2h>x;^xO+(D^ZQ!3?~4~N+B-W(z?$Ju zd(|*~+g&McmQd?vQhCD;4caR@kumj4^LzA>)9u< z2LZvYci_f3O~vqJG;N?S5l0tmX=}qpk(I^2+GEkeg$p~_W6@)nFKs+ssZp7*XF_lC=B_$$^?#C-y8`CEc*p$`uTs`3NbU<`1sPvznC8BP+MgVJX{ya<&M^QDKz zhaC@nuGMY4eEKvUEQuhvh!dsZzPrS#{S86RZ>|--eL$AS-(cq>`ZeIy*aTKEc@4Jc z;$YKIP}nL^St{*%k>rqX0oD<9WCu2wQ7M?{5NzrzfICzWs?PiO@3+BNwWltflWYaB zy=_Au%K)SF03VcmNB0xnVEL{WL0MS?k1j636>EIg=W&)0jS8g{0ae>`_XHzP*a2}S zDLFD+TYuK<%P5j+DA?yza5w-V(lC6{klIOW>sH=1Yt~$S2z4;sGDjR0CJpuY@nf=C zAv30_g4Z(GxKYo*;OV}}7yljGn9Fbz&l-#i;ehu@g&*NJAzOi)MK`B$FDh2DI7q9G z|5z!>Ie~(VvQ`_iG*orjN=qv%2~vM@2nl7Ln(FL--87~O86pgRg;i>5j~vQTj_Hi3 zpY#+eSpz5K;2}R?>#N!O-rfj>D>nto7Ex>w9EY+#7A6)K7gu%m{ue}J3AS83c3=ll z-zSxjWRbnx!KA|J!T=r0VEuaY1#_YzBI3H(8XFqgfIMVfI+`V|s|!lk0II;7;M=D% zi-;NbZlKgWd#}Fgl0=mNc4;2AOep>#{vni-!9WTW01_KIWvj0_##}Ds5X5xGt5&0 zXQug=?;`lpe(&BBSm82oV6*rSq+BUjT`=CIU^}pa(cmOXUcZi&@FR;>vw0t_Oo+_r zTw&c|P)x*Ig-Sh?XhT3duoJ;t!A0GM+j)<(^JAQx1q&7oef?SrB%Ia`kju%<9U`WE zoKS8svh}>V&JM5CmQOh(Fc-Qam?mdcfJjnuXvXf%kbqnT^6SwXq^OZ-B)Ihg}5OH-pojV-G9RlQbFYF z0>_y}amBdf2p|Q16U(w}iR1}9$i6R;` zfKP_ZEo$*AGH=SGSHw>M_%KlWTZ@EHlvg790?0Ca`S5^nI{1UqllckV>=_i$ybOgf& zH#AtTE}LZ&_n_vs0m=h(2A(ZLm92q7R_=}qeEU_<+Nc{KqvkhQzzP(6nM4*vU{l6gX^ zb||`@R#nkbfRRWB4%k>j-@hmPn=$XBK)!nQY6%L+*OxW)fhyYI(}95b;-Z?xpN03L zkxMWl_8LTyQoN>6gw3s|s9)N%bDj>!Bfd>rVf&LsUa|T)pZ)PdhNv&SJ0L=$5 z*bv%xc$ClD+w7AIm;aJ~xQ!mVw1L-y?7`dhDUfw6Q|HKy7{xB}qS@`_gy>C^oSghO zz=iJtlFq`}{9PmXG;iLO%a=uXms*DdRXun}+DB4opXRZ>2xa9XAd1wNgvUb|tD1%p z{uvJo+4TgAu2{ajU80p-u7wU9BortQPhIj%p(kDDiBdav?`}uB8Xv9{8AhC^N`R5T zV2JPjwc9r!QNb9O3eBE4A@uYZaL(zlox6qD6~JkQrqM zKRj3-vv9TepQFQB0&PeTKEq%j5F>BoI|8Bq?Dl91^cJKKY1;{$8hh;9Kfg(js6HQO zA;CtDn+2P|4t6MzbblBj;D4~&WzV0p;*$+FZK{Ok5ek|5F~Z5r&%wYy=OK~Wnr=cv zXnmr*fD9Cql$?Wvrw?jn%K;WKayq}ClBb~$TIN*M2B4pK*O+|GCv-n^8+Hs!bL%57 z*R$f}2W$6ZMJ4oO%)KwdR{{{&)7y(4k(0k-&ib8?SXCf-IT;%ouCuh{$Cj%)KU-z& zT-0|r$hNbiBV-OMZ-<}|d7t9n7r&+FIO6X36mOt4BO2e)1iKIF$ZXqx1Wc8>{|PW{ z%ps*h;ULKD`w*HF>b01l-q~ZN!L2^C%J={!00Zw#-{tR$Iq_Cv5zN@4*UI<55ONZ|+B|K71rPjOMG{ps2HBq1stGxZenVJ1o z_da)bGXma$C`ymFeRzl&h+4Mu{}vk=1pjo}xl?rY>TOOqkK#Y7pFZR`WY`E-`%>GY zC`i8@O?*N^bCD~d=48Z!k^$t|e&d_R*E9JVuD0vCUSE!drj)2TLXJ|JejdHsizFr8 ztjRKV4ep0C#FPh6g`h_7@Bh>#J-Y1lmPp7=ldr7k~tS(c{XeYu>Xq6$*}_D zz(5)nrfp><2yB_qyO{fga-QesTFECB6*m^1Ieq#V@s#6D5{_#b+L)4((wDLDgE7KO zqJpBM=f1kEkzISBWk;#Er%l@R>x^sH7N%SzrFD(3Q6BwmNO*|3L~^+@I~~9_EZoI-vp-vU(7N_4-R&V zLskbGr-Hyq5|aszpQL*gS(QOaXmvQ0?QM^bnSEuXP->BOl(x^Gx80(ze)5DK#DdA< z<;#|7foZ?`6^=SbtRaNtu#Akw&?M(6D=U-TLE$%;Au^2gzGX$s*Fs$dgiNcUp%L?y zJe&Roq-3ltTrd+=kPLxW?{W>gLJs?f3qol{2vigg#*_aCkTiR6HUT7w>FXm6R=_wA zO>Y10YjwI=;tmH>A9$EJ;#WgRL<jgfp^ZvRPOeY4+o|dbFyi>YqrpG5Tp}ePAmD&vi+t4$N(@+^ zV|3ttJ6M8A^fqB(T_?C;L8tEl1w1t8GyOdM$K)?HlfnN$od3a|xY9m=^akC?m?-WyD8b^SNq|q2!uPlEk$$#p@1>XT1o(Q zI(6qI&3Y(&zy_VYv($@?Hg*=msLGL{E7r}kceF44JJ%3WXj(S5j*@4uIfQ3klAZuw zjGDpBrvGM;z|5uJboce0Oqr8xLdct57Ht>Ml91gYjnHr2{D#Xl@G-(ii&RaegPBld z?Y!V{b(7xdscDXMPkKdhajlClXRC9oHmz#1=>+sZk^mUv2*0o>0+HS#M5x6@5_nu3 zpVFo#HCu1gBSazplXoP8P#wS|@X{hHHT4*3k-xq}bW`{oz9zUfKjgvY{sX#vxj%pl zCU$>9yTYu)K#^lYl1+~PBS^Cb;?OH}I-)nn z&{zneE^$=|FAaHic1^yUW{5K#@q>2vQum_Ske^}aC&2l0Q*Blv139Zew;V?g-J2TY zoo8%MqQil-4d7Td+}n3%gz0|$TB%<&_a%t3;60+@0p=uLN(MImAfR0FrAy~w=jvmn zg7{)1NSYM)PG3tC6&b0G2Ft7Uyu7?Be&dR0)?jdTbw%S3&!d+(Emz?30tiO;#o`Yi zKHNrM)&DETXH?npOHPJ9f*`|a2p1LI!gs^tR2Q84Gj;Y-jgDG9B{({RVfs`#(zi| zMoqE0H8ieZ_`oTZ!e08V-*@!O0#er^n52K;|GEkVJ?YB)7^zT1Is#^WfJNi9l-swi z4WNNURaE#<<(I-)Hbl&I5NJUIF;EcPNtKK)i_=|FMTQ<8%b=n||Fzq@_s7OzzMcPD zfb*w!19}1@q!&C1y9(NO#TKsh8OdyVjV_0GnO1S<&aogzLbD;L2L{@cy^usx9m-Fj z%@CJmWK2c*Pg3tMPAE~2ll4?wTrdJ$LNty^k9O)PJ$j4pq}vq!*f(}QDd`GaR^N;i8|TR>W^d1Xko-92tY6Pms8>Qvq zX!EEYAm^o4>bX`mDz!>bU9fRP`%R*2H3!U_L;g0jidlol1Hnj$0r-+H(>5s>5+{X` z(WyKD!t{F@WXE}8AMgtclZN0Gt}P3(zX+Sm!NVij4>C>*%}^(r5tWys56(2(X6=j1 z8X}hiek?{s&Hz*d(obAT$uunha@zj=`zZ>J?uh}GSH2f5u8V>ys=Zk0jV5l+nlQb7 z<~Gv#4G|tuLmZS4s4GBb!w8LOd3f7U>yY+C0ANyYIokrygOU3PBRd7+s08q9-d658%Um<9>sNR^;0!t?h)vqUlpfxHO;d=46c zQ7_4Tc7+uI;7=D12Zqk{+KU;lR49`0n-H~3etT{Su5+S!7g&9tv4Ln9bs>-u;+JvW zLQQ}LiZV`I5)APNeLJWvY0!29Xa06o8(>t3D#WdVGmZxcQTP8r>i5BbQWEM0SKrH~ z8;7r-nEr8Z7DfJq0wVnVg$q-!`lv*PEII>opZoJfISb{p=w!+kmVlpbFP{178a%l* z%Q}KEX)jmH%r2o$zeJG8@)(LMK^Xt4Y5K%;WqseQL{u|*?0*42|Ij~rnXK#3LVf+I zCnSk06$T_b$dF5W`9JO@YovQ}lVNv)c5}y4fa+&;xUmx1SE~5SfVG;#%1p4L^{57VA zj{OD0P(fY+dM0f+q>-KISm;~i(7FP0?oWDtME_-QFcz%ON1RT=QDD7DKN|5uqvHRI zCgDdzC4QfdB2DsyQA3XhU~4Jp?B7`g8oI!O1%}_I;btJNALmwJqd!n|72|E30~OC+ zZq#UGN&q(LCtMYYgAX^i=&z(7CK8|tWoWjS*1*v4F_046Fi+Hi2wL2^$col7&4}mo zJ|r*%!KZI$<-fuZ)^)R9KK}p4(fxM-`+k7-6(Su-V{+!mu~b&JaGF7XX6J80Dv<)e zzyfj4_DPC7g1`@b$_}xvb|YF++|YuE>!%=ABFvUxB{3pmqJH$^rAskb9MVDx z87Jt>89q!An2W6Jd2y+;%=<110HL&|#x!dH%h6H=S)vWoA;9B)s!rAhA|pmTv(1~+ zRN=?p)08*cm0yr|5gGI&Zd)5chRC(>B()*s$m9;<(c9s&e1h!c4m;zTfGK{M^B3nF zl&B&M)oRu41xJ)XNkzgq9kAj|;OYM}lwP3&K^WRCu|^`gT06BOW!1pyMXOKQ-UEFA zKNB+|AKGx9 zCjutC`Wq!JWbjgS69MWeRCs}Xs{llEpxVPVeZQoGeO8UWb!!RAcmt<@s`vStvhLtl zpgHt>cyM>NwkW&{4_cJKN)||I>yTd44otH{`3Yb7M28t^;xbLRN3*qq!` z20a*>^oZC3a3g4rBV~hmy2;1PMp85(mXkIShy#hM0yp7WHU8BJR2^|W$<{|s&fLTh8Y1I zu1u4ynScXMJFw1Ve#RRt?QH}YQT5Ez|C*%MrX~4~s2@vrw zfH{~z4>#mG&sU%PvK%RbtQLkY1V4DN8X5uV8-;K(%)o&crp|m-6#mRoRL--}Cfx*s z5ojowqRrCQpxX}HV$LigBHi~z1)cU^dl9f4P*5hR`>WzGz)~2AFhWd8$YFFt4x)>Y zjd4KC)c#FITX>I3$vWT3q#IWyk{y&bBo^y)_60ouHLt+cV6z);@4?(WqJePo#KIHS zSC^6MF(2(AfG_}M?L({Zowz8x9-y~u{_R}S_YeZL#Ztx-hoN>~Sp zCJtx;jZ+H*H6-6_X3}fd3L#_5SY$1XijF1?6zC31P({K8U=2=A7Q+!$#2y7#d&VG& z1h+vs#1^ni+@$jgR4IWz{0kOje91;DUR;wZ3OOjOWatXvcQs}v*w?IGi`+3cqnU&d zxZ4Q8MU_ot1t>vtfWH{1j*bo@LI5L?7W$Nwlxmy=Y(P7lc1f>Kvlm|Q{vAzSZIwm? z>;a-7L0ZY08#lhP$`|6}V?axojl2b5?$YAn7sQ0DI^4lsTpZ- zoBka{sDdMUseq7ZS1|A0Ze?Zs`cPC%%quT_kCwiIS$~*`clcP?_y`x?Xo^`N*V9K| zzkMqMlO2ita-%RCf;o%`M(5!kXq+TmuB@}jAzBqIEyBo0xTP>Y;;8unxJ!}!7fDIo zsEdt=Kvak}?|%E}=r^10x84cG#TsO~Nu z$L@e0CV@^X;tU1rqy&Wbo2`H<20>CJj9uh-WMrgyS8wkLTnF87|0;YELD2U|YMf<_ zu&4g~u4E84Mg_w5FiZ#;UvT7z=njYVU-%0_n+psgvChMW(S}Yu#1x2Qj^!tK{(!#o zbbFv0!b3zfn)}}0q^J@cBlNj@gldvKa#D%r1EO-#)K_C7juRh76{sLGqX3<^R#_%^ z=I|Bg5c4L{9tPROuM@?EHmasSU@wtk5rZ}3Yk~&mHyNdw{npVM6=>>Vbp#=T{~`K= zu91?B1EAhWX-Ksp_F(Pk3D(A|al{y! zKMQV5U0&l&OZfB>KF9*yzC<@8aqs;XPOvW1iVx^?>o4_hgbRMmvuMNEddo8S4=IRC zC^}Vf|Fzj64q?v5D~n5zD3H1S`VRIDX{LmxvLpdOl^NZIV{g#c5RH2JU;CmEEWpO_&hw$4_tznK|9$~ZV}cw#qJsF*7p&{(G{IM%9Uw;Mwh2SP@C|ze zIuI+Wz@O%$gN^r$|GoH?vu78O2D!gJ>2F=uU#tgY9?%{Qg*e85hku~`k5=DIe+nJ5 zu0<0JXwQK^a5d=}M!`a6OQ8ql^uIfX2=yc#2To5&qZt)qQMktRI2}d-l^`SiITz@d z0tbVyFk(i|;skGB-!e1{A#Rr;YX7DCCnY8_?AzDp*9onsfc`h7u8GXE{<6Bz=g*(b z-h*LKI@qMt4H19ZZYVf@?AC;zc1s;XzNrh0s3h_v7<*ON?y@o+P8M zAmdAjic)y0V;Ix}8>aaw#5`X@4x*CN!px+Nm)1mJ;8Ee5)dUy#GjVHG?7j;-f<3Ie ze}XKU8AF1DA6w;Tn8*yFR7eFJrfj076BKNP$!mw6Ls5Vu#4nE=OMlv-u@VnM>1c3^yK}oHUT60)^45b>sezDeic($bbI#f2}c+0 zjeD{GvA2l0^g;8L0aqAJQzd0(beR1?dIm0A5~2cz>ZLRw;6&hyJ6J>}t}OtSxg;kb zB{dbZbLL9_XG{E?XaC(cHU)bChO=GBE&%$N)%>HpmI2J-X`I22%k>kkB>V&s;-Jl9 zYfe<5ZIF|PXN$v680RPo{V%h9Kv(_AOa*w_-(S9Xk@QYvY+Of7k{A`_!Y;fYp#_GQPj zU91pL;0u-ghu#C%&MiNK^7j%MF-yR0E=DS%xjjYywT- z5hy$V#OZ#ilklm3f{T>kU84u5PtOO0`d1?2EWue#h$~)$RtAxgXi;)u5dj2ikkt05 zJ8So$&kQhX$@W|WCqq6G+RYTOffcsrlJb)f{ZPHh5C-%%ph3Q<j zGa~pI^aV%2n_`EumP`;M%~VtgGzSd5vHI&$NZOp>XeX-BuqpoAv#mcna`t~!oPLj@ zm1x>gf||kdwjB-lwWGh~<6^YPTmO=eSKz&2P66S}3HJ;h5hIj3dS`u2HaNcbq5DjI zueSw-9x+S3ahlS=lY%LM+u$vG{;zCDytBlQ2m8RT$7jrb7JnvJvTSL``)@#Kwl98UHU|-TypWOi(8OAWPV$Ao)CQZWiKGu+f1w z2B%}tufnJup4Pt(4o2)Sz(YsOmt*T96`Ufdm}Do>9VYtg6$lu^ID-~5;w{1muT!p*RDKbe^Kg0W~;} z+cD)8Rgv5f1cd~nCFI?Cz`RBj6@t*QsA7Z|mdotGOkWq`yv6L%7*aKn@vz_mF#XF| zQkd+G)!@x4o;U{Srq|c9^CqR=Cl~h?YVjcT=)rB!2f^=QBNy zCokNxY7~oSDtaAZvuVfRw}RI$VjogU69(ZMo@B2APa zWjX}oKx9^cSYchW3(aOps}Jrf@=#M#yZ6LEC;QCIMFSlXLC@mXm!ZU_nPqg@v}^)4 zB}N2y^rmAbMLTXC2)9-p?6@!O_$u5xjP3lNJr5jM3Lch>vrG7v9J>iw75#J=-a?0= zo{5K5bgnY^UEViOjGmOsfG`-;oCbddNYH1v2PyPw(96rezb0Ii2qqL7JLmlQsb<2n zhNSbfu{D?xH}=5+^{8j4!Jrs~s{qh{9pmctZ0A(Jp3%=+zhwFH`Pl27zHw7Du*WYY z4Sf(p44kk{R6II}wq&HVjJ!8E@&x;kB@qy%EWC7=;qFQA3>mG00)}us$Srt<*}2+8 zs5TQs-stY=u)_=%;+8`3k}xQB5=Q5BStg6f@j@GgBBBm<7qOc)Al3QaGoM>1g5eVw zoHydPnF{Y|e>}F@nJe_Oys{4KJP9o@TtSIfzJ%|4gS`C8Hn`@j6H(n(4 zDZ}gu8@bil0of5Jv@!{KEttFCGQy76=axJ}`!=iuijeiG;$<`v_7CbadlcS6_{8F_BQ!D6%HkoaNA_)h`p<; z0>%bqneE2Clc6*~ZcIR#N~o=Y7Z>+~`g%By8NTH(FYCbd7m5$nVj@F@NS9ZJ4JKHk zXyUvFDIwiox69-F)~p? z)*@>O#@XlsDwZQ}vaZdCUyg+!evFAs$lP{U5Aov!ylctJi6Tr>}en2ZXixpcY8WI~nS9Yx_N; z#_p~zlBTYhX{=hc6{Gs}Q0Ar>bnA<#Z-ZcXp3%UZAJb*x?o8n3z<*q_GLM6%w{c z=fY#x;-9090qFlRy}N5eLiu!IK>-HRMFF|ypC}a7olwIRGB!ltx@=op*>4zP2^}9r z(^z9W;Wm%~=}=UZL8l>#3wGwsy7y?qfW`SZv2|j`FZniBE(d-`Se}I8Ibs>Em32do z)h5ln=(x^b>++*IU9AjlMWv`g(X5eI9c{oQhAGWrWaJ0gB!~r}X!05Fz8q-YOL36d zM@HLFFO|dhVbe6=)#@)zd~Jo7$c#Z~r3-2}KF#P$!UV85-l$VhU*a1YF?NvCK@nX2 z32b|I=6TF%1q&&MV1!+Qh@bLQA_%e*7lIOP5GD=@=+C4-`TqHh{vJ%Ifc*T~>*c$5 zTVa*SsBUNRiRFQ+4~C!=Sp%t>uVM6NYG`&l#x_MS6#1?CoV}c3V@733Ha%Qp*t2g zu-C77T)@*-$+RGFknt!)nuA>tjN5TS>novi=V)4*QxQKFo%D5pW)(mSwT#&Z$YqZK z;y7WSLGK56Myb}Bs&8bL7=qWa*;(ZQ+wj+#Ya0XsqCSGcp5Z5Omy)+eDHt>C7AyvM7p#HLWB}RA-IPBamH`%z!DO78^9HE#Q)XY*~K(@hGDoJGFXXV0~C-fWXv={jY?vs z3mUX3Mle%48j40mQ38Z0I?+-@i_8Fmu4{*O({*785si>wR1ATLio~BFsm6%EV<Dru0I_^#CK{}Ao6VVF8H+!c@KxjR)|RzM z@!=xYpIL>LeY}PHAfZPZyS%K`grIsLqxSJ?+qdk=-%Lie%|dZqOI1ANo{gO;NN?P{tQ?Jrf zSGI;5eu^`1wR_dNCH_O5V-Ed5(yz%vdEhni8}tk`l?Z>8Zh6|-Y&80=d0Rw(%ilQd0Q9!5B5cYJUvqc5WSG;w5$ z{(-$+sSxWlpV*E-M$l+DLTi#DElvxM1mvNwLOj~N9?y5s1Aa%yenoIYTONKPK18g9 zB22=?OKz#i%NxjWMY@k|#uh;juf1|bJH`ZfMJfQPLDid|Y?>ROQeEczZcvj%dJT^y zfdYby{#YSnEcTW%{w4>7VB-<)7<3Fe@1yKbtK7;(=T>-EJM7P_?GT z%_C!E`VGw4zU(>5J_5#@RI49qB#AM?0R@eMeC9nq;+4)>Ivuq!47W-a?s1wYVkjg7 zN=AKj@)nv78ST-ml<& z^M~eiSM@J1?p}Y+#Aq5RRR(*nZ)XwRO7w$TG?OA&hg~9$w`dGQr*^QIa$Z)HgyaV= zUw@xo0qmefg(J)H%CCf8@(HqffqoC+Cpf^HrRmYS9~SPsA4($wCZP{s2s~VPb&oA{ zSMW9JN_2kN@3vz&!+NTJ zw&45sv_CgFC%wDkXo$t^J1yV{;_?2C@K37#o7P&-#@3ROjxOJ)NLsF3y65AU;qFB diff --git a/docs/source/_static/images/tpch_queries_compare.png b/docs/source/_static/images/tpch_queries_compare.png deleted file mode 100644 index 969043f93a42b784cd01f3fad3de1053232a4967..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32843 zcmeFacT|<=_=R{?*dEe)Kp0(zhbFQ^7$sIYsI+t@U z3kwUY)WKgBSXgG&v9Qcg`(ZZz<#aqBAO0g|yHCwl(ZbNyUeDTqMMlro($vD%)ackU zI|FMQBMb8_0%8K2H!M48Yintb+^)K|L%IHGsQzwQ=i71{O+0A+%CG!k;_#e#=89Y>tAoNuecn}>7zs{Gre7=3x!96dztPd`KxJhrNX-@Z3o$Q_C8QGaj6`y{|>SdO5n16mc^Z&j1|CP(~amuVNMSjtheM?2n z6YSSr@Op9VZB?v(lxJJ^`^3ATUil}tG#*Laup;k?~*7o+k+1c3vPQ7B<8K*zFJL?}iw)OPKqx2hl zdwcP1)kNcR_Md-#nCbMnW{_LlF0!br?cKWvB|*{=&SS$hX{Su0v*&x=x_vwJ#>xZh z`S^Y{H#eU>#4J~(8h>5A|=Xn$jUu3J?8JY6-QfO`tiu}bCBUEE5 zDxx%%BUCm@mF%1ve~iDFw7tG(+?bWg<2r8T;ON--?vHaC8K+|nivyw(5)#hbY0Bo_ zaas;fp*&nUV(@dLtF+f4@AW};JvBxj-~ADeZzSd26|)Jp>#YkL9qQ`pu8vn9?kNrN z^x3Ex{BZ9@T8M$RV%6FtvqZg|*}~>^RW%mH0TRY^?U;%(|su` zD>I?Su9)oTckhk{1qPlRXf2|f&YBq3oR}Qz!|q77cSv4^XVsK@?PlPHox8?j+5()8 z40Tl|8}aLAW$<<28CzR_xc`ua<-x}f7TfXQruv^RkgeL$kaj8p55hN0+w0U%=eM8g ztn#-g3%#-3t~>P3mJ^*g4YzDJ9&N~UN~R@^)=savI*_~6qTO3nXOqL~WD&h*Gp})r zs|XvHy@@ZDjKH4CKNj1be`BRa)|u4a^ghYV!4m1aLVA0Wa>tr)$|?mP<`CPqEp-1i zPSKuZ=P|pDIvHwHV~tZ=k3IiED$c!N{p*=M@ zF;s8g7=|t66&*F(wL3PCot-@pw;5sEQ6f?-=ForK`QsmRtS$4$e+uy2(f|7)pAAON zb*VNdXkn#&1&Wm4Wu59$>8Xm{+m}6+wPMAJ$R6czCHclI=b?$uja~0A@@91mZQ8V{ zB6a*@4x5P0rQd&l^FHSy4qrG|<&dts-daQj9Ia%Diuy4d&LadimnTK8wsDm8^;{oe?p{<6l1A6}1Sk>uZQo z)p6K&Z~G~QJP&SR(<-5f!BW}L(XJR_o7ThEcqF!K>d&4%yFuv9WQ;+f&r3wL;vEB5 z6N}2rl_y4r6m4x2Ja~5sNF6$KD9)wIO2X7P`ocg!K!8Tv@k@zj1-I6v*>-q6dh}@X zg&wbF901x!XV0EZjx26$Y;-KjevzBo>A7<{CTAFPbzQvE=>b*f-L3YQd^GE!*-3V)>Q`=jgURYVFipLhN z5Old{jyhJZ^ug|PNjuJP{F;dLIA56&!}O5Ztp z3eiIKzNE8hjrq`BUI|B~tgI}rKty}>WcSp!U&L2u=wUD0>&SAUe{GX*1uPm**qPRrw$#HnrA?mC&lu#{YtHobLXQk5#~_qSQ?vB>!3uBEMAp6O)o=aT6-R1z%XcYMiy zpC+UOSzN~Dt5=mieR!da_2-s6qlPoDguPjgyWJ2o)#_)V^l;BC+S-~!JF&Y*zchIc z<*)Q?zpz;6O_EvdR;v%ca&NQT#jg_iq{HdsyWK-bHs8;lCx`ezZ@FsCnh2xP;O$l) z&d;8=d)92pu@7Z=dSh4SN`<)X3@+18% zr|>+S2Z}^18s7kPTud^pew4@KbQou_{O#idbU1}gDh`i*?h`c|^PO^;lD#jfjXbV| zY~&5-@Z!qi2+d@3ja2I(?1B?EJ#0J2AFPu0If*>CUOg_bz*jIPC1sn$mv`sVtXnQ4 zey(C;V=D@f7)x^5vRphSH6|v;@U?%~+3)9cB8sW_3TX35eQt2-+nAS)T^qbJ2H2{=*W?ZS9i9Qr&#ItG-TGMNyt=2YbRPXO0sfn{uLXz z5(pvDynZ{*y~Y^gRkJuDaM6W7|EyQn9dj6=(w$GgF2yPr+2SfLP#-C1e@{&!t5oh{ z-7`S=dOHwF-T9iZs>#aIeGQqcQtCWE|NL_@SBQGNVeXrUd!sy)O{|Zq3L#*MY4JQJXcp&{PfCAo8DeOZi~ICm<=GX^u%lbSe}HU->34 zFHbGrP$oF`u85gq>E~yzQCg`bYqir7ic;Rcf1i8dr^Rs@y`x%4jJKlNs^d?1E`ND? zv?Ii<=D^h@dIvYVe0e{|L=#uLd|hc>!|5j09DIF2;e3ip2W&XGnu%yt5@fBO-+p@W(#cJUUU8U2DK+oYm%@`d%cDxT(`)}%VV9#`m#lVMfACLy3|p? zlt&!PL1g`cZ&-ZE0J^~55#H>~c5pKn7uQt^HwkRq+sm#WH?2)7s<9g#G|H*1t-XHy z`Sa&hS2r2Hx_vlt_N-YP0G>&buX{~gfPefv99C9l+lg+Et{iFn6z$3#b03@RxA#p= z`h$IqjfUOW2=pd3E|cR9d%ZqoPfZAFCL(GzFHX9J|D?!Z^>1Luj!#UKN2+h(7PJ2K z<{H_g=pQn(UgSra-@3yc9UZNK$H?m0z0Ices>}GN1GFK%kmpyO>ZwtloSdYLwfgYw z2l@F2U%Yss9@9~m8ZTmAce*xF*M6km0LkOHaduYnz>wctovFT}QJtNx6J1>$9e2WX z;|+_NyShvyaC0}pim^{0p6aUTNbk#1R8qRNUNt(Ba8CCsK`DTbjF!&ft~1Wg%IS7y zGPi|x1D6EldT>YFcNvwEl^9a;scad{?C(i#Jmlrh#%o?W`Rpgbg9^c!4d>3CTgby> zUYcavp&;?`>G$hZB5xx&^LD+BGmzHKbX49s`QtW~QG`?3m^w}0$9kqa%-zHQPD9STe?2wSqK2aQic<0^R)bv<8-L56= zHGoqA5*_ljBd}3(M@JYCROsP5o9_gC#`!yR_;4AbmSs+xX-z^H5Vc|HFfP0k7&3Ws z_V2bm6XiggT9y9wl{=DkoO%)uyGm*|hIgn?xIBM;CgN;Pu*iV}2dI8oDq?!wwx_GJ zyF2pQQqfMv@fKuZal2Ec0Gi^Rh{V9FEtFNU$#8-C^{yjnU78_UDVBev&B`s($~vQd z>Cz<;v#I&vT-@9isjH+^fa-&g^_4^Ac9$gX5&@h=f=Q_0_~n;Jb9m3ZalCf_{(bE1 zn_X?=f}*0yhsUl+D;zp>d1yR)$|*jpvRIcbL{KN=c39&@UYF3H77MJEl$>-I&GHUh zEpxZ3ZtKvE13dDm$=v60Jh~@t-+J8GD=>R2=9&W6l%?B#n=8aADd1UFyW3T)k~5?(rU9CT$Gx_=A zca`<+Lt=K_RjkQ}(%x3CN{<`6+g2g*x$b7y&XD=s+xa@ChM!HvbTzI7Y@uwYbNc

KkR<8>*B=&Nip8hJ>mggUXO-* zYHJhhyQ}2A1`$6`aHPzcIkV1EKs{Cdw9`t_n$P0QGHbsFmqU&TPVNS&^~&M5ur`kq zO)8CtI?FXO9TNd(skm|;|CA>dUdcln01M5^yN7tZwyG$D#0|rtYlPVatic zFQ{+Km-tVO8RqW2IA8hVeD1*(UtMD)L*SjV=a&`?p_&TCE+0|Z!ou=2o54af$Kny4 z{P6tJC}6w=N6N*E7uz~~xOH@NL?Rrp-4-rg8i8Y=b}q7u-#x}}%Zb-!QQ=cD8i~e6 z0I7uHe;iFVt2LAm)Ygpc24F{gdeJ&Nn5sx*D;Z5hk*)T zAAba)kY z5F~FLCr2$z>N@vc1|SNv?`v=z>q(9>TeoiAp@Rp7VvRL*&wRQ7$e3u?qv6-+c?g!!Lia^p-U-Lo{^LGRQ*GK33J^0MRFBWC6@> zp0`)pXG5&fRCjzaUxyFB#Q_hlb-;oWxKmO8LeM{30av9IzuY>;pXKDVw6p|?Hl8UR zU0$B8-Po{ROsqPMwfqz|oD$C5K=Z^8EG*ZxBZ!Rnz9G7NXBjx*>V9K?6FJM;&(7Wst$z?#rA;XmUv)ENZ!p?-5lAIL6Z zuUMA|?Mk=rQ$ff>V+{@61dOa`8>uW?!E!-RybX4P@~ zz>6rOOy!fpbtnO#3p)*)uHxehMWs}e<(w`#`9=R!Z(X&a;H00!rx%NdhAMT5bL_-# zrfM8N-htaI#YNi=w#zfh@0}9~ukVS6;G!c;tK$$xZ=({Cz_BVcJmrjHX_bM+())-z zT~5u7s7sb6MAU4k=6EGE66c;{5cjvYI;Vhv`^nq|P*1a7B-m@PoU&W9Iy zN@B7n!?D6Ob&q!yOq%qzwgzKGI}i<2k(iY;PFsO$YVsLtX`C-8C`i1gtmH@_T{KmL zU;r*(yM}d$C<~QenK?B%MscM4(Y~Kf;bh|9Z{kZOQtjcFFQ+AjtMnaIc#$bKav(XU zg#G^e4^4)y2G}Q?_&A6&ixXC{wvPK$p=rSZ4onn~koClg6D#@no-f&a{CBG_AKz>0 zH2zs5p=1-j60QyQVRUL5!D%ML2bV8?lxuLs8Mzj zq~#}UG7hAa3YU(Kj@%ThrmG-KA0NE2V*k;j4>matp7fJMC5EiBA4$>ti`HTREusCM zJZ4IOWR`YzBE%MMHLZGc8iC^~4hL1AIE4EVWf&FO>({UMr&_mM1&zK0v`HLC1TH)m z+=*-9$WU+nSa6>&f{n(QGgg&R120~@`0?paANHfhzm`2UDPDRS|DXN|%;Hi>NlEvJ zhzNOvfR&p!mpD!WK<9$;F}Srsz2xKI;8ujgYp4)%ewaIVv*YJpvz}x|p@*v2eM6V$ z<;$D-IK=I`^Wz#3bFN;${*N3yl^t+6fq*b8(`I8gH} zEG!=5n2U}#%tG{*U6dpANk9wxdhN$~IdTZwwjgpC{E1)MWMzRjKh=eh1X^t_q$u_@ zX6sTk!#~F--ea}4z-m#fU|+C6ed9`v!1FzH#!O~@8wgNo7mebK zzJ|SS-3mmtWui0l_Jyb1_Sd><^FdEAFe$$6W_D3;yU5z#>Zwo#zL`6JzK@@uA8Oc8 z(T3A|-SyPQmbgEx)NzslH%Z?CM;ZnwSRHH3czYI>C;Q&bPxnK)L97&xh%P|Ho934{ zg$)YWGXWSXBo%`XzeUX*0UF2%K%xKRM>%XJ1qFp}H8JL!jPqXk)MxPAjXS}jQOCu_ zg?OLDD3i))Ya5%=kozv{bA%wNJy@)BdiToxSFiJ}TD`gel@bzVB?0=VHt^hD$;rvT zELc!YWhuVi3G6>|KI0ir?~`w;t+L4hjjWO`o?+^)maq6)2LOe9xx; zfx&lHUD&13fA<+oPS-1FbWTm{u$Hz^h2L=Y>1JZ9?TCfH=&dNb`8@(;d zh3kL?SFR|663>+W&ism}rzaSb)J^OA7t$NQ^jf8nV)<~?sgpjN&y6^om{d5B3XY z*jLqG5zGpef4rZ_YKQ=~-Bp`p{I|)22J(SY$+r#yVJllc@)WXGS8!8WN=p8V7pr7V z7(4xe5LHIPrh@X1&Sw^WXAS;I#>*De5_EyfJqFh;xVp7juKW9Mm@}s~8O!<@HtID*{A#2cVFU-XfOC zD+`%kiQqdeP_wYBc;E661j`t*-L_F7Ltlqe>$nfn^ zJCgE1n$X>VB0Kfvqu#z>e{F~0rjc;+?Zy~h04r?AL1>o^aTfJyr@$}Cdh@Fuf$Uof zUIkl+|Hbp?mRsbX9%rC>-1AfJjmwwsrs{jOY(ON4A2}rKw7#sdt=P_|x~tyS6d#K{ z2@=w{H24|`FVG60qQ5eX^wn{c!khb~r>C!13MF39&(9odu0nzZu4y3`*O}5Da|qO{ z5R*}gd~|QSvXDO*lzF&LrX#840OnN1lBAaG--km*dSeH&K*4L@1Eizf-D>m*P>2?g zZW>ZrG;9TiU5+f(2bd-lH}SimGyukiQ2irP2p48yKXuk5Mz~B2sS@7zlo-A}pKIGu z30=hZqWe2_p$b$Ec9aUAc+DkqZ+p4nRY0dMoWD2tRTe4t3W43r9v@)CCClMyzN+9l z60jo-ca>Nsw9m<0Lrhgm(bLoszp#{U$`&`@CTWp^@}5m`buWDdCFb%@}(37 zNT9wAyu7@ioev6TkH-!7*84G-^FiD^sX&fR*F=?KbR-zTdI0oHI2AQ?^N91Dn~t6L z*`R)G%YF@wyI{{AK(y!r{7MuP)z6*B%DMGmHd&1W%Y^s%Fu_*#fGz~D63#wO^O(*^A=-%u5T|o1~H$?psdMJ6N2d=W|g3U zZgrhJB1Ol_Gqx%FB=OTknk{ z)?~$U{MxiL>B4V-awMwl+rQr!S`(E~$0zvy z#fy2YF)`ZdNw~Hll+`bVu9`}OAh_~Po17rLt1YtOc<%^xxT>zAcCScVAH%wZOdS!sx{1Z;@y zz6;S9(#r8td(dm7rE`j!M?!Nk1}{muo^KthWe6x_3hAtji2z11A-cW z8A59f>zFeLrOP-86Qe^dAihzccDz`utMqvPHPW1=NCq)5IFpf<9F&}K%dDeF2hhk|TE)2ADo-Y(8KxN0iBNw?R#p}D$LQz27++nN>fkV6p zpd!LOh^BIIArN;8YSw@}pTowZkcM3wj$4&hT zoG%}y5M;eokR{o0Q(XE(cFT*>eaeQ#?HmH0fmL1P4$Fm zsCxg0#yB{yAj!%hw#)D?oh7T%*Y@o8f`>(_!O1h?tS zewLle?CX^bOwIU*{}ChKR3nlM8IHP|5N$$j+Fn=W72Ops1*bWX8U3t1Ti4`)(6lq~ zDR&GsSpbHL$1nsK*QX^SB${*%Y|zcpf}p)WY4k3~>%81tiWp?b!h#KcxFXnzqC{P1 z3w$t8tmRNf^?&;G(smL~Bzg;Ezvd4g9sy{x^6XH9WkTJ<31TE=0mMSQR2~K`TWI06 zcX;GZZmUjb+&l`I?{Xd61_oBOej%-zfBpJHUhIn(hp6g0E5j6|YHAv3WUx?Br!FBj zt_x|3C=OtQM4V;>i$gfc3O82l8-$eJFqnocj)*FQD2XGb6Fe|~{(SHeH;K|hy-sxz zh=J|)z5W1_wxw1m&9QTj7$IF_>sFEt0es8qLKC<2Sq$p|E>Ui63h*AnEOa=K89oTQ zK`=pefi0_$6wDm-A}Ss+pR^n#KfF)zu+6xA`6%-F0{}5VCc^1t(Fru+??vfPPBcx` zdJ+_%zbKP!4I2zQTscf(UulSJalaTCAaSY$p}d9Rzy#sgw_g)&cmtK>mXD8JX;<#G zr6D*#9k9`*z8^XE;#;J<#N6Gw&unYlA)P$*%8H%-Xzo+Mde3eX*cU8ETbqIM>K(QZbBlZj1`HCfhX|{*`Dko7R(fAgPWX z;O_SoOMLbxL`cRruzm8l5aF#hpP%TvM5w%9axDlHg1Y%L#B0vDd#p0R@R62lj zCsB!sW>RUv36o*NnE@G!(=L2LFrkm}8q;sFo2KlBtig|?={B>}4z`;Sv45d#TW&RyIv;ZyMBFslR)4CBJ!!3dNEko|T z+a=M!3t&O_4d0ySv<`Lav$+npo&7FSXCaQtBaLH{_))k5DOz;Jh?v+8VNdO z_80K0_nr&zri~AK$tV~=C9ry#!!;)1>hZz9{PK&rW6_5fSA<~##FfZGgS$uT6B6LK zzL1lO9$sFHQoD&y*{a2DRDS*ogjAXpB6~l{C>`b)P|qi#T1GGOAe6mD?GX^g31`|` zsk>WFcoYp01(#%AZ`@M}V1~-!FvvobD>r&k=KA%@vsHaRZ5yA9@@WNRUTuig{{kr; ziyxH`_4oHDjzH$l=6z%eg4KsqzJp#U?x@uFn}pLfo`5sCK(TYd2zi~F^||Q$g$u-* z$N!jC8Kt=fR58+2Pg3Vp3_KoE3dY8rGA2?755B>zM41uvLxCL0UlrrpjzqnxYj_J@ zUX+kGmVM3^Ih@WKcr)@)sS$3wwYTPVs(1 z5zqzlvJxA<{Ndg?QVKwjpn(&CP3U=o^(+C1w4OLa$Q2&5AV{I1YO%*ghiff}AOj{j zn3i;49F{C1{z2A-gY2C9*K{I?hANs2Hhw^-6#n6zYX^S7nUqoy`V(+O_(VsM?4xM}SH^QOqM%Im1d;R`x$d zdLv6cG!u=LsYaL>`&=2R9)Bw!Ao`DUbGl;lcocW-dMeZQ5l*fMq4ZSjI=Dix6`#Dt>_x`CFIFSRF>pFgUFRX)HSDh)-I+Q6#AqEp?090E8-mBGHOH`6hzQ8uty~7 zpwQh}dhut(kuuUy$o7c$F6?(WJ8rmXUd|KypLe-HIAGFsOwcLsQWkDHB)&aNn@5#F zwXL0pZ4KV{F#$Gl8z6#2wWBWdG74#2f@Pff7%}61`(uNdxmB0Y2F;3=~j|u zQtK50f9G2VIr6nPOLE`n*q6~U&V05b!vbH^8q2J6DD>bAB#KMQ3j_pcoqG{8taF9t zAyjvvwg=R^f!Ab8Khr)xT{fy9&U+natv@U7jXf|lG4C(bpa$BP(j(%lf}>*^acDsh zG9pbcYESpTEL#AQafUQY{&MrX)G9<@e@D%l1^6fSWB**C$bDh3+>9S!K}$_eFGMyp zNO*Pp{`>FO`QR{LyJE$jRfld}M^;G#4Fr-;T$T0f>sgY1sME8D^6&;AF)|MtHlGxb z;!QrPtEuLv{SIW&0jjsNB zu(hv}!iJ&UhX5UpimQ=l6_fhjwK~%Bz*#RiC1rO`D51LZhG_%Q(Ho~5wkM6T^VptR zsn(Z0JYHKzmX#?1vQ=|2ulc&_4|Vbn&WyD95^cxOI<2g%tnKW4ua=$R~*qAPK=R^UvXVRa!Zd}0yMGE5n`fG6rLJiNSuuC7xX@@%ZEZd|z{t>aC- z6!awm`_!!cu%UhJbC|S9+>PZW=Lss;h8lPwcEgIfe#3@WJO+#d0mh6K&vVuupg}Ua z0Evi+O*Gb)AcCP5F8%mEhg3-i2M444>&xf<^iwHfzD3?;2Em9QUUXJ`?s=HubpW(F zYm+U4ZGbSnp=QvN$2tN#cESJhWUu$v3uLjhnc2JcABae+a)iuJzlN`Ti{UE*;r*%W znJ>wg{CDEU^wwvgjmLbn|Hp?FSO34Dm+S1;Mj2X~xn5vc4I&!fO7--o{K8_-`QHy% z|2x*;-*w_q1ic=(V9k+s6j3EeSl<5rk=Xg`Q1Ai^+~nhMe~4Ul61G1&xgc}4!blB& zT2(58a8HT%=1p9$IpJ}{*f$Rw)Ur+gf_Qf?v{d*TZa|261n6Kw#Sz#c8>@ zq`bfsEko#VMpz-cWq+n2bNhc0R6$TBsuI6l7ROf+T!9iiH(+b>93@w;-jC>GTpbsb zScF50^ety=J1CPU1JXx9!K;XQGFcp;%h`P3d#~Vc;A_Ez&lA?qah;-m5342C`jVKhd z6~L~3(^a3d*Il=0tP;4YKnDIJ9z*#RXW08aZHH==nx-oMF%Y z1F8N?(*H)wZ4NtHBZmd1_k8!|r`)M= zjzh-awKW&6SCRvb)!f@FfK7&ns05jhbnzbT70f5z%s<1vIUi*T`DBNBYE{7pQ=?_V znGtJhe!=!o#-XL=t_Z51hpSJV0#oDU%OoUNMxMQFnDy>8th(%wUX)Ra-#E9owb%q3 zP(X@WNQf6A;~r`_H5cC>nZI|$<0-e;FWlVsqFlfxzYUbS*`LD6{NCDa$l<}M5J0Jh zMU_>X&d&T+zWvPJ{6sV+qHD;(EBZnYkdiWvCOf)p;G(Wc{;K=JAaR0TNM1?Q4qkyc z68q6=F^WD27*vF)ptg&i?s@KCd{*4L2WRaD39}vG*>9&u#f-+XdcR_*MUhBO{*2Y3bXF+ zHL27`NMHH7bhV%`!a)RXcNtFsEL;uqikr zTs*}5#?!1H>Taxr%x9i7CW+222es*?X5WA}zX7yavqKWC0A4q5!h*e03i;qVfHsl^ zj0RudsIWC>+_k(6$|5vnj` zp8!<~@jRx{k_XHf) zEdDoW+GybGo1ur!nNAUqFBv6D2P#1Fsi1DQ+3Eu?8Mu=ziECD`ru{}F z2N+-hsmqryDDk^&4y=(03kH+dQ&go>Kf}LUl zlDHgco}o!gI%{xnaEP?ILNGJEhU31pYpT(yFOxhQpnemMzx)NVO-ou9(_WF9HJvc5 zfAb4oQ5k@KZi%V!fixUh@Bok8S}j?w_wKz!_fz^uOlgs1Cr}@+x7=YsP}d_C{PCtPM0IZRdAuinxQ07g_1xf z+U0xQ*KXKwb!jlOL|7|_kd!)%)|8VlDQ%wDl$1My@~Rx7GV$x^`%}f9O~8g2867?3 zMY|t*Sp-B^>S-or0r7N8@bu;j*1I5XZHAht=84`pQfz_TB2b=+w8byIj2?__g#+l{ zxw}c|{QgPXg4qn-RD=?Z3i{4KL^Iy|0v;j=kEQ84Sv9;mpA5ZfZnz+-yD-$Z}}2E>|}zenii z=HqCkFffLnh8^Yh6TiO!e7Zr)WIbt)W(}MdD8i3#-C{SXg>p}1HM}j=v74ETknf86 zHd+QvfoMBh4?)tnB66M7D3C`>XD7pO`cKgooz~>dCj-#rmnJqTG8C7A(Zo)#{@Y7a zHbRu@Y|PHiEM)#JmUMkK`{o+zNC1swU_1q1@K*Fri!04!zG8SU(!>~^(n3zoyJ3)% zI*FFzUeDZ4EMv=Yu{g417`jvA4C@Du^FRIgPxI!L!A zkO^*q^if9X58|&9?DG(IY1QBEAU_c;TJq?2Do3X&q@LAMh(P7if|-G!TWkORQeik= zfDp*$1H;*MJUQS+I#zup%zInb2<&FA3HFgJ%76~kVe76}l}=+-=<92eMkf@}UEmbr z(3u^9&@ANzXHp&y(=Ef&yPX0$^rkTEeDeDt8zoL)LCdjL-D>37rLX3l!&ocl&-M0j z-A6&MF)8Kws*ZKrb=Ta|7ViP_HTch-$o=8y2A{ z<0SgD9pRVg`R2DI-P#>ZEHvoeH}G>jg6emTOH2V6wiLn@wa?V1?_j=ZzA9}t>WN3Y za0sz^C}?nyq=~Vi7IUmh&EH;~X9kNw1%FJ&vA*dY`?Oc>y@!@2;IoKDgQ@cl_&$(S zq{(3oY1tyLfceXoP0^{RaR#7dq!=AVaHVPCTq(n2M?G`jpaUpH)*DeE;hk>sIm2yE zy(OeVk)nYgNwhh|eB0~U^y_Wx?8pPn*MVrJ3dWj`1kKWnvTbE$gX@$t8m^4+EizwVrB_N(6A?PN%)Y(QXWXeO zNQ!6|-K;nUL9Rz`njx!NjNiGEn)*s7nHOsrvceKg&M<92&MN_yF|WxJng@dU2x&hs zLHc=?Gt-;f#1pK+ad^udQCJGwb*rINRYWli^4blB$|WcsSV{D zK>HhzDP(|!w0cCI2U&p)yxn)oOI~7lz52IfaZTZihJ+uAr{70!01 zWsv^YOHtpVpGB6@H_K03icEdv9R)c+;3qK?btmpjH9GI!RwFMkuxR%%Hj|c;lK=JB z^M`J&T@LMlvMx&3WJO8lech&lgIF~(cPOTlSM--(tc6=aeGj6Kx|2LjM!X`Xx3flC zmu}QfQvwTEf&fEOmzD%`o3S*{!ouAJ{GQJ%=^y5da0WGDk4IiQa=#<=bmG;aoKy@m z=}z(|0*XZgC$9(90sqBY$ov*2(~$n?BD|jlc#9!u6(%T`sc|3nr$qun6Se`0`sUY3`aCyWwuMCpA;WI9V9zp&B3vKE>d z^={DU0kQeDK_;5H{(yE!EjNHgMClpJces*uK!(FQc?%BgebQ?`f;#R z5_qV8!D<>rQM=HAXmHx?@UfLb)F6r;A+*ZB5nClG7tMSZsL9bU31Z_RfOy839+YEf z?LP@7fqZ*>9Y7&CN&vl8GI^)IQnvF?z{k$`N~L6pR2}x`Id!GPhw%wybSDcy zOIw?7m)|tj+@-e*>roqLa0PZXW%SlN`A)+IJ7mxNMC?!~s?`lN{)E1Eouwb}!f?EK{YPG@2?kwgW}*h|-@qg|5i}mKAGT+l&r}D=+r^RrKz` zjU!d)BE770@||W)GdZZW+2 z?dzsZ!YnNR$w0Q|8uSd2Q~aCDn1KuL0cWa!2F`LkJm8aq49aJ^zVCL&Q%ZuoOH+A3 zm{O>KIO0mSXXs2%?2VXZAN=RF*|24v89>72tf@&SGO4Cv_5&XYe$>@^kAJ$D8h?SCfCQdb29G*b zxoC|(tUz2aaEipnNw%fOOv7m9XL##c8EMrsAj z=wyi+tpekVRLF@F0OW6y{fzmlr}G&_G=egjc7RaH2}zU7h!(*kEQ6ENu&Sel6 zU3s*&CP6?z8&vgSrk-1%XVfG7^!N9BAt3-AC4l|x8bJhxF(MRx zF*t#gV48si=Kf+uW+Zjfg3e)n1!*zh^oL8eoR86E3G=TPYMtXt+!-C4on=zYeo&gE z119Vcx}Us(Z)UUcM3RF5Rhfv;Rfi<$Bcj^9<)~^3;zXJo^l7<0Dog6+t^-1n<6-hK)^X;yqr8uF!l)` zhy?`&ty{kQk6_m`XEH#e#BRHkd0>)Mo~QQ+3@NK)HC-VtEZA#7bRmMEGd zu+d@B^cT=dd4M9<2-OBvr)Zvq^XFdx6!Y72zb@Q^T~zr4Ki%H^8jekbYeC{fAv{q4 zrtW=-!8d!o0{^6tIXc8aw_Q3mI z3ad?`eV>k5eVQ`aE${I&gP>amvH+LU#871m#+qO(Reo`?U@2n=0Gcbxkaj}wzse)Z z)39rj$~qDN$ppBCMy5gLg2EVzZA9^dL2=Jzq~w%Ku?^MTzPFQ6z|LI-PQZf}4hxvw zF+x-GfbKx)z03<|E)jze*Qc<8=VifA4Pgc;$o`G_{Rb?|@E)YjlXqW3LnGJ*X%({< zlyHYVa?_ciij80Z8A?fuT)uoc@zXeI^K#zcGz|hpu0entRzK0uerSfllLXyr+L48$ZAgjg;zS)c!HBK}vbifj}7jtG;`} zhKKG0TW$x^Q2d{{>!|&=j`07hT>*@a2=>|koWsy8x%j`}1o+dymw)Zgt+ZC?f5tM; zbe^-b=x4@P3vROLqDSi{zGT|BO7QgSpX-zhJT<(dTn?;BENnW2SIJfV>EdU&I&ylc zyaQE6IToAX7Fp&nHnnCD_V@o-irf8Nb0$s}c61ChjiAO|BoH#9f=r1U=J=OY_+oLD zX^@;m4x?mP9{uu2WdL+f0H7mg7>W?JlTfE5&b~AaI8=g5kOBa9bGAgmY?itiPJE0E z)+>S#a`xu!+iRuPqfefXsc)#F%#Lz~@sy+xSd1HO{r{!Q=s4&(u$tsQ!2gy@Aqm|? zdI0NfIy%Y_HhP2gsgrVAtQVAKvIv&GWE%PfLtfDasErCbt0xxi=~3Vex~#^(SS?u3 zq-Wihg1=>7LZfJrFoz%th9m!1*!`7n485JcCLD|H0cab9vd|xQoOxQG9D_yTwa`2Q zy6D$(-uww$F|8+hmW>9##_1mCesk-88D`C~X5&4L?^1k*!I|nTkb8ce5Fw+b5 zKkx2(3Z0(bJ;ml0Z(^s9Gg*Gv7#V058yU8r_HcIHD{{HypvgE+Poo)?wcDBN_q^Z# z5|M)^<0j6*7!ts^#(!%N#ePA}6tc1VH$SbfY5wN_@24rhjHbsBb&dWXFcmZhs>UDw zA@biMm~BK~3n}Tq*m1HnO_{zdzyIon0!$~-pQ=N$s&tIsTBkUO!8DyX=PH(#PcT}T zdVt|>E(Do2e<60)gEvdRx%b{BWYD7lWQ-|h{(4KO-HWW&pwgtg5{~{cgT{Xq3TBs4 zll=W4CYphdFlgS~43x2?>>&SFh(K8+%@$;$Xr=(pA|klRpPc7H_+(Cl7yLQ})(%#v zuGFP>FfkJ_GU~w5qelZOm^J>p-u>@9_LzJFeUt5_zTP#<+)ea0e;SL1;V#14;B9Z9 z9QvCVYL*V}p0r4)rzV=LMgB@^*`+zf zpa}+FZ7E)heq~Y^Ap6j;-xx0@1@jbK)7hxT$M~Nj$(Wi+Ta!(Pn zzOgB?lz!BMTgZOax6W-Qkh`!j0?q%)&x+?hRhsUJ|L>VRzxLtg2mJ{e8CNRx&6~4N z@52AU?lpOAhOXx=sH+1(pZpXw-@MQ3>%Fq<`sJ*6?=m9NAmc@oQ4>T3RlgvpPl0E5 z@|`}JGka5PGBAgXh@1e&&UI*CkwMZYtGwiB`^vvwXdP$&-7dSppdf->obXsA7NIgi zn~oAZlB_XOD1k>J;lgeCeLR=b1wDuEVXZ*xTSePkLQBke1+jw`6aE(7Ir(ZyNewhk4&*T&>)+tyoB+u zpE8}BJTmUjz1uZ%UGKQf(#?|Yv&EDoo91>ZXWY@{lPWYv|_n>9zs|Hg)Oz=U!tU82=_a`wWU|3*X9=dFXN~8s;1@#8Hg=iKFQ6xJYN*aP% z^3jb9L&bWCK^T>E7@Hanq1tY3m1v2v?s!Oil&3SLU&f+Y8Sh!e1%WupUR}%%9yUzA zpo)Q7A+fSF)}Svgt|vR@kD(^lsWB`|;SE#_#xyM&<`J5U0VNF%-@*oimV>ATXf_eD zU21xI1aLllMAtJ53l!{Jvi?=O$Cq)z;ni_lE~3YWrt=Ut`Qn&7$i=Km2ma){c`y`g zJ)Zv$vJL_G6OlxpTtU=($h)QauT3@`eaZs2wLDb?pq@Zy4Px3vvumt+{>#QO(C}c? znXsr;^IEdkJ+ZAO6&p&6J;0!fOkK&m)SWdSn1e>EV{}wA3{5nJ6Qd_aG))(o3wvVZ zM#JEV5h&X|ii57l+Y*1rymWM$twRL_Lw6|L45Mhx?$55;aQp-h<|DpEPu>PJHDa@e zfq9?RdP2u;N&=ecf9%BGC$4P9pKf#e>kz-SXks4u@}(-|>pOY!eA}6fI|O9|?&U5w z|D4`=Gkc$Ef50_SH!)_HnD!}HZ-lE7ef$c?j|akd9R%Y~7o^0_s#r7v72Y6s88zUc zp)?rOY6OkLM~@uzZ2i6xYf}g&R0d-Yb!#7Ds9!X6jOV`d@SRNQ0?e!MAQ5#gD{DC# zpNY@fw|{>DAYnt0L2Y*SB$0?XM|@Cb8I*nk7l-7T6zY1W4jP)q1pAOMK4iohuw+UN$jHU_Np2=_xcEGpTyFtrN=L%B-r|W@u zZ?MFj4i!eyw(19|7h<9Z@r?z=abEN0P#!9EA;l>@)luH_CwNmm{G@)lVJ!hZ;!X6K7D0%tD+7 zKE^7RC&h_hWVYRdOiob5t-RhrHZ4TPcI?G63>DjoK3yEph+R|KGuH~uW4Z7QqVP4i zB1+I$D+HW-4IfKVcoLJEKfwf0J9vdL_Ya(YlJvb~uX+b9`uL2!UT89k9pAY1C$W2b zkrS7{`~rDvAo&e8{(SOw%Hsn!n!s_0!U%blOteuAeyQawrUpX=aH!$f%)K_Z>u zaWum>n;>|PY?`GwCBwOJp}}biUFC6$|q>AgQ8I8(h zcn;CmK8kFYfDamiTZ7`U+L5*!pqH+^PK@HL-nt#eQ?HOyuZl({Nu(jjE_%(bm2R!(z zmG210w^?FX76vA-+qO*&>d9)AyTV&qZ@_tiWT1tnvFq8{t~|<)Hinst&}puOo_#V) zP~QduiGy6vwfbHzn!yE#6$Uxg7?+JblsvaqITl57HDqJb2LrI4;~rZE|7lz6PYQ1P@ji&Q=TbH@-NcT$@(T6&-WX*4|!}M%C#f zXysyKvf|xM>r&o&cu(E;chQv%*ipR}Dv1@wt)7YOuZY{?2~F`a6i|1*9IP*$H4+NN zmHn%uTCe-~7~bMnecg_U?N&&OE8+CH4MB1F%ae$$XCp@%&4&lf-aos5iJUgUu|Rni zSGmMC3l=2l4Hutuin1)m5%w3e$z2OD@Cg$m+IGLVb4TnV$5c$FI9qu5fU<+|1lS<^TlEoJ3rg_Y}kZ& zEb4U_OQO9!y*``^gEK_Yb@_VC`u#lnh|L+W{u=^68KU326UwD* z2YsT_%9k_e%<;oR^h3Du!_d}Ou%q8W6i->UuW;OaC*-Zw0s_&q!p&|gb6|IxVHlKu zQ%b$LC??{ceBU{Bf1wx-!8QB?3Y^W9UBbe{H$z3;>h%hZz!K}vZDPG&3=X03d2Vin zQ&i4-AtW2K8);Jooi+;gMV1zqY`UI3+oPXcTZD~M{mZKFbzTT7O0gt_K6$RyG@46iwk z^j4_3Lq4Q|Pj8X&Bw+W~kf~mbZV5}5dgL?ez$cfHj!poC;_ed1r6p?aHHq8sX%;~s ziaLqbRD2SN&!*g!Cg!Xe%g-Hm_8|8+1j|ZR(YgoFsv91y zk9k!v4;mK@^gARi?d|OygdL6GP_)W5Ju)BTXgM*?_=TB?qGqDtWLmHdLIW{npt2)E zLl5Gv5+fVj)owhr<8pm>cFtC`{798xfJ`W`mkCaZ#*?FG6ZeTe?QHA;jpK|@03*#n zd6h^(_&lhSiUtNWa)!DgT3TDzW3|DebymlRlp9p)@0))y=OKJ|j1j@qAs7*zb_jiq zO4w-H*4l1UB_saLsT_P=HfUuyi6N_~7X1NjDvq7{zTvl31}q8N6X(pN5z@eN2FbPc z+6wXiUv+2x73F<}aU>_TIY>P=#Vx2cabYnmgMfs2#=UIygJ~kyPRwCch^iZbPZzM<6p*yzrB3 zhHpuO_)1gOcqM^a%=yZ#J!@7^Mm7k#r^{*EQ*rsS-_)sph3nt;XpX^n@??M*+)0bX z>(L2sIN#L}SKQaXTooD`T4E|4gUwt$4fc8d{#>{@;-h6_VjNQSu4dhJ-GyN)sKctg6Fmf(m3NVUELOqxu&V}vq>G=-5~ zcWo<1FD0*W3Yt=`J(iSJr$}b^_SM%X-q9CnYaOJ&OD{LyuR^m5OM-R1ezN*W8^z`;V zQx`AJkOhLjQS7!j|JN&m14~z3Em>b^a!bah>u>DjcOjBV1bSn?*e(2}CMilyfD7tCfg#oY9u%X`%!6SZ3H zJR~y9p3bM9N_QLFWSsb7AHCP83APBtYuE@eldBW{h+l*%YhjvAi@4C|bA=Aa1=Mozm7NP=5oxC zJ_Da9-z)o|V1K{ncPWYx0%Wt#!{ImxixFEv-8W}CXdO8?In_ZT8mBbB^8$1@WvBg> z`N;;u(59IBAQ|8!)5h4r59a31sLo0`&mor_U8_tA>xL64euBc9r{^SeTO1Deu~-0i z%~iF*1i~#_w#4-fA$-9u7=CQpXCV`Fa&m+$9nNZni7|Kf?5(@kK2hSCLz1;vm-kAa z{Za?w6OM(D(3mkz68L%@Iz%>!=-}HMu5RmBoi(PB`az6+JXLX_N%smG+Qp3?&lk>% z7ni>k^=Ta}p#zXLN=kn6H558SKDQ_4AcO6p*C6r?5T^FyR;L48zZGL6c0Ezf0(28j zWPFyu)HDs2*$f$Vh-2rjHt#fHSCxs|0a$peK4FLd*39GA*R%bL8RI0xb}WmJTB{Ym zrPS-}?nk>q9AUU9qxfX;`x5AZB>DiHV{dE(#< z?g^=s;QDYD$h1xc`1}T%4&{)Yo7-GWW!3B}(&Z2*E8>DGA*{e*x{(&Amhw)fr}%1% z72arE#QQ_ML;U^yv6=y5$YbcwL3qo@4YvAisi~PlT(D!tKvFbm*$YH#mJP-lBi>)6WYGeApO_y>jHM_(doaSYiP_$u~4>Fk>hzEELju z%hOM?;w@H<^UNF3zw9%~{GfvrcRqXO37115+~*-&wysTf2nar(b83M`lfB@q5(a^t=V)=QHq{48;n51r zjSdRd$Jgkhu3=#ra`gHPFoi1kkd>5EL32*4CJy~@(8BSxUvJoxapJ^xR15|ZnP{@z zT{NCjw{BgGOKQD3HKPs_A$e^Qta_Y~T)0o2Di(z%!}Plm7~2$cpRpWH##=f4Kn`4( z<9VPJR7EZYJC_12=fcAkolJ4tNwGL&SCs~s;r^450gmPt2=|qw%QKNv9BF9Rz`(pu z@nv)UiTuRwA-$H(VyzBf1UH<`5Hds93}I>x+t)BaHmQ4v)kDm59v z7elQtIUmKK8u=V&wf4`C*Rka*_2svvkV`lvYJ+BDc(1L;@FlR$@t(UzE^r9Hd^_}5 z6dg1rQxOlyKpLeUNSYP7b3o?ekm{`@pd>3o7mM#`8wDgmEs&ey-1w75whK|(iK;Z;}&k z5ve3^NOXlK64CLT;Pnlfhp}Izhp$*5>o)LK%_ONdY(3L znj<2;#Ct~wxlHzbfWKYZD8hujr>BUbswq2Efq}W~mNHpGUjweJ36nlnDz5~u^Zm-<^$pVCN zwUSgJISuDTpqGfqbQfs-=H|^}kNCMF0d`dg$3p@JHD|I254OnuyB!nwklx z^Kx?LaJ8nu_Kj;;KEc+UPno>otbq;o4v zTW_4%*teCco&jMQux_PFq$Z=-QuDo}gHBgKSxqpFAwgjyna4*lB7>R5o}0z9f>7+>;h|>5 z%BV>z=h8(#-;7hTk2oqMCPp5vtehd7B5`$Q|8EKs9n{8(y)t3JqR5inRD`fUf>O5S z9PZxRd&d9%=0fuX;%W9d`ClIY9b0=ZQt$5vzIikGdE>jke(A46@&EDDn_pXmy?5z{ U{sZb<WPDy4X!%hY|I=YP% z+0!a?bgO&m=zhEK$6EYkl(m!t{}Hn}bHPT{+`z{Es?~Km#j7?JCgwIKhS&DkUAM9} zG&eiKC&njuaL-K}8w+c3ety&ce1Ol~>IVN_rPcMg$T|zzi`H~>j91A&t74^N4C&~6 zoGGVIsND(~Y<6%~s9i1`oo1)Xp1ihh@9CZL#&NGw{V(yR)vEYazEV$3bvIKxFK2sR zUdu@<`&^DnYl9JQqDop?I*)rp?eBZ{i4{n92VAAf-qr7x(Vg}7~{;GWYpsm$?^-2Y=4|2{ zN9x|4O4Q2GsCadztEVR>JbX{}>x-(=3%^bU>9HQQI3DJqNXx?Pc=g+>r`D`lQ{337wRi7c$FX9rG^~%Bwc?C(L#m-_ zaz^c&%UaW|Zf-$gPS-|9N3(7EDotD6I=*`Lx8&vUXuLka&(B|PlFpQ&Jm5NCmvcATna?5zJj7?5fFHW~6iMBeAziHNOI*>Mc+YX%D(?L}@k))HCYT77rNIUb#p;62EEU z<@;zZ4d>R=U^?H^qIpN1p&w>uZ^Bf&G>R z9+Am&oK$(dnS&+iL%e!|8vc)IB*%5p(FJS6b2&Rz?itUs_J`FY17Iwr?T_)A(_rOVE=ON!NE$OI6 z9(;@Pv6rJAg#!<^>v*&8(3@zoE;sD2iiKE96E8hIcETl6?D|{l~Hwk6nQB| z=>-e->DB_bf_?k;+4TO{n`YL035)wIBO@c-y!8!#{>(|@k)x9j298Zusp2(#+S}Vx zjcfTl*`#%zZ0D~rFIbGm^5#h7Nr;#=9fiv_S3de!$m*o9TinLNXnb<=8T%ov^n;Q# zdG$NLt*Lr_@d%>hNyT6hTSJqky{i`rxK*Q>Dcx{odn0M5sn?@pV}e-OoW#P7>(?LI zEn#P~kZ)AYv%mQDrL_J{$-)M($iA_ok4@{|evlPpbD8jo7{UWYUwC&d8t)a?vRoQ? zgp-Ho8?#=i44aESd*EvNO_5?tkN(>4!Jgu}ILDOn#LG(rJ0bb4C! zMtP{Bb?N8L%VieUeUq;FjhFMC>py?_!r|2T`eKR)`w`QIRQ-|dJ9Y$C^oc!w^5jSW zzpe=VCKlPT0@s+zxn-va7*ues!*~$F;}==M7^N54iFJrlN4mZ}J2KsDuN&W}ljm6d z`+U6-KAzlDu0tkW`oY0fz1>ZiY% z#`EfXRPQ372MS*Q7CFN#N{m5qxG76fR`!v@WVL1)-n#77nJ1;4#lDIO8i^hO0Re{C zELyS^0?_~vdiJIA@V!2ChIXVVmal{&~>FJdr zW|=Q6E|z864p2RFZ2DBjJDy65yDHb2mxW3;8sq#M^Dd^UL!GRth*2 zFT6btL(#xGJ3Bk8y}lrjH$R-?{^W_gZ~(qsM5EE-=y{x`2K$?`QfiWPlRiCUjJdUZ z%wg=njqf2DzXo4$jPKsP`{v8_G}L9Po_*TW-CYGP zG1*|86!p9uo}ixRXbY2*7qja8;WFLa!|cj?nav-Ms*s1E!*eeDHntX#l%~JTw@bw2 z+`Y9MnV6XmNJ~r0Fe+U`)~c{BmyBs^yD~d76N}ZKLs0fLfqyQ?!D+^(r!|)rW~+1T z&A)v9%w9ii(DRE+YI#s2OxY^kr2f_48+Y;S*>fB>Ki8aXJK1PnAZFH@YUm3zlRwv= zz&?FB+a?bGv+d9&8~LO&CtHpfl>We{5>Gy0C@wEoK7amvad)P9s}!Yj|I=gC&h)|h z6y801-j6!Lq5SsCY#M4U(48Ep4JarmIC=7^ z(n2&ZgzII2o{Z1GD$w@^kj;1Wo0Q zyF?U(rOnOFyS~4WDDGLr!=^b+o1Kx7fzj2~T|m^jh)XLC&5rj3JbWk0bxt-4F& zriL2jlb>GMA?cX<&5t{R?8ecf7qGRoEbE>SRy|s@V!Uoa{@uHGI|l|5kd6V0t+}Pb zB%U=aNq)dX5KC!>MPWbA|NQu%`sFG5!ahOGB%SK_cUF@JShIHRx7Jo&VWX-a040^e zt%%xcRF>ss!$m%S3-@oxrZ0gazLl4kkJbYZJJ|MDQSRKi^X(aLfVkaYEYK#Wn7zbk zkMZ5keBJ0SbJudo>6?XKyDy~}cy>q_SUq*)HS^cImSaDnfxzg;rObN0JoNH29(6fE z1xXPRjeGQr8URIpl1_F=!J1^1t;7vJO1XZxyT;y6Li@?yVcC6xf+{h}Vg3SVxs*ez zV!~YGk^K4ri0m57N}mfDHKfL2=aaCtMt~(o2q7@=$giHeTF-9+D6dsjmW)C)?Z3w; z<;_ljL&3t>6CH=HtppuQI!!4EE(reYL7DLkCO+9mlOf)VHx#yLJAUTOnZAgpQgQ-Q zz*;sVdDG%sH&R)2_hgtg$N%{8L&DKjG}~?{=zEwnxqp9B!BQo0C+;lsi`QV(RR}xPvFG<=?PcL18i+Pj=e?d@UI@5pHl0O(*$&hw zo^fZYz%}EM9r^%@cES>Jb91YJ-pYbSm4T-GfEGHI5FhfcUAtD5ZrT{!ovc^5N!YOR zG~=;fxBuR-V;>tE+af%ncw5z{zc>2AQG?C%<9xZ4jV5|4+#fw670j)bG;D+U3N@LTDIQ@RBu1fjxZPV<;nI0>z?oKh<`xJ?DgwC z$pYhz`hkiN8Yap_TO<1x>Z>BVW}60u6V6Bc{o8N9O?=&DUIg5Q=T^f$q#2a$B!v^K zK?Nm?uyM^nK+cN3r6V`Kt0BFr7q~8JIUd@aidW_~`rl&t@l)K12=Zw!mP_j(PPU1v+R>XIu95=NAo zpu%IQl7~3?_>^F-AqL+=;@~VXhSGL92vP9(aDdQQtqe0qKnI#l*e=9P3LuAuzJBC3 zKJ8fY_5e|FEhE5q&&mu`A24h+6El!QF1DTdb+z!<0QRmA?uu$P0O4(9W^{{G81+f; zHsm5rWX#E_Mso?p`&)PKK7`$IaCGbkIAvJBo(8K^Pcu^Y+$qe{th7UJz*Ry@N~bZy zTsfW28Sj;5QZINp)8YgYicP76cf*pD*mgc`i`-BEsA|CaNjy*A6aIXQhT)WQDH>pG z8ekdIdXM{JTG2``;^E`*XFYckl-lAleVK%@YE}Nt?%tnIjhY((ggZ{v-DtZJp3Sc>=B4mAi@C+X_KOKSn?I(vG!D8T5!9hRi}fKB(3yTVZ>O85)N zM#Eo8q2tG|D>XUPsF`KeRavmSp>zjKRolWCS6Uu|}Ig<2dI`MJETk^kg{nDtpi zXf;Ip$qAY=36)=&VN?nvuaIg9MTsHOCq=5nO#RYQcZAH-8a(G~ApBUjxaRdmqn6yP?(Z)W5NBkmq?F&t zCZ&z%%w=_*xr&g1G^p5+>ySeG9EMnr>w2cC0H~COl2UP8(QxK^ zdy|JiG)(j>LbEm&Hu|Zdhl{Lgd(RaQ0Tg+MsimlsRdSa-D+NEhV7Zi%rU6I-qJLHr z92y!LTHNA~ZJ{A#Fpap4by7NphD>*gnhgpKP9$cu2vS3q3`X+j-w=lgL6V70totfCRC8~QvX6WUezEUdg2thkpH_x0Qlhk62Wvd_kdVz#Z$$(r5MQ)G zS&&ycg5?en5HUYfK+>vV!~t3&|0Oq@-Ge%$n+MIXOw>NmphwQuPrROTBhWgfy75MDN(O z%NTr!p0kTf8BmC-i_$u?>E&8xJ*=q#+0RrWNWs zXXq)5i2bd|P~R5uT^V9!SE{tQbd+L71|r4t8?!g?)>_ zR$dJo=X4fn*QTmOty)<@Kpat*xd{PM^lccu{PxNntpATgT|t$9PNxoMD8CTD&K|h; z)Zgs&W`lL_&Qk01oH7KyK4;j<^Bk7LQ9q2}<5f@wLL)?pgEGUWp+Njzp@^4n`+h4qa z<>-$J|F^n*jTuQaz1Np>MCxhg{s%9%S4jaz*zBf;`tKb!Y%XBL=VS%PyUXOqnAe55 zk&+FYrj!A*`2IURy@bnJW^3b9P4tYcCcx&`z1XD9G?H{odDLRBum=JK?bNGWy8Id` zv)Hv@4!+~bJ`O@gyy4B|90LNQ^l#p3FR%Umae;31q`6t?&_+w;VXF6yb{5}V?K&A| zmsQhEF2>YyVWZU&G|K!Aoc~y%ff&FZ^Q}n%kxL;?C8~1aSeq-#o6UNKUe!o`Y0k4& zF7rbfgiHu^TbgB;aW@~Vm#`lW0kxnWadx9=OYW?fbrB#EXbHdWK-!iZfsFgCcR}kM z5Es7$ETlA=k949mC3^$a!|sfj7?_Xo>{u6hgGrJk!NVw~9M{cvo{g6D$Nnk{mxsB| zQ3!HF$QrDQ2+RynOJ>~r8j01a=X)Y8Mp=UIufP66X+%RDI#?`~dkDZ1iPWZP8W{8v z2($P+$90ms*Ka#0Yy0v2rrqL4Uz`Q#~ED1 z;EOYB&Yqj?3ERrX#;@l=B3>bzoAy`cm>!J+SC=zq&x*H{lkYNYI~a>S5Nn^CosBXS zd{Y%UvsFv9;;t`Wzltw1@@u6lATQc4 zTsG@Ull}2duTW^?z$Dl^z>$f$fh76l>3Eyrri+LYRe1T3LBroj4)1V0`E&`HXcGL2 zn55a=s;Lu5b}A_Ae9J}w$ z9bI1^52y$_O_@ygR(z@mZcQ`RTtxjIbj%@{um^-D=@TMIq!-D#0bo44GpuIO?qhkX z^ERXjLQ_)G2U-gXI?R1>Hx%D|#UjX~Ho1S}S*i$dMM*EA#3la=^mP%lZ}iVBQZ$2z zsEUwbBp48+D=FHmKz^uc9(d&LE;sIU9qC8hVJtG+eq{KTB6}P%CC|lV{p1*jyqWeV zN(JSpZIG875r9Lc91u>_P<^nRdJm9-jON!}vD3a?V`>BdPaT+rO(PjJi=qSa?z3Hj+ zQmjdRve;rX0wbXsTkXCZE(W*0LB!^$m*((LkuQUtgJ+hLobB}WsvW?J5j#Z6NP=04 zE431bLU8tpc2WIkFWMx@-0sCr@@>pddjby-62b#?94U%HLY9M_+ff0$JoV5BxnD{5 zt?I7)pl!`L_O)P>sz7dUk;EOxYyd@PT2%I>S*g>dX z#O;Sk6p=h|pvdxj8*)SC!X@@U7#Wqvtv77gQ1y__H5Lr!IRxe^_pQ7_*FOINM&t$q zeuXMvccbI2TSr~EpG7*27)DxK&0A^wx2nqlRM*IE|A;2+zRfrAE31w zSl(8CbTjY8Ji3NwD7nil`;a08? z9vc+4YHYl~@*pX!j7mVxsGbO(8RtH6DgXA@&!0&(>UD7djfA)d>$e+W=i)D>M6=Il zHK4e%Y~uPEGKlQUF)N+7+pyW3YeAvp#(9tew%w(H?2Wb~EzR%7zLou55O7GV#*p{8hHDhfv+1Y^$(uC=3q}kAjzL=GYBY z9TIXA!;nKn~V`U4F4i5NyV=cgTD3C_cu1{W=HRWU+Q;&wMs= z4fTnf)vtbH3>V&Bp{_n>Kb93-Q!qQkqn-K7w0vk3IirXFHEF`vClChNo!(Y5`|`Dx{S;^Fm3{W`;X`292vB%8M_Tg8+kau-J+YQ z-Uxgq%`vRQ0Ln^*H2ZX95(?*YpGah6ipt;o^pl^rDE9{52(9qgaYSX|uI}3_C%%07 z(g!+LiA{(kO_&*rgBQ4O@C|lNtlPHPl^7S|&%nDvR@yG}T%W97sWui&|bBaU%sLDH{o0g1yX%8rR&i^c947g<^SPiO>Fr z9r8oSOD?o)Qi+n*qS^KISGBdZvqK&_NdPfMz@+;S-@c)HLF&nvP{(fqJJdcJ+wmrS z8Df1+DnY^nA{fWstaue9Vj84+Cv;mzHlrMOE&3D2#Z->DU!NG&bL>oGS~9hs)hxG? zZUfjDf5HbW3tY9z02MlbG?G zr22ZvCN`;%$P2PVAw60?zm`LPWpm~Ziz^M};H4Y+TfRH3;3chHN z_dD zf4Ej|lymB%@fdjA=G5A}TchpNNYGwJaJ5T-)2L{p;UU`S7geIF z3h^)T8{A)t@>Ed7A(EHDCbAeAn7z1S3qYVY;kN*tY{f5$U;pN3^Q3QRD3PEhWWY>h z&+0rUd(uW*y>^3aI@gg8*dm0Q9Wd_l3f_H}!_=NQr6rwi49Zb-@4@fKog$b&ns~ph1UM zKgv)9Vu)fj+^nF4^yP>87!DRS*90PF0;b0ywmA<=Ewh5&}WVV(xAu^4dn$V;2azu&{Xh>eZ_d&Y_%VwfpiqzrWD>;=WbB@(BwY zThikEw8ZWi%7y8pFy@ISdi6M!GyJ)~?h+OkvBQ{D@&@me5>l6zS*4sWAh0M;UHJ%( zFBVC{5K%URMfz%;wb7_NEDHrWH&{e~o(vYQ#CXl5Hvq*0IV1TrbSs~=uZjoZ!!DM{ zownL3W8YOb)*U z2ER@L5tF)bl$YXv-xUS1LXlGAmCi;w>89e7023Wq&JgLqDG znV$+0&^t@L+pE<4ZAmiA<>oasSk4n7&AjDUOnK6YduyYNTST*IBiE>l=DF9tFu)Kh z`t}r+N!k1Ea`8D_x$i^bLfPf!fskK|?@LN}6~jKQr?rwFNq;ECKhg-@<=8ySk#l zr&sl*fh@J5hJ-qgQ4)0vbl{STQw~itQ6--~ox=SqfnLJR^JmYZLwHd4vC=@(NAo9CwUx0ER=@;m8Zitwim>_Xq7}bJ)L0i+&rgTUF zNRNRw8&^?LQM`$spO8NDL!vP+Uq9J)D7&xHFitY86Z{E$&yfTb+M`EEH8;`nCf-O} z6^%$32Z&6-7fCH}RdE-?>}R#@8ejzu)&DFcSPSmuJJh#0m{QJC>nSjY|B@F*kQE9z`pVxl|PoU`%MBy80P^#;+k(DHe(*q6Wi zDtm)8-^!<5XZcoGNJx#W6v|$hKIahNF2UTzzFdUW^JEcIA!Qek<8w3JDq41{|6te| z2pX@W7sTuRtjc#7 zHyEl<;i^_&4LBTyVhu8otCvudQXBcTGuw`~X~n52432Y40$SyvTy01<4el-vL;rWPmC~?N1>BA5;z=79 zWA}34ZT-SN-!qJ0AF`y2UVMmBnCv+O*rArZ#=aCI5Ra6zrje88eDwu4qHT*yC%ln< z2!z8%E7(nxdX^s=o$e%jBQgx7M z?{{RBfS8yVshPIb0GrHot1nk!z__>9dIWU z5Si#7@)G3}#Xq#~$IE-*KszSt@cdqkPwzAST`o1RfJ#w?Ku%xXM4h}#XsduwNmP4v zdfnwbM|BYK094~@*p(RTwNTr{f^@ry-m_QK9nfc!h=lP_HId#8DfqC${iw#EL;8f4 zQ7BW`4J4fjWM{%l8&yU=29**Cv~Y8@{R`#qlX**X1D7%_UXs*6l0O0`$YSc!!niUK z-XMf0(Od}bLSRrM^(x@=ZnqS_4Lbx+6I2I``Wo7!7snh_VBV@ID12W(3`Va?JxS*> z-t9Hf`=I}wfNkrqjC#U8f8s8|1B7ixDLw%&gnH$qxK-yP0uXQCrYo#G4IxiH`9Dzl z2?#z!;zKrwpjZTTE0&|ZxRTWT;1S#2A1B-vMjsLq1p(>nJGF98@JbHk(KqfC`az0R z1p7WLi0#B6mw&@}Rv?hFc#Rj#BL}u0{dfx(4=3kY(6LvkLydUPJFEZQ6}P7T3^jB) zWJZW9;hR792%9#jsmH5XzP4WK3ed&3z8Yrn7E7_%15y&55Rc}Y*Jy7HxGm3z5vf?= zuTLIz^vEbpmOP3epgEO7ZqZCfCYD9BezvTW?8Xw<%(=;#Jx%~s#LdGa2f|$qWVy}4 zj3rtmiXgjcfPO;yy&Hjac+FdKiIRyrAr+lL5GyIW@$1)s`*HY(G{`~my}{b)Kpjzq zKpgZ^jQu8dTxxm2nsl{9f)$Cp2L{l|+^rL%v>QDU6(g8X;V!?QhtBkceE@} zKHDGMJB5NG7Q{9>?zKQgsTMy^f?YjOe-|x~G%xE0Fen2@xe5@Ny=Yq^Q~6Ug&`gz_ zs?)<0B$1{CNL|vfn?oa_YhWOFkUdg#Y;3HORQG@VMOv!;#av-k=(6uaVKfBY9`K7M zdS1dNb%*g-eZVn4&`cpR!oIkeP^*I}c4$>a8dBo$lQ>}Qol?%3>1aaYMnuZwG(B7m zl1kX1lo`-XPS<7f8B*;^1p#qRw#Co8oXVlEVVN;Le-inmrmhR=th49%>0LvECkmA* zHFEKprVeF%3AkCe7Rj$F+t!%$W#TWqd(~kscJ<@$NYhH-R|rW#dXU0qO&6fT`9TzU zaJa^u+&2VTDiGl3?HQaJFw@(r$l`@S}oHfb}FGI}-PE-*zaTq)uF94wgc6dX%Yk za)>dBpdxq>JmKrUfVmw)AL8}H{;XSAwzAE9l4+EE>w>OTNu7RqXfw#Y77sS*7Q-wa zJ&)P;C;SR14LqJceaa060*~Xa$L+tX`MaPoFqHtPq9Z6nL|a-~qGf&dk+w@WHdpDX zBqV>TsW;De^!4>kJSS~s@Wt+G&2@Wsm z3Wyf=Mvj9Uts?GUKSIrEnqgU%e0oe41T%cq$Q^tzlp~45~udms8QIVHnRH4U|sWS>)k}j_aL}eEW1Y!h;D~75b3j zl)_)G0d3pRuGdt~Ia5;r15}apeE~s;)(I?x9?^p8X8?C3b>w9tCh4QcV(jA$S#Ao= zuLEpq8PF`@E)@VW6~VtLAhjqU?3W=>zetBGMv#^lXv`yI+acHGJbJWl!%m?h=+VSX ziPnXkhSnjwLvvoqB zT$(0m3?~;x>axBxMO1g)s73_Pe~3XZ2k#pUPBzLx+NiWF;E=E1pr9hyEH)|Ux>Td; zZ(yOG19m*;R1A6!GL&Z|D<~HLio#U$)|LgRnv@#Z2Q|$MKvza1XOqizmIl)NK(*zLe%;lLuHIMX zdDkffKtYgo0_c`B;)C0x#{CZ}lfoY%TzkjZryL>Qe zxfLAz^BbWKU&->le}8-&)E9YQ_KTsV%zF8YQ(7yIioPj?+2)`lf(EUHI429T1g5St zBYl}b!{b0(?%LHnn9}{VC$K@#_$zY?-_MRYOYBh6GZv7aODyr54}!P4+3!n-LH~^qe`W z#Z^Q||KQ<6V}u@fly460-D}ibyvL4dqpA(3U*!8FhB*=a6K9@mQ^&qff*H(_*cw3i zmPw?)c>J6Yw?xkYkS;^Y767kEI3q!3ac`J^Ya*mrQ?=)}2? z=lt0dzM9Va3`gkeg9*&XZoNg7L8SY!ux=&ajT*7%PJsVOq@6Z7ZnQaUU_+*shQv%z z;5mHh_Q4or>`U*iJs@fvIED-;bp}Spg2c(5RdllaA@??s$&TA=EH;U+jCgE2e&;_w zaomO;brSrV2W{l!dXr%#G9Yq0m{||}@)}5`i$2}U*WwL7UVhDsRT9#lbM`T0a^(F! zux76jU2`NhKKf0vq!^SQ`@Hm3K&9Xm@_{x!sOlItZnUlX!$O*2;vnbM3r@$6)2&%~ z1SOZWKb3@59_Ic319K+!K7dXa2@wT}jy`50(5sSYHuI<;YA^&HB1hY|?&=2C`-+rMn3<6qN#Ia zB#B6=@c}PWBF}=7;wD<0mr`%~xGv4!0FbIyS6&&u@xEj7mAPY>J$YwvSu_Gu0?0e+ zU|tWIJwS(!s4FQ{(%oQTc}-OJKwaS81LOku53p7Z@dbw^_8kU$tA>RT}}?D;=IqR7eB0NdfGHDS+=; zG4)V^)AlklGLjx%m)L6Z)|Hwf_FQy*U>>NEaG97X{HxFuaD|96UznAH(+9ZhsN)oa z;kiG4Y`MU?bt?zO-P2Ru#Ka`KlRQG(@F~eFl1#mUO&~gjZ~baxA0li@SXkID!Qb%v zD{ByEhm}@6J{PWa9GY!*3W%d~Na(wV$X~F4yBkA}IaKz!g14ZdW43!*SqVC+su*Rq z!jXSW4I3~GYywp@D2M6Fx-o>yDxbE$o-zINQo(s<@LK2%K{Gh9} zMcZwHN1?rq1x5l~3;27}?s)XS4&c^cJI_JHs~0^@SPn2#1jlhIJP(IzGH;?%p484k zH!N0$vMy|2ll{b`A(aBHq2)s8;lW(M`ydSEwA||0NR(Iz1=TPUqO~R4jzzFyS(pD9!?w*3HlS5T>>4?mw5N!eZigOABeR|mG^0dOfnN}9 zB__d|p9_=q+an2&E+UK+o3uiBXz}TKns|$=|?Fy;`@0^CpA}MD7jz z1VQ;$QG;RD-4fXnD-R(nEn=-VeXuDWIEd7)aO0mhjq8)62$KSi?i+K2xA&iUC`BEI z(rVc$plQg*AaI4gs!>hvG?WL251%7H{^O6Lw>|9)XWV5UZpxBakLPR~_6sz+SGq4a zXUltAbR%h)ZJGd78)$JFAzx25YZfO|9@XZ*V(2{3AW{Dl1}=Gxqk@75)Tk^4GH1?E zs1QY>Egk%!y!ZPWdamVd!e_HBPPlvKb+>!qcDGA^n#Iy5?_{u9PozTV1`riCS~fgr z=wc020mL~dh_M0eYaj<|<_Dg()Vcq8V=Q4|A;g~b ztL)-j^^d30eAcQ4xXt_KL*n8M&D?wb!i@rjjs4L%Sz*xuff}-H6M(d%ux7ToWn13N zPY%RLxy&7PeTo!))!*C0?SGf1VjkF(LAzUCAV954*05FuuB* zsdw8L^ePFMYVms6j|fx^7EaiC#)1q|A^3W;zc^}n5*2_=)5zz?TX>)fNPgPa(Sb%< z82gZpjt&?Cxn(Y&@7-m-xw*&uwBs8yID`r9` z%SK)oT#gYs#$>8O37jQaLa3IBg2iMUIwnwD^8F2I2?UT~8A;!gkjJ}@*gxGb)AV8` z`$w|XoG8C`Vc<_<4W~`7oS6v!G}hi8DQp~h>fvC)cJ6)E@f;40*`oaT0G+HdQ}5f< zG~+0{p~#aDBHtc%YAjfmlX)sfi>3%0XA4WNd`Oghh*dIjc05Y%Sx6&5g5$7QU%7O% z?{@P0NKyZb_o6OmQB#iwL_TDtUc06pMv^lffi)L^AS=E1Ru=o#HY$Z;qR{{2r{;iv zeGk8GsMT#O7vtAZ7@JJfb>Hf{iPaCJU2{Eond5UmS!^x@1q4iV1x%IHr%a_Eb7%_Q zvU2I)9e-RkoE?`h3!+Fb?CJV;#*%=~J>##AKa6_turWZzX*) z)njTqIej`K#Eue@zxA&6<`tM=wq5f7cc~k1dRrZBUSlW^i0mG$N-T!QhzbpI4AR{J zN7MXb-^xC2c}XwngZTzy6pbh_pgi>Tn!(+j?p zFZzb#i$s_M#xR; ziA=r{R5!uCqBmBIqDbL6JKq{`lINfUDijmwjAGa5j>!I!ahfDi&Mj2%JR>R0C`W`J3?Q(H}$2IijbmR^naerZtWXg!e~+Pz1YJ zfqFt6q6tbRQJsXJxM%YeT}3g;v7-HiF~$$mtb3GPP*8t+A(5dMOjK84&SQ@WdItLr zsVEuAss%M$Dap37R|9W1Wj}ygTayryWd3h(=qocy*k0&`sxd^tNpbh^P($yXj07O1 zNALV(WW2k@bRfL&3x%Fh?&V8rfM)hFc3iA&n?}OM_w1IGrf*qYeT$qr=(3Oo4Qar` zMxnX$hRE;@bgr_=%9rdx`ky&{`sgq;+_5P9jDcQ?xAp$J8Vdcj@Gmg%X0~Unikddc2tI#?jWD_&iTgR=8jrn|U(OA7kNN{Qy%$#4`zV?#F0B1sY4F_%0Md^&E*N9Gbs@odUiCCm1(wtVS2` zkbuYas;MJ7H)FlcYDSY3b!7hzs{rn4R}WT2zfv{S#weaGB8|w zl=I-ny--kA(k`vyku6YlQ zYW$WKRL3Us77N$lahg0wl50L49Ok}I;F?FyIf%?d_<%3zWfwZ3Nsx-xZ%jJcz*^wr zsF9F}Y&~%9I4|;36RPJZSol%L8J~PK7%$~$Fs%1YHxFQSQOoXOjdO7_T)1Vw;`e)w zAQrB9`SK-E-b&%TgP1AOpTiIOSd;8WTF4+OVt^!(mJ&Axafz4bq)@Tk1p1M7!|BJH zK^E2CUbUKO^X6CRSHw6;d0TZ+0mi0=eC>w)-dy%|XY%3Eu(2nIl1^|J;ue^9IWU4m zFot^hI;9ePFwx%>z8#;KAagWe2H-DYn8SL7SHo1pCG>0U|2N>sDZA(1fcw@IU2=~- z5r-+-JmA-)G0v!jUYftYG}i(K=Zm~qZ8YCDTzY8VDBdgT*t2_{taawDi-@t3y>XEB zy(RApj%t1O)leyNtxP?p-ngTf{K2SrEXKchpn300hvXuYoA<(r471_yag3G!y%M4P^hu`>1KmZX5Lp*Zz$&~BgWNrI1C3wmA0yiS6>9|a) zuMF})FNoowo?)A|wFw%O7vl9dtXorL;?JMjDXR3%=TwliE{|GK^A2LKMF%fn>mYAA-yc0)$zmKsjixA8z09B3wt|q8^JAD^eT)X1&f$k+3&Q7n7I{q zdfogaK%Cv+Tiub$GZ?3KaUg-V$eFo4^e*;efV8u~5R>SP)|yyz3r;c+y}nkn*yoc} zX`a$R>*;rSBUyhAh1&)exz6h0~153lAwC`iRVrZNod99QeAG%-(52` z)lnEVHPjru3-`KR`rtRoHID<-Pt}b7I92)c6V3!sX~~^Rf4pT#{`E$d-j*iv^!j{E zy~i20@hLQj-1N>9y-Yn?vVs~Nk=?$aCrDJZLvzbJJ`ie)v_~NUwXim@c+g!I6|c@= zJ~CkH<1_VgL}b?reRz9<_|*(%$uKgz7c<0@gY{&d(GR^JBIiQv$dQoDe7{|{&drO( zFzldNHBa2}RrBS~c>n#_U;fGJ@nq1JNJrr-N<-w(0>FaZdHwdj$hVjg<$-nzv#Y~O zG9%_qG`cs(C{7^j zZBZ_Y>i=&4$9iAzT?>mtXB-tga#li?_YTWVjtoFHY3Mo;NpsvH>S0oP1t;H?s57Zd zNRr#QQ`WdPTY@Y$UEp8;vNiI)=*3}Tu9}@fk{wBCkfCOZNo1Z16r3XUEWkcYCKtMQ z(p;8Wn$aVwPU>Rr1T>*g0vnipUrG1g;E-dW-`YkFe#oJbZ_?>oDQph7OVSEvwPh>w zuYOuf()-38D~Vi?7Z=71<*A2^Y)X}`fC6L+;J<%4*p#J;QU5E{DX{!EL6w^o{c#a( z{mPVbxu;S*E@O|!O%G{hb8(TI?YIXMLWcumcc-aSpuD)XM3h=(zB>I}!>ub7Cxckx z>~x-W`;7wAW!LF$0^awyoDYu)M{|IL7%-i?O)*DNOU~cG84#tOfqN6`ax~|DopQRG zf9(Mi6T7fE- zxxHi2Zy!HK>blvQ&3A5zyC7gVZC2ZPrF3#&{WjSY1Bz**T&wf`L~S&5a_rK2(BiI2 zjdQuV97$2Jc$>GvB?_)rid1@lX?4u#_74krpHU3l=ita`y4<+Tw^rPQs@-8sHp^OGD?=_uxv??na*D1Xio!)maMU7;Chz?w z66{hZjmvsp{52`E20%qKOv;4}CSdGI$#PkmoFJ1W>^O0An)Bhokr(#bTW+*U8J28w zvRs;ORk;$Lx7Z|x!B&r_BR?${wHG{hC1vR54<1`!#WgbudjuIQvL?Al>g8>m76zPI$ zpCtT&EhqiAvW2CVG01!D7He;rd2{qQK1-L$05>KlzOYI<-bDLL@as*qVF@__bQB}n za_bYZx4P8g{Nb#mB$uPJajLcE30k!-4jjzK>heaLU2c=h$SNO9@xO~|M+cmnBXq!+ zs!c?d!p+Wp8qwz|&tYN!-~>Y|xF zBGp$RGjwLobHF!8{sBYf%TvQoMuu0DWOCdQ$>am7x9c80VE7jn7Hs;fxCsa#<_?y= z=>==@?cSU7vpwrlp5qVzmyfC0wn_Yw{uSH!a%W97-(+$=47QwJMH1y6N#g7XSdQ(F zxh6iY=kLYXog;>VyC5)zdc+t?7Dt>Vh4N4^ONQ0Qb)mu`eN4m@jTB|+p&*t^sov-! zyUH#;Sigg<^&9;kxaRGN)5!LaBpE|bOpU~7Q7oDc7*c;iX1!2asf6$UXLdj|NyhB1l9UVFIbY$aK{P_`P*U^CFpnz1O;glcJJ|U`IRFyW?c0~-- z0ZCRNS)L0&T=NWR9_26fWFUg?Tt5LgGcz+RbsX3E$0&%bjYxpV2~A*-$SDRC@G)G# zB5mD+-dlyt`P3769GrB`pLqD4^-N~8wm);HB}un3{l3n^(&7=R{2WF?C(*qjU6?Y zID;+@8IFocM~AFVv3?IPP^|xsfWT%6Z%~=Lmj#+~C8X_H-GM3OnY%R5yv0E`Ol+H5WZA z?ogSQgxX>1KU zINp5*3yz%{__AGB?c6zU@REBPH*)1Sn20dg?0JIjEyD&sjJP;9dZv_+Lfrl?vL4jy zMCwOmrr8o(>CA*|~G*Vw@dG`PB1a?>yZA_4{}-()N5x7AEDED zcjZo1l-iE8j zAgubyJ)<&gv#}S!9g7YgrQc1bQt|Sh%~<5(+_z_#M^tIeSIM%foNmn#CY=ENeG}na zmH*-t2*r=kuhL`(@8$c(a9hH(k+ZAZyg=&K+f?T)wftP0w%Ze*x2TMF50?_00>aO6 zAExooo0z=rm@BI@5z&nL{e2k(My`_L4LTw6Dz`G3w)gn^;j`ZB&WCd)L*}EOw7Mv} zzcv6z$L$nRaDU_-t?1j3=IejZ=kDF67v#J{!S&xy(BD>bUU=8}bMICj_Oj=DCC5EF zhYoq}*zYp>Acv49#|=*02`wUn^bl0?D>lN_RRQcu+C1nogEKOOjQnf{pQl>9oG4_g ztcaLmo%m(3Qk`KJP%d|Z!@@j71tdJL01CCywN*h!kPP;Kr7;9Jg5vZA5if&8hQ_S8 znY)9JL~Yc)8NcbhGsnP?N{kXUCT3!?7H4e8Cuqt$V4Q6mh4ENh_wO)1Z|-y9Tw_p@ z_D2QiR3ag~foPUoSSh2YG*A`2Ug#{SP|&tTA6%7L$(tlQlDq$fG{3MB`^Mco80wkg zsL^p!yp&#AwhjSz55!u^XTg%pO^+l&H#7kI!+~JlSnAg}Sr2Tx$Bw2u2RV8A=TwYV z(>Ii#2pBXS$?b@aBun3i^tlf_4&iKIv4R^;4XWM%bg}JkS|Enfri6A-6^NuHBC>_< zt^#5g4hn|VD`fkI*d62@`nD$T%i`K{>N+H*~H}? zRW2=84w7t~oW-~ISid@+@f@RA@no_9bu{hA9FC20K}sPV0#>N+$(c>6(F!Xkl0bz~ z=DB0_Ma?VWYLSor7pjgq=PhB2p8V2eBKEO5`J)(KOO8L^1|l zpNI7iS{pD*L&}Y(?0pcGsiIgu!u}mIT&WOtEu!qDm`S)UWI})jc8hPN-;S>RLiE!e zTed`Ca$u$5O^pN*iAmyU<#zk``4@>ezjAo!d2?HF$Jh5qWNJ>q z!*Pa0EF=yXp7Y*n3>K>j_G)<`hROSZR@dX zUg3>;T3DKPOGn{ft?FWtOrYQP@Zz{UJ9`G%&juTw{NAj9IyDtX__;v!8V`9ryCEns zM;zurPB1KcF0jA(nhTD)AT zI=73Us?`xPDuq`XF}DW6+(Mq0$dQqN zg)kBm02L7a$hv{~y|OSfN*HprNZG%(&MtrPV%#mx4v-5`l^mp{HYPcE5Syn3j*~_) ztvFBZ4KyxBI6R?`b)}|k(?R)i6NiB0sFF+4>HkG)Cmfsepa1`l*a#HdZ@@t~4Kuif zb^G?7k0vYM2;2eii#dal2`5Bya(Gu#{+t|{PE^$(AGlo#HPISUs#ewjeLUkLp`>QW1Lx^?RSI+O930l>)uL-;^s%>TmW z+jAia0?HyruY5cC4~B`;sN_PsO>ySqoeiY~{L*lEMYOe-w z0&Z0Z;~-kpDhN0+#-N0wTmSoWA7SGFqqcx4_;Xvq%ldy?SCS8b+(Z?DE&*voKp%b% zMnE_zc5e6yhMZEJeE(@_keYf1f^*{0qj zYrDtDgfsyyMW)0V^6BH8Cm6v|;@mF`Kv1xV))S%fDhP)pwuu@LLWQo#r1ymK2dK*k zJO2$r>9|F-#w$RbH~Ia`?OuVd1Q;eqMpTl6x5&-F>yU4IU;GyzLw2RZ9%?qEQzJ~d zkjB&e=GC{CYu`ougT5V!Wf3-toTr3pum%|pqgE1V6ltC~ae|BnqBx|xS;Y~=cMt&eFJzAtCowpCBw)2~r`#!Fz?FN9k?SX9P4$XWYy7YE1xa7mi z%7$MUxDWrx!l!&>ntVxTx{?i5E{-8t8IHl6Ih7r8R*syofsx$4?ovAaty_8JZj{S0 z9`l|^58qSU18ckO@qyiv`?-Yuo8f|G*F~I%iNTQ&A_jyT6P71uw>% z7NFV75+Si$p9M_HFK7c|AxGaSsQ*ZH9z8E|G8oZpnSI-ET%&ztWSH!48v`PB8-(`c zU@__xIKfM(z^qT5rE4>(WdE;q!G0OT$S4i9iCQrT2$_5L(^=ScJ~D3C&f)ZA-usN7 zzw6s|>FGR;Lw$v8m$$6s=_}X(OjgI^d|*|geZ%2q`zWMMN>sIEKovN_rePbMw&|}c zwVxeE&lDT@uH8sAD5DPNC}V{{db)at%WtpkL}c;~Bs&jOL5(D!6M?I^g_D9=6$CuK zvW>@UQfw0@MhbA=11Sh9UcGs;Si+B%{?veC5J+@_M5_~xM<_AJk+d_d;f9Z%KDD=J zpbH;r1j`(C>S5%Q?L%K?efCea&phFeynWlF`~ywZ{YlDluGJZLuk;N1H5As6gKO|8 zd^4BU&~2OkxV~~`R7F2h$Spr_dTz#Y{_uE!uASnh$$>`xkplDO`rNF%#RUhapC~JC zOUT>_^#Iy5#`vg`DLGi}QZG8fj49dGtri@BnY34(JB#pP&T8gg%+E!jzB2!V=OgF;~c<&Y@#A39-{n zOmmr){)rv$;uvZ%Em&$a|32`_$uXOgK3LIl$)S&?9HsQ)O~(9;?Cf|BCzj@R>HpE* zmxoi?_U$f>PXkJZsF1Nlq$ETd&Fax0b0Qg%nT#owdNM^qNCQckWu7ZaNQsttNT#qP zS~BnR;(fmN{oe0A_83_e4&E4skkP5$3YyN zFQn21>qNXB%INnt>|7EQS}>JdxV+F*rGMmYN=*yGNYm?VJnI7uDB z8XhlaClWb+sEomh1ql%E0U1B%M4#;{&}0ijpDe$2anMpoqMl3)cPyv74?C&JY6=~7 zp2uB8ESB%9Fo#)~qz}D7jaU(uq!0z76J0m#Vu?ry=%y{hKW5F&nn zmNJQcC+SQ=Nzb{3kiQ5Kb$<1=kY(1$2k%g^k#i`Ql5y5CaS#GB5ii|f>KyU;HrxY$6v+0P&FnU0mLeH9QDsD=4YM7&7^ zx*yzns)AHI%l_ND&ZmPEQz>KP+B_#Dezna94jP2K&a$SSx8m7*TDPdhs{=tm|1Dw3 zBE|OkGHg+g+w3&`HyTaM`$VDrl9q;prhgp{bItno;jw`30BgGE*R#cnM=YC8G!^fg zXy%f}28XgzG502G@ehLT;%@r-GS0wSJtnJ8P8N+UP#ZjXeG?sd{zsam8z@yuNYH!vk+byJMpxMe%t- zVcefj2FrdFdDdoH(I54=j^+u?qMp`n7<+RDn=^>>BY^74JNNBNxS#{J`9N82KJ{$O zu8KlZVy#^T)Qk5C9Xm1l5~Y9~X%y2H(LTbxykBXR)(-j)jl7E>Lo&4>A^nx^;4O`) zEvVT(XuvISUcq#Opy$^z8$z~yG)QBgdUai;8Afd-`%5F6rd?eQ_{=+Vl#RSk_;Wgp zHtVOcyN~v0)jy{@vN{%5i>J#wB#PxLAR+t^YHWQ|P~_;1cS<;wnx)Qp^rQbpH&rQDU`UdQ8|O z3nZJGL^LLms9g=9;`R|Zb|N%M)W?Z@&W4O-6zbnis5maE*zon#GFrRQ`;fO@GnrHf z_QVRRLHq&jFoQ6XLA9*^dT}!}1ned~!G-`Mtr!qGW+EL50Ocf@W4oZePjoVaW1*(~ z3=~&#J4A`nC6kl@OXI$5y;q^5g+d zF;Vo`g75au(0*_2 z#vjS=OIt^dS2ApEE}H4dHpmImz(=Q~r+=FyI1OBQuHU%D#xTMDT%={aB>HAgrQ7|E zUr4O@_&d0+FyA+TC%gX_wTq42EYRi0LyZt2SMyObwDg}lVp2zE{w?6k^-h2uY^aAuh(pD zDihC#gyr0+6`x1bC&y$S>*prFTiW@nc&@(w_sBL@9`Q|kYzqq=&L_B!=Rxk9jWcEm zDnhsZ&h6Fhja)i3Nsz=Ydio(@RUpL(pu-nE2h9cANm0ke-AW1GX_n;*kot|G zp&_Bj=w@BmszNF+006|IBrY@+{%_g**kzXNE&nN-$2uo;d*79^7TKgQ7?+wFn=N&5 zu#M{ci7!DV#4qr3b_x!?AzF(8)Xt9m7M%wyew=ADqIm1QJ?~52sWRRvIWuAq_eRIT zF-n^o64=lPN28ep2;V=)2ZBfO#M_%vIQ_E{WfPH|Ig4G={v`<&=JzA!3EX}I?h2jM z5~-zby&tq%^9Vxn(Lp*^Y$&5qSoQ%jO_~;Ps*+yQz{w&+jBbK_IpK(_1S8uOL}E^? z4>ml}Q3h6^_p3SsWd)qHL83b?E#)<0BS~REhE*yZDc~7U5sXR!p~it6GUwCm4Anj5 zHf!4W8NDknZZv{{;hmL3W3iELVUt&5al6zG_pu;n+RH%JOQRb7qkFjn#@94BP9PoSlDF&bQGh-wbZHqTT4e^if$?t?#vh zCSAHGJqqB*6y2Vn5}v;2i}6u|k*~FXvDGFVr*mJZXmf}bj~TGXb{MPLc($YRZHc^@?4fb;!c|5asdblVA<|3O-)SO2Hm}KbBl$S53 zm@cf4)je#45_gEoV?{(;#XPdVkQyarEib$9MXO=0lxq;$LK zg~6AnGex>g9KQ<~6p5l~efV&rxZ1sjW?APdv_8EPuW(}&kk%b7@saDRk}Ep#X=&>aeR|~YNS@d*JrTd_e1L;qe{F4iOdgGw z5p5JS4=B8B-IZ$NEg{p_%+G(w5n8@&b;i*TXj4;@IeFfG{yaOqcyV;I{~c~ufvzt1 zOUmx!s!)x~o}0;*mxjk*Ou!xqa9LlHLJk}VZe!>A-lYGtTTShxb(j{pC5wVx4uzvq9xRyey^w5p9hTwHrk3>ES7h(i>=5ZA3Ja({Pbip&H3 zD{VPxR`#i{B&L$|(7?{RK<{ild>e(Rd?9*EF1?~ZIKx`6{H;yxp{DGkx&;kI14+xldTcnaepIvNnTUVQ(QlMnHtaU6gOyUb1=+K9xLO}QdBH2 z4;!q^&y6fD_WXj7NOEOH&SHu!hw^%}%N>ffYPPoJoi zU3$g|a)=bp+(h-!g~5d>WBg~Z^YA^@dv{U1cFoD%c3z^`QRU}ITFm0h^hD3ASyMEJ zR@Waizqt;ditG*i(R+a{V4x)L$BNnJ|80NtC0bpC#%THIa6IIlnxL&1xB^wun8p^i zqS>Y*CXZu5o*KX7WA3fw$O;r$FL%oFvyj7E2L>u9vwXyZ?4e|2>& zrs(^qd_{oP?RrUB&bi@yI-Y;Bo5OvI)(SzMo?(@^i)G!&)IioEx5;Q?nQ~ zoBH+WU>xB3y$?bFq%(9E>sDFKK(SCFw=J6TC;!wErxljedq933I2ecW|K;hsqNjsK zmgepFFyb{?p-@+R`1Zi-83u|=z-A;}9m!q&7W;(T49e2-yJBPolk_cyOJnxRbagde zpr*LHHQjdTpGuU`kiGQL(5mr=l(5M*h8Bmo>e{unFD>YuJ-0cUSDkRNuNS5No$yCU zSawXiQa8pb2jqX%6)0HngefKMV4V$UgtzPC?;NWh8oj?hbPHZ=~#C zNGzMbaHm*lTXZP8=clB+baN_y%Om$iK_&H8ek5Rf)_9NK&(ew zTkW$y={n6E+wac)J}B#@ubcixp{jM0kmk#|je8rUMlS5`pXh~FQ>d&{UK_2e>we?d z+rVt*{;)x-ViXWr+0N}d_VI~Q`P*z&Ch5;k1TymDJac$Oq zpIBgTsF-PEt#AwKr6X3Q?R)fQt}NEdzDOTP(o-8Qgpp3V`?@+&x4Z2(>0&Zb-u-NA zw{H@(n>=MZTJfsH+Okdmxo&-1NDQxJZQBc*jND>Hqg39mRb%eMp*__ZHTN&Kd)DHN zR0N~7$+>~5DzjkpYv)}=la$4nIrLeEDp#9T3WXvv zQe_iqXa=FA-O7y?byqHXm#cmGTQ01o$;`Vu^ji}m+F8Zyb2Gvk;Xzo=AfNbnR40F( zqqD5%m;b3;{x&R@!{{XsE7vaN`%;x&4vWE1*sxSXajip^4IjO}U)h!;=ld61T*mph<=o0{kuOfI z``yhkBmoohW78L`$%(EhYu`|&$14As+nm7_fs^z z*)~4BjC8pz&So^;?yLxjLzifSoXC5x?H=)(bUpyn28XI6p;n)_JM4{qCVN4?!;f*7 z2fihnzEiV}+Ag=xX1F@IPfd&_>#S4Qb6U5!&8yJXc*=5R@XSV!FR@B)4-x86{sE>4 z^~?1t)Qpa7?_o;TUDftpM*Y4i&)4Wr3X|^?gkNP@(=V-1qfJbzzR*7JF@01aSHa?J zkq6zCf~E5UPiY_tDkN2b1!pUt@rQV<8heBQoCeC=8Jc^2NNd3vMNJTfbq;SN*$(s_f096h8D?~VsTYgZ@ z-*YCjE+?%}F~HBUHQ0gnp%hRVt(~sv?(qQyP$(CBX$~E&n|QvsOr(zv+Z3LQ z`7~Y5Cza6~Wb3V%&cl0YSl)fGyVAfVh);05ZQIOD_S9E`M3P)Se}}wmsFA7NQ)cAS zw}I-)R8wltJdz{Mm@GZ>$;7BvA`I`+~!qOuD7lW}4q&3PHQmR_j$e*$gQAVLT??VYcTc_PqPXo+5Sh}rEa6_Prnoju}?!VA#Q zq$%1BITV04$s@+29|6j7ORGi4GOXPl%W_pYi|a2hoz5nK>BR6ikF{d(_F}<{Uqu z5p;~dWM<39jBxB{wwhfHBt6&heMlTa+TsP=^<+6I5z3}3&&D#vu`4I`$`$pU%yYi+x+5f_5mgx`64#^`4bETQ2SOBzwP=t z4}mLly<|(zU_;Zm>=HC0Y&&1kG`j*&kIQiq?xX#O-~R9CUvD$cfAPApD7{s_D&%qg z``BO?lvjBm*r$v_nvUe(H}LP5_?LS{?exxsgbHbu5q$-cSIz&0wRX#^UmLLt$ zgeM~{(L|TEmWI$AKs6fpJleL}dA)b;Q2~nSBEu~STosdU2B1z*2h^JZ%N)8>m~*5? z$E<@v$BNBq_%gv;gsiywR%jIZ&;)X)1>HnKcLcb|>eaOs+i%Yi-5vnH$X5eDpQTy}q$0R*zm?Cxb_zB$%J-nxYI^~Ocp=1|QAZ~Q2b7@#v z4A6U+ACZK*ARZhvR|SK!@q)9k=4770003kmAifY{J33BatVp)n=lrQVUX$!M4lpsP zEP~$^Mn1x}?I{2Q{a~c`2OrdJTMR_QpD0R_B_&NOLg<24S*@U#u0v2Px}pQ0lz2kH zBEY6QkzuA`fH5cMMg%L?ldj==vy6VAmx{X@EIezX<=sz#H>C~rANaC3kd6j`R&d$4 zZ@;dg4{#0%Kwpsu#D&J7PV#hMs^f_C)bw26lR!y z3~qcSJQwNL0D|%(psQUSLYh%$y)Yq*lYLtn1QX2R044$X0P1Bs3eNL`wxDj_7yO&3 zffL3nVRjIdCoQ*^n@oBmg8eGf8>(aW85PgaBcU`Pxl2=bmKmU*XXvGv1OB0liG_YU zAlUo>SUr6fqhNTWuVY2i@J+*eoOQdDl$2iJT_SAGt`+P_db1B2yhNZ0!ghT`XB<8% z)3kyG@H;>&?Sjm{M!3Z0)>&=~SLQKVs~!;-H>-J& zgzi!wP?-d8Kff0UgOLcZqNr~HgySSyD8C>vR`I&)L-AzWdzTOt6OSr~NjKrCNm@%Xftn#|=G+EhVq!C#F00b~Nq7K7`$Y8?nh+VWHwu6uGj`Yp} z_O=xe5+XkCo*3CkgkqsvO;o+Gx=lr%`D9QTx?B08%S~C9r9Q+lyie51eoswBLe4sz zX!}7da$ZW$A|HI@^(tmSNN>Mg3eV|k`;+6y2pO9m^cIpsH3CxEZ?6x@XDncps4 z#SqmXQgNE&u9)bh0X1<0mVIgH2;)luPwHpcmn1{f-=2b6(u0>I3FfC62nd{9HQ{>h zJQtDLDNACWaG=`P_%Uhw!_!NkBr6$QMfl(>`%1|~SWtX#`TJ`)aA$`czj~KmL;GtS z0Id5ze*AFsznoD_W*_>a10R9b^Al@qqa(pOczJ5}66N8rc9B57$!Uy8@)hkUhiNQh z1sicAYP%Rjgn_%%-z!*(mxF<0ArJw(*P2$1GwhF1r({|Sb>l@km#9Hr9UMqF9!aU0 zd=>taO#eR4lpS*PDuy0JK!$hhURR3>6Zm2~fM9aWD?jw*5VNc(I$lOp*teHwN*BS- zsN=PN!2uOpyAN>R;HZF`7@Ps3trE!aYU7tGZu$Aa_o-1I4fc=R*M_+!V_``Air5Vp z;XXVJnLDBOwY?7nEi%e)_*t*0m<*BN#f?!h@++RGJOQJWJ9H_Vs~8)CDCI)(kI%%9 z$kP($E%uWG&LM#?fGtmCvvA$8?IWLq4|;eMJaFydEQG=?20N(Vc$FW6%kB0K zBS(g^y4qByVLigp!P{d9i9?%F@cUcs249v-mxq(lY%l~HKwp4atwL+-GjV5^z|5^O z36wX3z?c7yo&dLxpGQ6Sprh3_JZvKn0hI%Bjl$l3w#yiWW`>ty2Kk*H2oFnLU&$s3 zI*zT*^pfl*p9gEFu8@&se24!E&u(+urXka!qFEjb>#9@$eaZW9(K{__V;9aMGzW2B zU4q|w0bWyuVH=&n8-WAI*8#r~4%=6Ip{<~C?0bDB=B!0xKgzxC+F+uNQ>%eh+El#F zU%sz@gnK=RVY%=mAr~SsDrx7%mK?|GPP3F!uuX_Cu`58;RRGde0Y_L3--FrHAhbTc z|9YqgvzK{1eq5;l6h9Y6OROw6WFyNFmERK+5dikx$1w~Cijgc8MvFyXbmg_mDSTa_ zxU4CAc&;gb{Nc0j(yKdaMDkbKuR48gQ>)ah95>BypG7|?T#9SV7-fr36#vrdogEDf;26^Rd(AGXQ!;9a>dto`3+{C0N~)b z04rN_pkeQ@<}x1!6%~~=2RZ=R&NxF$OzHGOYIZFvYxhP4kM(HwZV?eVpryqkA|kS8 z81(+vvanpmd%GJNY7DLb`}xSoIY<AVvJMr95DP=H}nF3q%1{~v0EiHa{i!8X0a(?YEk|$pycnS&$@d3%a z=^NKky=32tii&V~_uX7vTtT6s*JEP^AV&JFh|1R_GWTIX9beu#FmN00V}A;jAM=K3 z6Zg`cyfAc+m>%`0J!^+Ch_KZvy^P-Y#TApmZuB-BNb&p51x$k-|iqpCUigX8#*&Bd*{h)7Z45^Ut zy{`7fZyEY*`{kEjE8A|r2K zy}GFCzSJrh+5={0LQmBruHtkvn~MQFc=-HzVKPyUq59YOxNle(hb3^059;gduj7!d zYHVx-;Bq^Tb1Fn8L=6kp>*(k>*9`AK0Arr1d(GFEf8RbHCMKqLH8ngzgMauQ``S_z3tF+n!OGar|pf&kZOa9y)U5EM&x=J%9cR$mV5n_a1C|f914z z9A-2w2JXYw&aM>y&Kp)abYJ>o^LdA#a0KF2Q8MemID8AwVoY_Tee3G-q0wjxPsU;Q z+GiB%o3&saDc;xybT@@9Tg(-Y)NL0NyKb1`o1dR=*Sv&+A#@T0!Qu`rHk*EojO;&t z{CLXv9HMA*YwJy^#S9ckTpdiIK1oPeQC(e42=0Fx8QD-h6Zo8cP@(qgf1EI`OGjE(a(Zn_A*Sd+^ujzgbmpT5h0K4)lvO} zVe_?$?z1V=51M%*bj5vZv&Oxenw!^d+QfMMz!hvxJttP{huJk*?_Z>(yaoBi&xKWD z{AQfk>@vhfcpz-%9SYh82L5?@vT$yzl^Kz<9ZNXzEIFC>YZ^h(!%yae59WZN9c$dp zPy)=;AJLlUor)5ic=`AiSzBB0fr`gSZf&hr3zW{NYtL9&_Az7aa`N+WE?6}*G-iq_ z_h0^w=A;rJD;jSP?1*26TZ7c5?#mBrXE=`930K%TJ0DS1U5V$yc(WC7HW#3wPGve` zWwq`5_wRn=GlPx1&iUB^!6zohh@3UU+;;nMSC}g#8E;^- z?NHrnDR5oq@9$5!7-JZBYzu|r2T$hD6DLdy%5E2qy4QXA^0=U2<34Ec%y1N}cxnlvHy28Ye(;(lZ{IF=E96co zq;tL06$3Op@%?V$_WF7@=%c)Q_s;vt6Mk4V=tXOLKr8;;hYwlqvmgujz#s4`C~#v7 zn_s+G4um>3Umy}UooE|PMv#R(oz=VTWe}%}$_i512Rh-H|2%#ACNeH;HT@zFPGp-3 z8#is*asI=`{N90IZb<7^LFng7e7rC?b3T}(hf~KVAn=AZLHXFwaI?JpNY~maEY+*B zvZa=plSPX|8~KXkWrha_-vDCo3&7^f?Cixkp6`bs8akZ6dQvYWx4HL9)quqZDK5tJ z^mMmY`p3HQC)hFtMt*u=Ebr(S)~ojX_H znE53oS>QDe+1c&znjTStuW2mcb}M20`fK>mc-^N@n@OFq^V>H8R0hN_t-{AJJN;w7 z<08cvB{Q=g?{B@|zTJer!ymc&+v_YdFM;e`24R%JVCqgvWh*yPfUx62ATB+@ZPn8l zb!2rZGCpYdi`eQQ@9^Qet#}2zK!(J>YAtU2Btj?-9t*zhe zb3dBz;**j(xoh`s-DoS#qBRG0?b>w{1&TjHRRxa(Y}{HN$RmzNGutRp{a^wP6_B|< zpPdkSbZpBt)E8hnvio`Mo?7N?u(HW88*)2sWF(IC^%}AkoaeKcS$-BW-jAO@w^aN& z@csKC1jWc)*6<(kTcewVA=9Q~dG&YXN3XBp+qrWzdH7SmBg{N!IPGE}4o`UI>dE5^ z??0a^Dd`ec!#<#>tsQn6dSOI$|5`{$h_bpmDg!J?>gQ@XIy#1j+PpaT1DZv*k<_lj zdZRLzH0dSTo8JMp2S-3az~iK(>c^|~>Vjfo#9!(rtp`Ui92FYG8|i9dV`oBE7?(af zb;#!oXdlFRl8orsH)i^@>Z2Q_ABrPOllzTpc>K7Qxj%^5_j{n_vfU<+-uuWVL*_oH=E`Fm4z>!Ia&7uzlo*rQ~?aE$nFg2zV=f{I9<<%A(wlpF2xZ|Ko2h zi(dF8_wL;*-R_SMCcYt`H@^wNoV~rkRzHq*-Mi(pE2 zf_(Sz)Qhzo9KP@^Pl}4T!J1h@d1oq#lIX9=#Qc0YGFPQ*7`H-#w3gOZzzDBgyS8K+ zm6CEQhQ?n3oyizkJM8`XeO`NssLx)EnehA$g^V(YJ`L`u#F{X(%`eYEMVMn~#@$2- z=GCW9X;dqeA`#6TF*e?OZH-X;If?a1h!%5ja9|5`psQjFs}5=66#i~OK|%PFatI+2 zZRAAV%$1>`p-2qEN90kZd4)zzDn@fQSBtJ)`|03&-q!qpX**WW^v)hU*86r!T0h zuZ9_caEs-%{)rQ;sCy+vALiqG?8_av@S2_m1hqG+w^w4!+m)s!T_}H*G&XW#x8lf@ zpg4!H%71tj6BBJ`VO`w`@{-UvJ2aae7;vsC9 zGZO1lF%`thjO>=#K(hNjt$}a6g&eR8q@}}dZf+?9A2)2+kOtBj*%Kv0Livv#mHu`^ z_#rDN=UGPK&t3;5vI|O4-um06HHBql zIFSp${wW?lycD^tcY3;H0l<|x&|t5SMcU(!@PL(rV|2_J?{&l3g`Esl*RMc4{um!u z!tR8#gb|at*s`Zfd61HdsT+QqDaL~ zT3TARGm}mk0nfnrJPW~(a0%l@*t%xf6M}qvZVm6Q$&8z9#@DL6uc|ua?7SNZ?KQky zh6>gLyb=;?FxR6D{%n~*$jzI5#wAwd$lpi>&**hc4Ko^UWssV>mh0 zpQNST8CQVmNZ-LdZwoRiOeHV5x^5O0UXHW0VbC7$jC;%8t2nQR%(~`&;fjO4K5rHb z$XXT321ncu`B`y>tOWTHV7TyqY-`AM|1Y|z_Z)+5)v&gM&T@CsXizzyuj4{94=atTE-oUbf zj*f1#yxd7uI=WT;bacOK|FIUo`N{rqAO0ogbV}Pv&DPAx#n|Bz-8o|?J8N4fYm1A! zu3mC*w6L`~!Y9TjcxacolarmJI6wcD|M&o(t;1#h-Ab#Pagp_QaypK5bX$zc|9(r7 zNwlD&%gK;Gd0gE!Vz}Kc;_eS>`On!$0uo0|hD|yvS&B`K9#?$j*%O?#|J2RlTf>wS z`xH&Nvr<2>R8l_Zu5UgO;Qa8-;i2Wl?v+nn58os`a#R{^n*8~&Ji@VkR;;N{`beL2 zpQF*^Plq?j;`VfOuj3{4*=Rq}?K-}W_8Z>&Pgap1zO9w3`HlQww~7;QkAHH1+v-I7 zp^a|6IPC|UYpZ{KTjbbvdh(lVJO6!|&Hw*p{`ZZ|8Q+8IN%vcFY;8-ZBRQRwW-kU> zZA)H;y|nE#ytQGwgS)j{`-snxHXTc9^_=b8@@(g&7tKQ6ORide-vtc{t~PWB%8U*k zrpsOb_)|hohkI3nvE=>1#`M&D=b@G{E}ylZqNF5b2X`L6a4khWnX@sZl}U_;$tL#hFr;f9ixh13rBA>QSc#j$~vw&%~E4|bN6oRF3M(bt#A zqmg31yfAIyKHOF6*O02kuaRkJP+VM`)okr@=ebs<;X)^MCGr0KKj#-0Q%x%F2yn07N_>BxR zi)O-C7KZq>{AL5=$`?hN8`Jeh`r0=#^0hfqSFNM(e{?xY<-wj)J!4~;!_6;Jua30n zKEHS>d`Qy%`%SZt4`YtsSSv4BBH6LRe7{9`B5d}=tqt*Wg-IHzp`rXnQI=j+VfosRsa6(&M{`IiN3lFg96od!yFrf5N@^opUS-p;^H_l(jy!) z-hH1V2U}jI@@S?V;OFOWwtiJG8r?WK6YaB-9LlT5BPy!>_uqfJ4<;u%EzSK@pS^qU zUSpC6Mii#?5v6}J9*IOL#~(xW4nL!+akD}%XXSC$v4 zm#Z0b@|&nH3KW$N1^GxyNohPe_M5O(HN%|d8NLOZF!ED_3yiU zd2v~Jq_f1x$ZI-8aj;w_tfJw!*UOtYr1fuYWDGTC68D%%GpmkDKYf?QqRfB0Wo?|i zZ~gqwa*l)C+ylkzlS7#$mS3M4r|9L_L4lAx&p8 zBe1>LtDwV&>J`GHp|DRev~b9;Ay8%|N%y5?vaBEd#PoDSN0Eo{)nRR~$xk9Jg>Ek- zB}ctCUunskpIu+VBFIHM%pRJAhnRWPj#EG#Jx zA2RV8SeJ0Fu1;Adom(xT|3{cnC?~UT%sn;<%bDTkl=Csy z-DgMBF$F_oCDg`=A646T>`1uM`YPFddeB0AX3AlpcHg;hA$i3xzJ@n9)`dKLm~==d zD@ik5cd#w*MflMx;k|trxe&6>&#=3lJmt^u#kOSmYPnaPdq=VRbP9HIVp7t9%`8H4 zhK7dWhmEff)+fh$EzWRb1ZTW3gem|0^G77!|5LcITI;JTA!3eZkW;!PaftoX5W%Kle!R6oNzJ7MZ0d7^4F5LZ1;VU{ws^Sn8QB z#Jr4GS6an`=jZ3;DF_c?QC;n7qiuQ9^=ir%*m~|06tpk1o*rtt`}9-*OGAoAL0>n&VWIiSGiP@C8Z(U*PTt#HA!`5qWcUyQMB(m=uf<-A z2D1UTZwq8ysxlL@Y5N*ec<%Ah)5nfo`-_?RQLnhI%VOsymrz4$HPyK((?|sGtk(X@ z{L3C$ihylLp*jcS_U$2Gy2Rwpo)yY4DzS+Wwck_bvD~RJv^cA&mT;DK-NKm`D__p; zrTIx}(&){bt7G-^9Nw31k@2kd>i+)y4&vfY+v53Nw%NQUu~!$~T<88Q<3nA}`RmU= zFGx<+$j)T-OkTiVy&D%7$54F)b8b?8kIm)otWvac1s04+FrUFfMzwvaMye*W&I>bP z?4-L*Mn3J@SO?D?ekgQf4vy+(n;W~jxiQ~m5zdOP8vF6_(UH?<&)zX+^8J#oC(^BC zlsVd0xIEKpyOVM!Qo^OBik@B4+|}Kk1zR*btgO6z=cUSE0qn>->3TT{x??Rbt?uH6 z&0}2AIXn0bho2c)Tc|)1l_0l?W@bf4r+ZK zpPdzXW!+4<=%KD2EMQ*q#D}4E1OXwOuX<9sZHU?&!pV$KQ8Xm&BJScn-Jpf>iw`qg zXtLKw7%-A4FVV<%vWQiR6u0OwqTmg3gvO$m#6O1%53Sr#j78{TI{NsNC<2vl!{1Y9 z7+u=b?ChQ`EiDCe2buT<{Om4I#bZ{6@;=7z)RB>Rb+URm8XmMTO(EV-&D~u*``G-#LW)|VQVZg$Qna*JQ@P*Gn+FhlXYHr%QH=bM-a4I2u;j?TCFx4m zEyt$aX{6;Pg8_@036t;Xd#uTx^Ai=Fokk@0|GjFRYWpm<*XSu~J#peheT=LhLm(1bXl7>S!qk9clWSw9QD(TX70G%?MvYi})??lW+lNE* z?7B{n_>#0&|vm`z-zRL#Ru+c8v~eO0CF!TGiUx+JAXe$d@|v^YW~RWO8$XOD4-v>!jDO zkC8NKY;3&H<;OAH>N3)@4@gMNu{X|j80!gWDcj*k<)S+;rQ7h?g}*yIC(r%(^jK?U zWySyAJtg7ySoiW25_&E^#+zYwMrK=cY%{Txhvt5E`~UOLdAtHvzp&?=9kXxO_}A>H z8`ei38+uJ^e0nUR9IqG_if3<3(+RRn^ELjeHw+*UIwo;zU6%^}X26=o|mMl^{h8olJ?YgeNEiI*LMCMdQO5`Yn9!lQEr6fneuFnuKd7pIh z_Er`dZ_gBi5+m2HTOO}c5)xlpH4Wb8iaL+9j-eB{%OPV3@S@)R{_gbm0DjA#-^*G` ze2|5P_=PQMPuknt&w74FXlT}}8141`+=^B51%cGkwktGl>pq18hYp>^UdjsNH7wLF zS(*q~yM;Ys(Bmv+uqiXsck|AUBht3EwwI78%KDZ83!M6Ec9*?-*EanV|Kxiw=VNGp zoR2zkS;@`SwQW*ZH{U6FW~?()q{DS@Lx&~65R0&7uxv9Hq6Rhu!Zo+Y%&_~)eBX3m zw2%03kf)*8)!~QuP#no}ShJLm?1wN0>I1d$KZb@<@#x7&h51eclu}2`r{Z6K{Z$zy zRX}!FuAMQ4;Ackpq;31qQq(Y6!40Wl8@BUsVg)0!C?l`F+!A&>AfWF4ZdtM;fx@_V z?K<|-vY{54s#dt8oEjQ$jInYRp>ba&jeVH%(GGf6IN2ccIi9WpV^?IqC(cirqUYh zpQ$(NIQsEcWXRZN9t&Aeu$$yxTGXA!)aqhAN4fPfr|ajb-rXs(q9u@KRAN{yy_oj) z?b|7YT}284ZW2ae;ANAc8UjE=?3qA@cK#qAgnm72TFtSJB9i~FwB@SE`fgHja>_=& zi!5j?av3v}LY|}dNz zc~WG^x&bf#=<13o@|aDSd)#jG;r?zGF^4!Z%>+&2<|#%c-v0b?0s;bvWcO%+V4(KY znhgv!vzm;dtR3ki{&N51^A zabsPwT4H_dnJu?`eFuT|&ehE|d&pP)Y#R_p&J0ByX~;B^adUIC0FDJl&M00SmC#65 zJp~l3isB_3`5t@ubJtJQHR=c-+yDB@-YD}_)?a&1e*i%8T9{Hn@+<9J(nRtI=KK^X zp+yq9RC(Y2{rgi;;-u)lOu=g;VilRAh*FS~s~PVq9a(Iz7BfdUKyH3Og4899c7w6N z?c28pzdn!M!Ka^uosl>oIiuWJQi1KB1}vJ2Eyf~n@nkr=BLQ*)3)4gX{{FREMqcqq zO#}nXdbVL=_z_vLz1mzNUAA#>WY#4rw-y-Lw&!a+c<_LX49X7b^dy&$biAgmu0D_= zK>U}-w#5U7|I!35@bvWT85l?guuB0VY00&ZP0>hASifz5TW=Nv*MkJy_F#cXPS-(3aU=_RX6dMEIq{%79XBP;3t&`2D;8* z+npXUJyNz^yJ&O3#5dm_U|#&jsLO4_egd(o+tjsx-BWlH?%q#)83z&SzHT* z90Rm=P+VO1h}p-Q7uC|KpCcuP@T%crjtP7A?8(Npordm3c?ukzU-psdEaNjQOhRl< zaUNrhYBMx%3T4aSVI+mNg8FJqUHR=4{qtIQ7zq-UJ_5SImt#RWeg?w`&%_j zo3`8s-&sR2y4W#s+^@(s!se`eERspy0I8Q(H(L%7xuG^*5#w+~zE;n+C?094zl_mH z{YPcUb~d(jlwD*y({J-xyJk(Y`Io0HudX}>3JPfq`y->bcE%$&q=!!{hdMVm$MoWI zwbFAR2bW?~#IDni1TWpee)||B$!pNo`qJt$!y)Y&Zz=-S5p15jH*emQc<+%j8kSRT z1~?h7$XMFh*_mtK{hlql?(j#xb2%J0jG5jeQf!ickK$r%VNNeTJ^*R4mCbO>kHcH^ zTzhwp`iIaO!@cxML+nXL&$Ih*ER$#zno9cS_ z>QOuiZYn)Yp-}E1-|a-EqA%aEZJP-LpZ-_ZrJv;^mmR=%iC4o4S+%+;XQsX?5 zS+XJsGr78?ch1v-7qvb*p!qj<1;f?=csZ+VsaD&Ok-A2yVo&G zjW*#(V5d}Wg<(c zZafsZzT-nEZ>zzqydvW)qRY;{>S&SA(K5Efoh6;-?fEZDJ8cQSz-fYIYayLb-!?Eg z&={TVWbXqK=?(_JMMm#)gqX?ATep}gn~*63#cKc9!es1D) zq1Vci8)_-fxp9WYb5WAb_f1*Fojx4iRGNhH8zU5Pu>>)wg?bcE;liXqSz<3LZ!l^X_5MF2^d z7yaf=@JJAsawA^yrT4PC*G%)}6s^oCL(iXO3|juSSDFg|%$`1d3Qmi6r1?dkQTd;& ztZ7Ig@ov3St?ev&4jYfB>%JtU4bp7;^mEj^%F_yoK-yj&?jRI{NfE(cJa+5?EZ(qgXO*C3a^5@M>rFNpPkP%LuI@L^#)=Jln z$0d@mWoO)%mzV2=n45~)hFn#Hz}FxyCtK7d@TkW9QzWSzCFwqz*O%=)r0EehiG;*0 zE#1*2RcO;L4l1R2#U(GgX^#%-&)WL*rs`^CQYCd3d+mFa&Vy*73ZAMSC`#RLec{4j z>P%||?*fA7LhoZ+)+mwYqR+>HOH`6o<5lnS^YMM|m?*Yv-~sTUMwJH-M;H6-bG)8G zV7!)B&Jgl58EjNWNyeqV3TMxrRSO%&OY>MYrcrY{Oc5T{0Cs}+o}nlFBL332k-zZ5 z*nIGk%$%+7a@$~Ib)ue+O=Rw<4T|$LoA&%>y-6eYfqkUHUzqC-6b6kj^7VPC@xzlS z^qjihu3-~%;57KIJme-Z@1iRm0QwS1sFz`pq@kITbGY49n`3qS*ULbNGfj3~Z*RF{ ziA}%XY5UbLXrBl6zA4k`@ccfr3d$+Ws3z*zZ0lx0ucES#EE|#3ehsO&n6aRAAvOnY)L7vMK4y2g5edPds|sNE0i7DTAX)PujzD>_>s((#&zQ~J*pmPW)g=AAEZNzD z%h6t%+j-P$03lNLp9}x0JaJGfqyEE#y$}wh{rJDm4i<;+Kz42bYgG@VOKM(_SXxz~ zybT~Pg``$q|Gml`gzHPBREF-0u}`0lE@;chl&Ej>-6>-8H779}L8=SecIe}pT1Z&Hgk(lR>~tPEZrhMTBQ_b0JXH`x&ueQxjPhQvcnPdr zX8kp%-1qcu@vYGmk(g&a^hK~Q+2QY7c2_Ez8&8;&Ec zppb=1c2sO&IR3RA1O3~owf%X0Q3c&%RHdz3w=%G^ zXFy{TKg7m}pOCtSB3(M38x_i*yKvzfTiAmFgNe)9mVK}Bdgcf%^p^nmREs_5Su3&Y zpMs$#Y+-{qrP_^zFsL6y*f{s_@NtAs4Q%Xb5*YG^CYE!fCl{Zal-C!!O?pW8MTj|0 zjTVdz0%_1QGLQC2dKZALm#>zbydxwAh_Um6;5Q#3Z zSbj`Z1H@P?cl2PDa#!~?HXr}?hDFlVE@d$StJ7_=w*f^Ip^*n-{Mc3T%=J%>uOfVS zvSFIgro9Hmk4xR;YV4KS+1W`Z#MX<$>?b)5)V5?@3LwQfArLscrXB$3?S)8${I7_$ zPHl4T1(gMSS_dZKB`V5v!y;X9)2aY5YLUZ(nqmA#nh5y}OiW48*HVF{i8K-=Wgw0Fr zpR~C0sGuQI`j2hfw$)u?9hLc#qHz!$u>zL<`Orf;Qw>^11I?GCUn;K+Sqg8jw)&~e zuetq@_Ia#(bqt9*02jEm>fOhc0~V#$wevP2@uus%AedbZcV^S=MqSfc0#>5oNVVi% zP*jVH0&WgD2m;_GMmZv_NRH<-$nVeT^ez|*`)b{s75{McQ1pi=sYQd5JNu0!=jQvO z$MUS32kH{lI5*C+OnS;tI>#z9ttN)Nhd8Z)%jq-qu$54Y^*xJgE_u*6c~wsCAB3yn zXo;Ns!z6@3&?zEGj7b2)CJi zv1|kCElmtkeZg39Du?&n?gsWvqdh1Y$+~LDwoscMY1Kl44MseEe}~zGLu5snf!}cH z-WYXpnQz8+r;PPcy%a_^+vxjz={}BHa^B|%m0PYk3vwDPD6}+^%1}bqV`~eUyx&AN zB&cR*Z9X+YkB5XyFu8Fif6wtV;hdbDa+IQdR}n+WI>n-9%LoNmez1LTYG$U9gfXrt z*Aql_0|iZVA;?*YEtE&YfU*r`0gNvZ_B~SnadOoB;=yfHTu7E#f4{lk?e9c*>5O`C z(ZnPqIM}H=bfrg>BzIJ&%~jxc> z`@%2Ih3m{FdJ=gvoj#T4PU@i-c6|hAZU=Z8>kpcj}B2(a4fXH)S z{3C*cE$T5)1-_L2yoVopI-J@n6mg3EhEzq zu%5cuNmWN08)?lMD;`oFo$<_EPz*j0gJ8zZ!=r$uJZqq=OQ;M8ISr`AGQb|SPyem~ z!AxzFResa0qQwWE(t-Z|SuRx}A8)5aQ{G##lqP}dcUd6XbK>?y@28>hk~!PB>B14v zdZ=8mR1CPIuvpk=F!!Jml#OT_%m^)=jEpZW}We;!MYl(o%)Dzcqh^(2y%-AoC+EwblVn9Yx+~m z!I{JjQ-AMHP?wmAWQ7nOdiiS5h@)Z&%SwDk#Sw$r zub7XR$vH&)5OFJPOg9ka@E8`jb@Qe{MF9$?l6<62XWiB`7sPS<- zu#5vYwn*E#rtepZc(^A43WOyBf^NapD=FpnLt%Vdwrl~0I&n}lttK(rI|bZ|&?g=? zh>=eZ%~uVMv`fO*FjRBtDv30UV+w1T5p)m^1GP(dM(_ ziN#pg^}8DGN3<|r{oL~ee<4q^77HOC*q1;=h_;;V2A~{->Ui#D{45BJ?9QW`WD)ja zFmVh!c9;cLNTi7}QusL)We-sVgfZi!Oxy@}_ihf$eFyxqjR5V#2F;8kX&c zySuwdl>#B6-M-K(3u_BC-bhE$V#eY!q5X;7g>dGuR3x^%oko(fz}O(bIg4uD^7Biq z5}h|Cyg7i(FryywdM3Du6x;>zFN^>igq%U(F2dj=wn$A6H>)9QDdzAGT1eR{_Fn!1 z>qGzjI1Xt~ZNTho{v$k(`ubGk6)fKU^H(z8n|ThJI&u|X^_$=t7u@*K*%-{jyLmEa^2lp+VmnV$n+ut~ZeK9^1< z>d?iD7gG)%;LNPq^@#U=PKmF}f~Ue36gwbI@`zkR7-83_o`-6ShS}KD>PFpDlD@4& zh0$XdDf4-c@8AD6?k$hC%Ow3E4fh9gdcX8?1I9>gszq+7Q(v&;a)NWk$Eo`ef*R8G zv;f}}U(S9^FTc7t3y0B>RPE;%xmP$)V}P3yfNe<1kvDfv$iWYk4Zu#1$KFJBlYl*<-Tn5Pheil_Wve zoC#Qs7Yf6|>+oK@f`XkGZvv%e9s;msAXujmuLx*dsCd#SPK11pqk#4&Lds@V(es#Y zHIxrRmp`)-3YR-H%A)mEI$`hq`tBh_`nYoX9QK^+^4ma9uL)hqYa$gwsiu_s)Oq9! zb$AEL%E~Mdj&xsHKLW$ibJ)xd*Cb%km!8eih~)#S{Jp2|oJTD?_3=pcfjxUpKy&zl zf-V7b%FW5ylTrQ%MfmhU-5GGxDu}H0Nh*6s+6%N$NGN^TPJ9;x*Vo6%ZzDD_Xtni_ z5~e1rWvs{!hChXky$+DeYy8bRM8qU47c1)H_D=}0FF|Yw+q7vzQy@SQk% ze|--AJqYs8Q$VIvB6;1pQxEhy^+}}jD?ck{P={q2EixS zqbegv8=ycHNu>^>VP@N0(q440PYKnJg^@%t>UxC1Ac7>me$__N%vZJoZmAY*S`U;$ z$oBF5qmLS!zI&%b1q0pG5Tq=fj11D!Bu4ECQiftQ^sFt!^lG`m3$wdK&Lf640ELBx zp-f@(nq3~VrFTTv0Sebk2k$?8n6L{4-Nn?I<;S`%V?8@2L8OQmMyu+5Pk|Bu@mLci zXb8*+kz$UgQNPrI6dvCSiN8G+K_N0e-k%;F>C32+ z-7n!b5r;*rCZEMsUr~CL@ViDwhavQm(!ZR;Ck+Zws&cgS#Kc6c$5>G!3daF>&vM54 zAX;FTw1)lZu`rMb&X9M_>ec;ZPKoLR$QK{}%%(Z3OgceIP_R~de+kO6nXdJ!Enzy? z%}?2VmhABiXCR3|$}3)3S+pgqZDMj#(RpjF9?`v^*lpOdMT@A8S(g+Wl2l~jM-sRD zc8z3mY(r@7D&(;M2h$;#CSd{z`7=_L243U9IYkBVn`aF2oxVI2Fac*R4O+{HT*0B| zA3{Gt@=HdB3O{oBp7cWhUZSOvgaztqig?rxAFhevI*9-&jEx)#2TemftLH;s5j9hKmmiwHjK1kL?&7~$K zp444xMRaVAeTw;EfxFThXaL%t0(QRQOTY8D8VO+B*xe}VYO&XZoCnoI_zYT%mpnkq zQs$|Wmns?VvWlr9sHuV5KCt0>5I08k_Y z4#XrF2Qj#cDBSHIUeFNaT77iICr^CIPt+>kHMgWb^!d-}Y2nH+e%rH;1j~%`cuEi7 zLF!@w^cF!Ga|eYv6N;@Y{xXMDpRA89*nxKlHIMp12r}#4-`8wl!te`_x%KN?0l#;5 z&nbVVsHiCV>bxthM`MvZec?d-i-pB?&3w)7Bh@3JvKhKQiDNIngdf55`hadg|P(@;q zb$oMjavs3vNeo2q0cP$3kNJ6_#JU4M2(Bb#oCNOB%{z}WDN9WUlUH%|@Yop>6Z0Nb z)zGtSy@{c5REj&HjtM#RezYLBLg+cF6eaoSyBI_@D@mk~)36vgTl`yoQ?Mc3GvZ9SYrb4XKa?X%$iW~gOY z90~c#x`vQ{82Jq&5}1ffY7?{IyClfP50Ibt!a;_JDs~5vVJ9Gn)7Kot3gS+)M{bS< zXHI(}h>Bz`PG7wEw6xelYMs0wD#Fk`8G2zl1r3l*P2~h>rK5>L=ImJdX!b{4OGSMz z$dRW3OdsIewEF;+-337HMnzd@#g;>|E)lmRY?P}6tt@0o zsLvz_o1}tC(DWY_Kv^PEj+1Q{%x$uXRrGM#>@^ghEaYZn&tc2Pf@67)-Q4v|c>*k3 zf^xJCIL$BQxOG3RS;E{ zJQ~P~N8by=jN3W|t#YCzY{MGIF1mP74GUerz<0z*8^r?$KG1$9o(}D(kB2vPV}~5^ zXW+g_Mi8Sa(wJfJcu;`leGw!lW4_X_x%MV|5u98+`8x-dqJ?wV`-(z!j%+H0gYMy% zS6B3CAtEd+jArm7Z-odnym*IItd?-xM9IW%YyVl)3Mdnc^4;a`&QXZL1BdS(q!WDu zW{24AEZ}&euZmpOk}@w3Vh16UiFj_Rol0*b?!A+2YRr^GP`{F6anmb`$3f|)ZwlBpx*E>h9}V3#=3G|U@2ondkTTr2GJ z-~BorEbWt_rp#0y>PiOvn){}oxz(|PD3jD(@6oq$K_y37(`-9poEml%G386awMrkf zSu^SUaZ*8-p0s{l)F$1?Y$u#*;OPa!0-8Oll3_kMYW-G$ZNCWVdJ0d^jah@KKZBmMkz5O4@{NJEZ~&G>cJ}P{{8d|> z0_I`<4h9U-@*{mEug^umKMWM^=*;xz`urd8{-2`E@87g(lasv=xu5<%xErv^a{B(T zlfu{KM;qChnB-`UCf8yzZEpTe+HwBhZ%e7G=>7C~`rS5V^0G%CpCi(0>DAK1LZ%=7 z{815f28jxghQ>C{+`e;1{qfN&gsz23L+nB5fN=iWhXPv_u#`Yjq!_EVTzh6ObK2W+ z?K-Z^1>Qsren%k$JN2Bvogzw*0*XP z3aVT`5bMOuOep7Oro+d`=m59{Cwv8=wVi<>4y68?wQCcoZ^$B>Q{E+FC-X?3?b>!2 zm`_~(96>0HqC=2NG*Z;Rz^BoW?`(CYv)JIxn>RB(`V<%;iDwMskOag<%&j1qt|))r zM=%ueqJqzjBV1dO#L=UgXib?}1p1`Jft-PQbopJ^@8qsqc5&FraP}+NK{S_FC2D%5 z!PR082N8E=Q@S3Pj}O(y^&}a| zl_6Bl`fR`3$X;K$wo}yZ3B3DPN;u$q!PKTARXUlF^?&G4o(pf=y7l?ndOk<$QJj$X zv0BOvz3~GYS~`lW2`A4UhAj$1=wdhMf^;NuVtr@_MA()Mo=HJLg z=c@POO+J1fU5x~y399a;ap_tQ9XRwrkH!Fd$y11X3j5p&Fl5H-s7+fEh*8X5S#dR% zG2458;sykEvP)@irL})i**Nm*dUb3~V*QaPC$1*e({4y*;QjwdyWOs!)Kzxz zD61jg%M!^9ZapIC!t6& zSpK3ZilO)XC743X8fbfx?$B!ogEsKU^6=d%psB4EP}1o~<#vNYR~2IQ#wH_rH*re0 zJTtyQ1XPsp9Syr_qlj1PGTFY1)O85SxJ>jWP ztK(y}@d4OJVC$1e)PybAr7Z`+ZNtWmj)r#RV{}zn`q!gAg3=_4D~tvDY@|b!j;`@s z&Awz{QGy*(OKHD)-LifG(7FLN4e{BMCJMsed(-}Cx1;LZR+m(ifbxQuujVvc&p4DG z2CTrn`%`LYAK76tu-Xe$LIyo;57No+P_98YO!pw|f$8p>{uJn^WM8V8^F;;8Rj!3-iytS}1Xu%l(255k{s4j?g#o(km)J7}qO>MX+NU@mypwza zo>3d+^2`Ltb92y#e0MgHD_sx{RjPq*kwyrR16v?#WA{aJ&>rS>)}hEJs5|w}1hSt~ zDTZYQ6Z&q=U$h5LtE^q`jz3PfSY=wEx&=T0Z&D&;+aXDqs?5R(|I||2glYd1AErYZ z4gn`X(3WD|7>u6oaO#-D7uXM?OMM)$jZ4co4EZNcN2a7}X#z#q+X z{>e4O?tr~-BeR*VYMk*NuN@veo-+N`S=enke8zhuHh1UIqH&w-Uz2mq$z(wO`@dY_ z&0}EUbTI!-#Qy|REAOyiTZ(IgtMlAp1+E#>+V3%sG@f3KNXaSQU2;s9)uAX`D=$3es1K@ z@}nWW0W78kxE|Fpnew&R*aUQ$Lx>BPbj#ArG^AZ6ddr8W`ej?U2OpFV@H18EP`y1U zv`<$cOiHNW%#;6koeF{e3M`B5+qUhMC%!0TCp~#pNVP?m*fO?zd`h4S-_|%0k!_Nk z#~S6AsGhj&9I`pk*`C(_eBT>y(*(u^RgESTYQn^k^hKEhD@BM=zI*f(^;h0$9UkJ^ z^72O;+*k5TU)}eV=kuopjN0h*T$s#F~p zUTvgxx6dY5oCam&7_sj&4GI##+FL;(rFlI@;0ovaXG2a)(c3dcVFs$!cg~GTMQTVf z8mY>4py$=JuAG0(16s}CdO_R$5j=nk7NF+Cp+upaB?2*+j{?$BVhw(tITkdAY5}Tfe^XzLxtlk~--zBsu|`xYK!HW#Z`}@;^va zH9%^8wx>Vb+?mC*%M_bpr8pl?)#=(wZT&0JM8>_MwA9Yom5kth8_e99)2CCA2q+(a z_QIb<`S_#B(Cx=|Eni>i?8y=MnLWrUiRpW9X#ZcXz=({+ zr4(`g@88$J^st`zRS^&x@rJw3OSwY>7eB%{CO(&MGyH@j)OUn9%- z+Pb^(Y7v0I<^Uj+I2dHiK}SI>g2GW$pJQvpc*alS=lGu$Vzdw6i`T31m>#@T!Un?( zhzkqgHt={4+1+~nVm$3@+SaHD!pbW|mUSN;mu$%8KTW-TPbjbC`C zFJX$M&8MHcOP<*Ifp5j`<$-l_2Z<~Hx&1>_NbyXomCUB|4VP`q-^~PA)(?djd-K1( z^?(HNV++W8?2L?w(Ani{VWn~EUOQw?)ryc9O}0$`SfcglnJ8vG&&LQKrrUD3%Go|s zGd-NNVqruo4{w@AODKw<^bP~*?wDMr#|}H> z)u+U^Y)@y+?3K+>$Qqs|CqmF|VKQrWpuBr`ax=T6Ht;v;wD#Y@n}X(y{zm#uunobm zSQ*3>NU#rP6V#HZq%yio!S2U`b6T@*T`IV%eDveK*ivp08d#Z@uqtyf{nG>YmG*-? zs@#S*8qe^Yd2`ZB_?}K1nHot^xthoxVj?7^7VI8_-`?DC2NMRhMwC$NGhU?G1GIz@ zF5G~AVunBeBrOVl0u!Xm0!DcFUEK*pA$}7@{-Pd#SqJ~QokzE)T=-0{m7)}(v>Yt| zs4#A!{i4$#7eC_}TGAXSuQh0k9DwRebNJyNiWH!}I7D`RVEtgzS5eC7oUtvQPk?O^ z!5X>+b_gF4zAa(gCaDA@Nk6m(btG<1c@EEU()1OFc1UzF)(fu-`&&|wS={BVn%G9J z!FuIKtnVW{Tx*KRX%nw++*sW%Ph>gJyhC7g>SVV_#ewFwf+vZz7IQx8?y8Ouiv@?c zXad>|x7bcfMj)d`6foPt5s?B>Qc@pAimc4(E;}0@_11p;en(K~u zo6uyg*GP8j$wJRlb>eFe?h=(3P+@mp8Bv^&Q>Y2q$j+m`do(IgxtG42g!bHR#d`}l z04PwXX+kDY3+CeFq?yHzn4QgU5Weu5?$ntxsc`U2Oie*xB{D!fYIZpasQkV1#O6`j z4H^9ZT6tdV?7K>OPGHJX4bFtE0J1Z6A$KL+Z|r5!wW+T?XT=@&AB%};>KTjPwQOvY zTCX(II-A9Fcaf|U&A+y719r6AiARIxSc8H-9fNle6gwrZ)NjiWEP8dI`0Ip^8-dY8 z>68fKC6)NoQ1p=!ucqcaa1&qqU|u`;g@ytb+r?$XgCEt=MgUGO0r3p`mz)HyNQB;3 zq6WCTOI?hNpTdo=Wy_5kY@9U)nnlwXX*?m)J$Np~lI5Al5DpTd%nhQqP@yTGuwaOW z#0gD_gIi<}sujx-vpWQHV*MymK1WHV!ax0$gK*^J%m+%ago_RQQlGorw!vG-%xvMa zG=db<)7>5Pw5_U2(a+LX7VRE&Ck)Gm!Yk3^S5*5dQ2L!(6YAtfma!&K+?CimZzJD=A+ysZEMkT?MlCTQ{+Y;rmv8#RFV5L{wI$8HA1QS;j*}K?9mSFb>SPqic zemxI9N&Bk6fZ(M{bFimG8pO^waCLIWgA%+pNyp($i?Xd!{ZBb6GJ1oJm$JQ(raji=COt6 z*4K%Zy=d-x0Nf+d$gw@N{s{Ns=jHkg%;bx&{ei?``7$D>Hel1m!vBMZ==uJuQr9%CSiOvL&;9#t zuHjke^2snV_aoV_ zimfd|4A3odclMtk$0DF;4i~k5I(dV(p0vaFZJPooQqj~T(r^WXLI@9nU%Ae#p*^+W zW3C@5HVh4!n6iU}ajs@E?#CCPuk);JXXF9wDbf z@IEY~jmS41!Uv`kBIG)L0qZgOe9Q@S&D=-om_lv9gWijVF0vx&4!x_bN<&+T0pz+m zIJ?!;b%o_?kyEA{^=wJwJi)jDdkKd`r+o`3A5qi_-ZL~B#uN=dIK!?2Rd0IuMK$Hu zfJD4U9`*9{R7Z~y!w&)(pgc~l`t@$1_&#KH4PyI%r?lu0YIk&})`DppeEWgif=<(9 zfNzvE6vAZ?%x@%(<)lQ(Mq?G+B;4rIhU+EUs}sU-9?x0w`D<&y2yB4`1)!O&5?%Xa zPtD2i=`_s{p`p9|C*Q#FFRs7fnbSsh#6Gy!Mst@ThAQ2m6=N4JVa(8$JzyszEp14i zfsns|uT86eEkjP^8XU@_oAQJ70zrFAbqA^JJ}LgUIf02u3|Le65a^}5i!SRtfy63 z7v%APu;>l`2WRqTHR87Tr^U5v*P316q`h~WY2IonQPHo)ZS(CAht2G0$^+fQLt#&B zTe2&>7Xh7`uKl*EzSzqHb^bX@w%6iSVqriow%7QFc8!gI$A)ORTO5HxI8-#-Q3nuf z#qWXBL}+I=L`xe0mO7sL_ot`C`=QNi5yB!^D@9!%fXV4Nt=ytZ&2qXP1%<>5jep#g zmywU~&IP;nko#QI(8&l`Oy7U@;S=O%D_BrcU_jy|PBY?eK!exdzr)eRfgMvo76Swu z5j_d~@hOm_I02vrlz0OElqXKxn%8b==dUpkEXczFFD(tmwAVTGxt90;5w~|g@`iXq|Ai9z!Ru_a*-{pa$QTI&VhIt-cV7?r;B5LWHcMpAo9h4x};A23f4h8)es zly@*QbK{K|PyTue9y|p(tpKQLWcJF@$Dq_m4+=ER>R%-tT^b2I3DchjD}Kbk<)RSphrU3lWLLTyo?Sq;YwGmRg{pOk_$c z{3(N;CQ@D49cpuY3_Ly*|9G5~^Z{fng;2H#FXTw61ql2Gth>qbdChaE{kk{E!6F=M zL6jo2nG@K`Cgm>FKC}qi+cZufI*4Qq`;i)IMgnga|9jakYA?FSQ7@?=ed9z9(n1LY zBiwF6i*%x42-KwMbLM$p4tYlc<1?JeTVVe0L!SXRm2Tv-tT9%+n2}}n$+Gh@1U`3o zw}j#DA>{dQzx{?YLu$ZR4==(|etI1}YYpk1#F05yItq2bj;rRsr$se8>0jC1E(FS=xRjbt`m!;h#9WwcBeF>rybD>0gqoID z#hEvN8u3t8Qh_AM!^2-352Jw!TuBs8(msJ5OKUf;sj2aJ|JQ#ls>hAk)ZCf=TToH9 z&1p+ufy=IE&DMgOWFd-zGoq$skj(^Lj?ni6XA#m192jve?UOGLnrf*OY|KPq5V`5R6QuxFFT3IiTvt}7Q!6YZAx}S92p&Sl;_SD1R~z&W_O7=U&TKN?`tY`>U?~$f-!cqK#BTy_5bU!A zG3il)QMme*qjkNicm%th_+?9HK=;FR`L4Kn1im}s5Qslulw%W#ja=7Vvl8MLsa~BT z6QTYWR^m9vf2qDe**fVOAQq7d;dCH7uRJ;33mzoKbCKd2*osxp+%fw};741>xN@GL z^|g~XgYt!@dgq>oke4d^`U;{4T8YnK+O$Bd?IYc{b$~O3Lf$-OT5T21?91a{r zm4Mv9BTw26IR{{_Ck%vFG%D1Vf z>8L46PgjJ`aTifdvauq^*_xNN-FM6a^;N-^O3R5}@V26PG9v48HKL|_yp;jgehm^L zTc!8nh{)opYuL#L;bkIPjhDy&f*`rip!?t3ORolI#B^G>N%7qGjl`8s1S7bdNX}2N z!e*RSMM4yIpE4(2P$D;CID}!DlJC9g1l@!5IFb|Q$jNL_tu_BuLseMY+txh!Tij(d z4G|hZ&oHxiAqCQQWCAmdV4)kWLX;({L0LdAQ$O|yX@d&m#9>N<*q{TtO{iT{Q2h-d zplG4-OX%XeKggjWh!)Ssf(mO_=t-{q#xfeZ^YwS$ zejqJP1b!#9vXd6%=B!6YNZqYRPIn22{iUB1!T|@nwIVYFdPLdEd`HDYlB1WgEoycc&t;wTmCzN z{uMg>-5XsKev2@v)S-5L3HF@Wlu7IGPTRt+r~Tw=bfl&#!jvyeRPm3@9=TK*hjiaO z77b@3Iw~^$_@ynX;A+KJM|3eU=dhw1QD2fHIWPtah&brIy-yBlsg4>MJWNZPJ1*j? zfcLa&8_9wgfXRMG>rM)A1`nFTQPJLquQ0WgmP@t&*;kOujiu7{TgM`892{`v1`*g*_41|YI) z$eD()D44^@PtLT#=?Z&sP63&D*tV}aKO{E*G=ZrrSNxC5h~bZM07N1xJ|3hbAkqfp zWwXu|Xlle?OzQiMu^*XWaUcf}A-dyK38Bj=D zIg$%EIH-x)S#nqkQez5%tvLJTUu-*Fz;n~<((vLEAcdABaxfwBwV=+ie0y^}G1H?W zni@!q&KLhx;M~&A3RuMHnL;>Q4~v4g>=mlBRQSNiXJNA$i(~<}-mkgB&MRgqB_ceXMT$T5O%U%a0joM~>}&z3_{6Tz()PmQ-J+ls z)=MLNZ5k!sUS@W*`4(}$0&-EqjS(5L4gA&*y6 z6GJ*%K=h5}d~`Jnvzex_|$E zfsHa6XH+@uTuOcA*U>**?-sPb_QL}lKL+gG%S2Nl<@hm;CMG7+l?OAEkRFn4WZ)M> zUESN0^esanEVME0f%L+Mvz475)rZK{uT7DV3Pe>IEmMM73>%yQX&&@MmXd87Eo2$o z)4g{y^B$@BxcV1~>M6lu#t}I=se8$=jz4*Z=K{z_(5HR)a};-wkWEDQt8W*=ZjxvC_S>aAzEL#i%ZAV1 zy?vWV7lMabatrRU#dnv-cX|hfA9eoMe3*tR*5K&7HK$IVG>6d>t?9Zci%X!7z?R-8 zf()uY39%EokA6f0aJfd|5U2bnV>F+Lh(n{x2Cc zpULYRZ*eG5*WfPwOIZFbC}`p4PKKA0fKq}22_Xotua;lf?*D@~E?=DWZo9`D?&(cQ z_m?@OGSs;plToe2dCtcLam5cee_9MVsCefZp8d5?2-$5HHZG*2O(hOkK2Lm8t^fyz zW4fcQt)pu0Fg3bw-Rfs+Nw&W>g(8|QH!Mvjev3@-)2APMmPT^oua11QANcLJjC;g} z_4>v;hoE=A&6+GYL6nv9h}(WJPBAR*dl?QNimwPVU?bJ+wRP)x`(GO{9Nf=q)ZwT`uLtRn>Wvi$Xh2~uN3{jQ4sKxqbTZWkEq-v6F-04%We?RR__v29k z(`(UusT$0$lXVitYP;hn?L?wI_Xt~j*T&&>^;z)@jD6uWBp~^al*&*MiHnI?esJ6j zX_-Nn-?QG&k*SDt`Qnnfe`gje=p1!Q3g!xQgXS8!lT669b)-%u2Y!)`PC%0FkOFb| zP`?j#F>Z~D8NoSp`tvM^W@<qSJJ8FA5wXUbQ)R& zqO(T@^8J+)UOm3rQ)2#o9+G|Y({^S)9~}90>~`W%EFWIZv*%N<2urZyv?a^~lnOA+OG#yC9 zdWV<))h!8t@la!GJRyn_zy+d6VmOC3Byqq7Llx_dyd#1ndd(p({lBp^{&kDZw^%^` z11V4HP+>4A88SYw-X?@){Tt2bu=vA+5&W|OT|^5adi29WA(pN9fjZu9SR%f>j;BpOjBh3Vp9WKuHhnmVer7qQ;=u=mTOZG| zLLCGY=`u*uL;%U(ihujv-oBZI#e{!9D$X^V_J)y;7ns0=D@_1Q)GG~B_klj`kIxJz zety8*_w{-~0)VE&f(t5j4!BQg!pCi!x#v{D?joV)u_2q-GSAODHI4AB+b%nuwH5mH$>!SpsIU(oZ*%6Wt=}pE>@Tw$;LI_0kiuvw0s6HoYe_%G}NWu&lw9B=5iw zbv(3IJr`WS5w%A3tg&&Nba^0JZfZaAqO~p@99Ck^=W| zavB1qoE^?%_=s!+-4_C6)4%^i96^!h?~%q%sYS87^M#;&z*L8`Nq2ku^&vL1{Q$)w zdt7MDo2~OX5i-VamwMg$%(}s0RkHn3e;4tz<0K!e>EWx;6w07m?aM)08KjkjyXt?x zh!jK_$R@G{|9|(aQA~h-x<{Ij7$hZc6U|d=*TBGgERF`>j&al;TQKP0>nk3p4-k!# z0a&!DJ?h>&}6Q(E= zb2~nDv$Cx>+FlBqe;HZ)T~H}6h7v$(toxswp{E>tJ|cfbYo1K~Ag(Ad)f^b6N%Jsi z27@N5Lug%+&QsC3l5=G=)apA(Yt(I=bP7?a`a}Zpzh}XD-{bg|k1|0APrcP@Jy#90 zQcgL)I1OLXrKXx6cy^L;0>_<+=NNF-|lZ( zvYTVN#p&7ci$#rUAa|%{Tz=sBX`)$2)fSIbw-00?KMrS5&`2um zwVUQ#X^=D%zsy;0nSr0u@*;cQtFZBEaHGo}^vq?cIV6%Kr0DTm-O9G=QABP}B=HC* z#PF7bSe5|4u`#!D>uC})7}ho@*^2g83FwXnqdIjQe^{M%S@-Xq_`_yB&^teiZ|Fwn zkAwhk*bu^9;zI|PxGsOZLd0EQFzsKyxdp#f4*ZfA_48w?`JYtN?>8D#kGHg@U;3G* z@H63aCMgbgIjcCVn+3*s2_m^oornwQizlLet9a1*Z9{0&cUiQ6y6eQ8(~p?GfU|N} zqKS}&RNg)0nw$RmJx>^kD`*IQLYqhm1-{xZZ65N4eTl@s-KmA zDZq7iY%*YW;xPZFWRshkN6cz?Mwf5<8&WJZs9NmW*OLwxfsz1qf-nHhQN9-Pc0SzQir8OnC`JRg}jdsdb|b63Gv znX1>#v8h^OkcmHxXT{|b*2t!YAp=o`6_%7jHgRA!Y5oa*`@Tn8ucm|r?qw4hxqZa= z*%gSNH3FM(iNcW4Ijg5_*g`nBspu!Hj@A#IFHD(%gHY z;Q2g;0 z(vIzu-O^o0HlEHVyplRRI=}%QU@^plmP4c^gcnQbO*Ix@`_;s^t7nHC5(+HYRvi(3 zDBC3u^?T`Hzx`NH(I=ghHxPe0Eu^X?e^c)r_;D6`h)^vtT>6OuPvb2X(EkW1i(NxH z%J1^12KPGJ^i;`%tHHiQAV8TgQ-&AU z4go1d&#|uO_hC3)`SnrdDF$7q1&GzNlhulRDj#g#rnU6;QWwwUM4#;3V_#UiFGiVi!adu2{oQr-bvoI?i zEbv^TRXNU&#h#=6y5}SR%kcp&OB%&~(%c%VZQ54Q6{%K{x5@I!Ng1&;daebCg$wtYAVo8y!UqPGs1rL1f2_YQ)OaFZv z`Byi2U2V#Ac1#^rOV#y@&>*f9@l4vNKsGFR$4Vt z5`|EEU*@HWUPZe!rHm9I`i04xw%n}d|rnYR-S=32B%jz%pQlQXX|GQ1n z30o6`Z%-b^YgwGTwehdWm#zU1Hkp@=M0a^Gl&l^#7U=nPuT&c1V~U$A=##UcQTZSC zUrZ;ku~fbL!z2F^%}u(Fr?nTB>XwlV;hD+!pJ0u?cb9xqOyyGgm}PHi(67?dCSg#+-BT)f zINL=!NL0gN;!=`q21n;NHh>?GwzA#1eT<7l+7r`QFrsKwlG=_;PJ=7v5wxt9%RZEu z6Y$*xJ13VH&XSfwA3s@^ZW&KFV6fnjbm@3RL4kDV_uy0y57Rhj%}w_@DR1 z$7`9!oobEt7UEl~k}tjOJY&(FZ(+ z#P#bf+*v^5X!MK=w&bv zTAN7CWAd3gMl;K=Gs`jfzR{Vv4&UmHBezzTnsk>{y!BT~iS^@u+2El0A^gfzftO7Z&HUtmh%q~bfWnG2-zDER^zO5 zz>FY`om<)hoSaHtR(F*4HK}k|-v1~s`gC%@cYZ$PqP|t~_?NYc_vnEG2J60fp=UHp z$ExGB@b2>S_MyHyL2t)-yPiJ^W0T!4#il2s^@g=AP23Q2w(zQN#G4d%NYY`NLd-Nm z$A%0A;F6vtMnwSSNiRnDT{%x1XOr{u)Pw_vS3|qZJlFItP$}ZBkk@RkJCc$f<{R9q zAAMx}4yZkGoUmxR94SQ>k!q|QaV=pY-TG^FwOXuw4XS$W8(kpO6 zEcW3k6VwJhgoYtNDf?0Uk?iHF5=GH+G4pwdY*&9ztE(wzuj8bYFV}IdyPrKkCOM+^ zP|LnR$hxJscymvLH|nL*%k8~2QJ$W`MjyofOvrXsI=};YVdS2D&JBf_>t`+L-EOT6 zWx`StKa;4lHhp<>U>xAXg=|ci14$*W=@9P(hN%61P(f*87pP^v-rr*&Cle6z@XqaL zQ=xmTBJZp%wWp$`_vYwHyU@BUryj)^Bjp$qjhG8;NyY=JmXo7JzENGJhf@|UyGt%R zM1U^IO;urH@}!+@1E;_kW>d=XlMPc zL7^Co*Q&woUe61b>3Xu0Y36#rMU4V1uOW4YvNE{HK5-)#+CPUyB`{!d4FeW0h%**y zXDqE3e1tN!af@HRu$_wJdgzn zTf(aO=*-EW8OzQG&%^Xfdz)r4ag4Lu=aYcl$&-kFC~>kHNs067VSIUWZ(vE>z(y_^ z<@lhDG$ucFCXH3nDnc|C8%^=e2O6XP4!09sTb<6FqCz=(tw{PeW!{$gxL(Q&rhV+| zclM?w|9rpt;u(i=1=W&MSfaJsMSO#l#p7k6zUkJ7P0K^~KVrGb z*@K#--8J-cdHK0Drt<4TJKx0*W|nSa+NZ9Oqoc{#Q~yKAy157e^{7$M^)|7krhbc; znJadY?TrblZMAV)wwYo#o(Qqkzvu0VJ7Aa4kkzowCs0WC<`p6ROxn1(B{^X$vldDB zX!+ocdG?O73$xF^4lWukHMCQIf-j)^}{KTI(3o7hCj-waQmpE;ajV*5tR$6-qDN{Z7= z>1vu*NB--5rZqASo&Lc_pW{_^Cx6>bX~dXc@9B#_oPvx)s6ytSUchqL+&bs*PL|8Wc!COSnM&D)PlI6gFv~P%@gofc?B=Hg0 zcuLu7-2C*y9@(J2$M6e^b6gxLEL+&Ij)iBU-ykJuZti%zCY~#AfTkX78I5CXu0w#W zSli+Lh_Y+nfr^mR+2+?Rza%$u#ebLDAt=+3DBX}Gi-YP?JfFx^h*I(!`z48Zu`937 z3D&Wgw#;`0b(CSE+;xk=w^tUZkQ@?*6H#iB5tAT{D*+S?;2_L5;*_kNZBHeI@(S)G zm4nS??=0KI4f1GvMs*J#)JXnjT+{lgN}bi|djMU=V&kQlYqj?M&&GPKCM4Z)C~pS{ zFOQjwC?75nsU(!E(v{#@L7o#qDty(^bK*V(ZFzfuDy{y*^-3>(8F;~;4-PW=0Gz`F z_~FlNk?VclUcSYFf~ItI!9^c9F2(U^WCeY&&N#J@DkF19e|T!l7vliGqh1pVc06N_ zkGB15htZ-lS}z(9ukTtQNFBzYKv^)%TJiB+gN5eDw?d02cuLe~d1bfse0_w}W?xU& zfc^{frpcwX7fw{*MB*@qMKk~udvl%H@!RH5QRO|>ASP)*Z)jm=&wLFjYCV9m!uHwT;r!JxX1{``nD_%+W{XZNT6B^goE;;+ z3!PhXn)Ast2@eia)5Fp^7JJ(Jz)-3FzIMwI-)g!nm!f`|*0c}yj`^!d5*;}cY#I`{ zmrH(AnpMmg*BZYPFG;czQ>TUxIGQ<4KDVEzqAGi|r(q~3*muI8Zy*S@pj`#%eYWs_ zwRdo68KI*fe~CY8#Qc4B{Gk3J+pm(kh2F9gz0Xm$4mu~PIln%qH{+q1!%>ib%;qDe zeI_~DsywqA3Ael^Rh#)~X6b`~I~#Nl)6SPG-Y~%8FNl`nBa!Rnf9zUDBpG zH*I3dpXT)!XeSAknE)wS6hip0Rs3jpDGL)$A%$@BefVE}{<50uZ5lW&Y?6kmi_Bv( zIs&fi^H@*SDBHGl#cHl74SIgPHD1H_=SSZyd`IqWUB`Tbvus;)r%Nr*Sopb!GhK|{ zXx^@+t&5XRwU$ESxWu@6_9ht;NaUdQp+9(NDF#nY&}K_`1nWA_KHGR_qGznFj{j43 zu(O3l$j^r(W$9ApgCR%k61L?nE$RoXq?PgIl5KSz3faoys{Z!M#qEdknWsd}!~G6N zTlUgYa@SzL>%*ElMQv7p`?CGhHw=p;u2XAw)3^nzi_o^zOvuI!>1k!AHI!ywc;(j! z@Md~;PmF{0z7y&X9$7Pcz0Bl|2>o{fpl{P>+N^3=^|hZMU5g@kUK=^jnMqZMoH#*7 zj*!u4u>G;VszWjJ{rj~PO*Hp6e)Q#2mlBb1o#{l0>GN97pPu;%K)-t|7IL8&uu_hmxz2mL8n-jw{ADa;)1_?wi2XTS5obKi^W+iBw88`O5Xj zgHBe&G3b5Q;^H*5)IT?y z5x~V6v^w$XGoWU15-)!Ck8b-_E>=1_zYqls^=4z_A>BysOCc8lo-qhBDATnWgW{aL z^pHhFm6%k4u^mn(4tt*%^(23WeS_gLvb&km#;F(_8K*1T9k+6+KS@t;EaGfw?Yu%n zWUFi7>vQIEOk=h!2LprNDM`=vAC5uf7*HLves*MEdtXlT>+@X&?(C<5p)JRZZJ#Z8 z|D=*0*rhbHU&~6WT2SzMiCSdvTa9lu8AJB`zFC$-WkV7uI$#z(Pb&}dADJkBaWf@I zE6ecl>7`Dl7_+OEHgoCLQFWT%>t`dvBg4Xir^IHZetyWy)cBN`pBayo+`Ut^!t@fx zI6vBUn9Nh0OoXoj`7=oOR#=D>+eB^m=UZFjY)vhl&GhXDjwWS$F_pf2v)6Lw;}c*W zjfH26gKE^LVBdHc@MNQ=CSW;Vc^TP&rp%!MOhmfAcr0SE8fCzzaqw$;3UEFX4GpPj zN#&()yad7{{9*{_fC#e9n6Rw)8a^87VI1Z&-@KdkyF}&_pa=U}H_(1P&f2BNM`+OH z*U6;s3wUm6(+^WjLcx8xG88G@b%KI5a`0efAuEAH=h|otW-P`&+D%_gI?ehw_~HQO zyEW!9qw_A04xP_3xwagc1t#zU$^`<}mongq7b^=#>V-DU-!B~C>yzguKzy#bG)ZA(;f!0vQ$*0TG&1= z5NvrD@jv!+<@Jg^2;pLvz$@7ddc*Q$8X~xY5BVXDKG+#QWN{u>fJxD5!jwRR)EZsY zd(MPb0siDZWM{>&*aT)A2Qsp7zvchRQLq9>q0QxlNNH3JHpUuy0rWSxFP8|dM$35L zcbFH*HTwvu%~wWMmONLS8tk&0H>iqqr@<@$9C<)EqhI)JKk9Qi_CQ^Dd0_dx8jscs z()hsbubzbsKs7`z1Y3fFMiE_N4nnYneDoTGzGS2eQOSTpLS+9iKqGc!;Kv0yzXYRz zK!|zL-n~b_yWu>IwpbJb6Cg)O|ksp>2|%og|1RP z!8y+JC0;A9Bnd~mAQ>%3Lr848jfN=T43V_~^EBcou-t6qSh^6tig}tCKAsHt&9EIJ zx-l|k23!I|C_jkKmqQwi3PpI2O~Cv84R{WO%})%UoY8V7JY9I(#>0_;^vE4BDvEFr z;ttYKnhHHP=+)Wa+gzRzZ1r1QY@B~Cz}w!(uU;SPnoH^F42(*-8TEK)Xdp8aA5YeF zxX?cJ&f>?7FPsJ1`^21ohHNcsRO_viM~mH%99d`Z?#Vfz;(@vZCHyk2q8M6FX4RC# zC&A3-*w?W{7e_!NP%B^%m^yS3ULu?KUJxhhKom?LAvJ^_^ko?TB@S)euwj|v1Z{}a zgbIdSY~HcMh?vUpN)tmZu&eJfH!ciIS;$SzXY71;_?nsZg00u*nu+84X_f0-Q@6%y z<{Zy&>z%FoJyIBTSl^^YD@*lfPHVqj=;3I;AJJB>OXqrj?3NI5kZMeA1a!K5QS;06)hs}29c~9B!syXyVEg21!wTHi*6?1dFtNWyY>dGc* ziKZXw7P9l50S|7-r$yX1($O{Kj*8NK!<&*85On2%>g&6Af@ac&qI%lzPVD6#Tr(Kv zNiEbpRcOEUG$ZXZ$Dv%OmUyOky}_MbV_8hwg{DFr%if)djP3(YO>b0tA~)SS_u)31 zKTTKe)x5j3^{cp~&ehrqmaq`xba(hedV&V9d)KaAg=lir?N6sq|8XN9eDGn{r72fC zENCi9QBhGv-vmfq-$s7Iv|`tctLxJ}TwL8ZLiC5qTLJ??u+Uay5$wiz+YNktd<7zY z*WN`bmZf|BtPQhvaXES;H2(5>)$VR(kN4ebr|>7HvlfSjzwQ_~U^}?wd(O;mlU9>Q z)_pe5UfA|f16lCa$awLkI%aWr^ym8 z9e<$i-YQ_cb+9#ufo%o2?)T7_x(b^4G)%=eecy?HB4eHVTi(yJ9)?!v6jol(3@AXD zpREhRRIc`cy)U$$Y#R|&D%z&eDqFQZxvV^%_WfARTZz8;dBcPWC!;qKeNLA7g>W7G z@y87$7N(i3(hWO9ZNKKlDEgI(Ic>G;+)P*BKy~oolJ>lvqj77TN2(wr)0Fpb+0dQB zI`=+Y&(l6aZJMR4H1E{C=jdiLTi0o)qQ4xOh1eYj)spRKt6tY!%;IF{Zb&>^=QEMe zLpuvD61Bd*e&*8RoRqx$S@?YY>h8XWRPPm(Ko4}FD`AGSdjW1n0mhS6HqcDy=;^V8 z4tbV;7H!Nnm~%OlW;@SWWNJLTvV@$l|I5I@KuS*TD+nDMVsef7Pp(>r?}-$nKHYjhNHv!In4Wf! zdFRd#y5IBI9stG1pvMysBvMh}h6x7xUP!MD9ZErM(>}13Oab+(J0~ewyL*>-S({7O zxxx6tK^H|y7bE9)2NjwI8pZ44a>FD>*-tMmS~%j*yO(&lyW(@)5_PF~hv*yA6drT@ zsac!mv~9X$V_T!C`<0{)*ZZDtI+A)=tSHkM36p>b1=Vyoh%gj4cuMAJAIL2&ExjfL zPJJF&UY;PrMSz-fg|{Vt?RZPytV>g;76vtB}H6H zY8AK`9%xdvgCla*!r~B7gRa{lz*DYi!OqSt^5u(LRWy}YPSc#Goy5%i{N4K2PZ3Uw z5xePrrZ|s^Mpef%RKMfWHy*0vUs}6G#@vEhbaoXUIp0|A`yB~7_h+pcH$1gKoNxhi z>-OEdzoD-F_U)TE(wYmfC2@kv2md!0a|(C@CU5}KP>ApTb;on()`4iXf&x!@+W!9j z3Jh@8(a{M5P4OAXZ9N|+Cnr@hYkllkP0GBxATb%fV_oj=0)|wsJw>ltWZtPU;n4fEWo~-JV*R+XCKbySr_<(V zBGm6`#0M^gitJdmex+X73d6d~@$~HMijU6=k`DWQ_(``x*)}}taLimD9EAh0ob}RM zeVA_uQH&PqmCrb?UrYyqAr-5ZTILhAVZ+m}q9Rrr{voz6NBBmk^cX&rzoW~s=)AT` z_x1W6qb-BKsq?>bza09rcZh1}$Empc*>(qqe;Va_&{>E~Gl*WuzB_CtB(3fsJp7$M zx9O9G=Mwz*sb#E^M~~v!CgRy*K4C;{$|WUWChPur*N(U$11jB{rli{iBcnN~J6y9?ZOL$9#hwO=(`I>ROxqm#O6 z#I?hiFSn-hSHxZ^bA}vGZoT)Wh5)P$dim>a;ZS;P zMbF2NYjvD2;>L>`{q`}idhg)kGM1HkA~Lu8EceSC+xXFKEAKz@-0sLBzK{Styiefu-=r5Q7o7vdZrk}u_&%T_S(v^SD=qAUtqr1UM zxO1#7wkV5Nc>5a_1yTC77qjoqSI4dfk3yyR31t?Tc5 z^5o%^p|rKNxgl3H(4fWTJKI}C!ej&QiS#h>Q~PrlU9Gd>Zc3>8xSx?ZWK46sT1ors zI(v8VN3`j+Mn5cf+OV^CHaO9Uu(45TIZtec3l%M8)tWV5z(H3CUi$zyeC1J+{#eW4 zpS^+17k}PbnQPlhRTWJG1TJ+YZSnoe>hKBG@!Aynxs;n0-<%!N)w@h8c1;-VVoyAR z*R}*5_n)e01_*_3Vf#^Xp`BI0v=|<(Y)tDLuSh#dUZJpg>V@#BKGrB)T{S-MMNkmxkACT1iqJce9e*U1w%1$H5VsRd$z|r0e-UyWvehF< zNWaqxV;}ifIM@m9+VO3NV6o*?5V!36;MkpE|l?R*~Q*?HQJg=wLVFof!NYtZv@Si z5h*1KvvxQOpT7IkD4G$k9zx!=c7vZXr(zD~vrxE|o;=c_sANw6nvCQKJGZTK>|?e# zKE_sDI*qa90uX}Hjw-IMCbYs$Eto@gS`F*raKzAIys#n6ymFQux2EM46>?& zYeni+S(gSHExKkOKD_5!-O%{GW&DN6zi$wYG)8pa&%$i(S6A0vBx|SQ>w+iw)zcWk z?%RG@3K#P8t(DC(X!dwNF)}r{zB+FG)X# z2@c-TKQLg~@BQcz{v3R!1|fDsrvS+sVm|!qhHkIUeWQ?A*HmzN<^+4c(^lT4)Jf`GlphHzU-SNhrDvMhJl;J7m)G5_?y2S<4w*D)?Tbk29)!pV|=cY%* zL@~MHseSU`=fxpccL|RU_z!~!Sa$RpE>I!xbP0xzbjndIf`Iv+d}2z;|4CTmbA2mPs(QxoW6a$+GBcc-!^F1 z@r7MGZU0028>4ppfzGk}-vkoqLb1v`*F;Y64DQ<}?#=Qkb*i64;?bEiyDvoCb!tl9 zH$2rkI`V`qw>qY{d*joa+BE<9m6n#=saBc|8{C?gzPfFHWDy(~I67luV)4weF(tRy zUmWU2{lA_LT>Y#Icy`8EoZcwMjH*6|HO*@EF)8ms7_ z7bdX~zq3F<*g4su6Q(Qs|M{7dr`#__yX8xUoq2T6_sh&I^;q}*mpDK5-7{?6)2%-i zaj}2D{`b@VwvL5Gy#1@HG8;~LMkcyHR^snfktC`rOG*ah{{46i%C_r0RgSnDUUn@< zpMjBiet2fEIk)!xsQJR|mc0)>W2u+JH@>CGqLX%yVp~#q#|mpvQH%S6q4dM!)#*4` zHywY!b{AQK;qwLg6peIB=y95y#Wf|*sFdDv+-oi@Fi=)Lb62>pVq?P^feJ`lJUwOH z=y$qJ^tp!~cXy|8cU@S!^&s_LWw~Gnms#l+-&QQSBCem+;z~cwJdcyN_|!kcOhRs5 zbREt0Lh9~v8s}Nch7IC6k-|qqglJaKiaN3ESesu+bCR-tJEgtw-9cnR&M&GsW>kt`YW6S-Om-xY@*Ut&_YJ`uTFl-cP>xqV) zpgSgYmvbpn-%BIAdc&1ZEm`hAKk5!yD=C#ZV-Tit;r{q_pwim|tZ(8}@2uTrT5o z4PRODp*?S+-hay$+F13O;ZK?j%qRBdFz>91ahEdqd+}}Xrv&m*85oR?jI^WC&*1o^ zrs92YX7@t%K6_TDKt7A2)R(T8^~2L2?dV+jKr;)4V`s@otIOJwpgl{nO56Ya2IhXR zwJ%Tg^)-I>n3ds(DLvT>ncmDXPV1H9i(fdgxt)2`A>cvv>@eMQJ45GKXo#D)DD`6& zf2LLG)-(Y;3Zj8h=llPTRf!f}TZ%NtT3zWh9oNHeC+?I>o0~&cA+IWBppgcerMU{$ z*61S<3UW%eoe$I*ME9>RbgD!NdI}YlxO#}d_sk9Z-2b&q1dRfi zf12pi#cHm1nvGGvTvBt1-dZ`&XHO!nsj2qJwJ>m^UjsQBHK}UVFY8nD~eK)>-#LB;8!0?&yrIw-4Anu`4t(n_yR7y@7 zPyKm?#9d%ZeS!r2hoK6I1|IhJ|Ni}2loB^5jWOdv=$(Vgv(cmVjo;ZiDQ&;lc63uN zWTgu;t!8F+W751J8hs$;Ce4|LPqJpgHadHhaWmb&c6pqU@~&fBEBaVXWY4dBEgv`) z`R<2!YLc&SeslWDrmX$?eoHw^zFEBtEPLUm8+6otBbQ&f^m+e{HM+sar+%r9mj)p$ z7W@-@pZcpMP^JfuH|%ol+V+X}z=g@Sx1D(mofS66k896Zv@J}ze-Cu;(9vFW>Ktot zf0;Q{wzG4L%AQRpe^r>i{ET`$9X*SmcTLdwmokF?dTUK>pL55`ze`uDrmSdjl%EWF z7h|d4_Vv-d^2X$+eF=25)S{xr-Gat@^`A1aPi3F&Qv~&T^d!J^i{IJuqEukho@%O5 z^?vd1OUj@gZx9a=)exvWe5e`-nhO1|l+?h(>E1i{I}T@Mb^c{G(Yr4g{`&nb>!)Wr zTa-T~CN?yLgmiQ`o%N=sp`&h2FS3--*~eWvL(&aN$%3dVD$}ezjz(PPE(kgLegeQ z$vx+q(??~`L-A6CLg>2v(&DIjU0mX?xa2EI7N!~2YxgSCKNFOnaiXkZ2=vX5il)h# zXXm=#_jB{sqcZfLy&qqWrsvM87hld15~*LZq{qpWWt7`7^~|wkO=a9{C#H zUWU5ZPeF94R8UvmS>^%rwWS|t6R&0)`ZHh4F^?EMx&qbWY-LqvqMLcwHgH1UGFPF20fa#|p%gIpA+0YzSZ z*Ip?Xdk4`bIVJUf0wDXLQxmQr%8Z&b$3im48;uv@GC%_#aoQ;HcF=| z6$z>Fo&R6K^v|_p?_bfXm8ag_ZY-kp{(s=@w+t^-LB&O(u*ctNmfk4*`Q@4W<};(Z zCnl;dLZT+d27bX7uP_$TK%0hsbW#0+IL0qWJWk5+5<&D37_yQ`7% z!gT%7dWZk{muPhaGE$?Kaz`w#jjC8maM!$rL2h|1EiABiDf+O01{((1m5w@GX57Q&}4`kBmriSrBM&`Rqa{M!Ak3YGFBpLRu&SOXK0e_pBIqhgq5=A=xAy@Y5nds#OanLH{QYqE zbR%%gwukzkVsimK1Kbnin)}0n=mm1sFygJANE94q3r6R#^bXalrd1n7qHKCxom09v?q%VABc{jxQ-47 z?wm)_1D8ghf^^;>vTg$oz#`(H($Cz{DgAPRj5xdYTuB&-$(~9;1tU$C<-f>agXG7c ziu5;UFagpD>bQ6JE<1F|<8Z7yRFH^Q+|$#uYW?~GSe0R0*avyRyf?;>7<92^|Gcz8<2gW3zd^AH%|a`U;>bqOpMJrW2XEQmh4=o__p z-0=dM&&*;=kbVM%S#4rs0(9r%hs08m7KFv|-kP=eFy70Wnrnm?(f?@&Q1(pNnwxYS zc~p~EL0Enk;|&UoineUtd20B(ufS0rQhp1*@4EmGN!2z>U;6Xrbh(4&4u>vIDu zD=2TAVen>r(DB@o9%QD6kWw)pw?Pld*o~1I8_sUH&7FIuL9Z-ib*P+jf612)SR^pv zA1ZpRNSnUYwSQ%;RsK*54n4?k=a^YrZ_dun zzJ`8RI6Mxn!IVTu(-tNORN~CSR#Fr5aKn2*yjtcF6xboNfFr`Gn(ego%zFuoN>x+9 zvTOz(Oh}iA$yr}fKu{1j#Sh8CCV^SQuC^)Bdizpl3`q4UHadL?5Rw&7jGYjflW+$ufI-Gc&so z`Fablsy2c}npMwVw;AhGZtj3FggbnsPLGyAkGK|-bEy%Ev6}O-JRGMwDo+;M?y{Y)>iP=sTy=ihw0~WX?sBXQL%PrmM#kO83}TN1+aH}=b#;Wr z%F<9J#rU~OmcvI+Z*L~F)7fEBd<4@i@$SVQbm(4;plfb&AQT~-H{4djp_r<>k@#L5 zk@uYUUP5f#fI!9smd8){fb^QUi7xS^M^B5+?My9G$iGx?6 za$?9FlX^CiU4h-^{M+%x346t@>u3+UF3z8crP<(hQNgsEL+xhl;=Hw=)%sT@T z%D<;EZ;2B=vg8~njf+S)+MMI6wGIZ>6h?Y_db0NbIqwe$w zgu?obPYzZ$l{l8p1B9Lj*!2%!SF}1ylOJnBL3{n3_xmzz>kA4_pMxnm2{Z_mr~g<9 ze?TMOt6Su4SZf}LZL5@(m8rzBw!g==Sju+hU?ea*#<`C(>rIBKu`gkJe8K0zI^h8h zrU%K%M-Jj7rxGWlgSWhw@XQqRA{q$dY@;4OF*33NsqsgZMc=Z|7}<44GroMH1=EZ- zqkCQ|n3HyNHCsN$6?5VhbKy8w{=*X5WiCH;MgOV< z-50Vir=4}XYAF;9COs$;ryS1$18Z;ay{_Wb-GDt-!C1A9i3nuO1<0;|E?Scb(W@93 z1SgpHJym{kcIZ8J01G7Jo;CwN*$y7u0LoR;jq{HmKc2>vU&d|QR%2n;H#SD#_2Jap zc$@J~@gc4u1uRlrX*UuIKI8CDZ7FAB2U8 zSGU@4J1<|;#~^mK-(k`eN9J(={*(oVcw+GhFg%9!6)ilV>pxj?!QYBjOe}{vOCJlQ z5TlR~!fqk1=OKu|UgxBp-7(zn9Skwv7xdEEdjy6LP}$MZP+r(~Z9|s468U^SXnb4E7Tzm~)mE9kZvt zsMX43G;+E+UW*k`_bmK^4#Jh|^r5vrNI2j=bsdUN zEU}&N6Lf>-67t;@;94l0$bsaP;)ZInqn-tmZNJ6#wzRa6Ak7ySH z)D?ILdtvQ%*)70o&I5y+wm5uqRY2@zKYaLbo#7$kw0Xi6U1KRJsa)dfbc5bs(>Tnx zCg$?{?KllBn&)dqd%EsV>NgZ##s^;)dV zuk50tLf^sdk6t?_)qLA8HJQjSvvdw$=~sKZ2af;WVoNg&Br%cFoMgR@hGssp`Ck~j z+2;&h&TVuzpXpuTD~tUR@V3au{>}&O&(tJeFDxv)HWP57CA=fJ9VaSMJa^6bLYOK( zsPRy`a2`ADTXAvmgrlCRsa9meTV8Ye#O~0&GUo%n#VbawnV+AB-NlpZbKsOSGBRpU z&OKA})r>!jg>dQner~oEw;5M1?znn?-nP|QunrIR01N>0^Yiz z!J+HQJ9U-cNqdhS+X|zgV`gn`290TzV{U1A3RMEc<23yI{2*boASu#(EM{ZUl6fD$ z7+l*_vY3}!SC?9^d}eJiOtyS zP_!Sv?cMCVPY4NZLzVQJr5&C(;n?)UQ&aral2=#OSWXRUIANQwTD$fX07w${P)^c; zf^!nn<&Fp~!qaRXNW!0{Plr*sFd}0{z8wJ#z#FZcZ1$tD>itq!NC)QuLMMZT*!?(U zyapDrw?-2a6DyMR<(W^|F%s$>&Xy-3A?w8H$Ex}!UVpV07pGu=XwdbdjPQ9^gZn`| z&*`2mH4p}Lf77Rpc+eLY>@au7^z9g;)$6YkFv~;x^!1++eZjJ-s;UvNb=$B8(tt`( zgy6N%($YV6Rqi~k2gyplf_a4!?jdkff_z}o_-FW(to7H8vAmFBb5ki^b0Y`luW|-V1kCA>0BeKnR7BQmume5jGJlMC2!9howX+VQw?{ZxUg4Y|U;xPi zkoRdk&Ae6i856QTfMQQrl6WQ%I_G1)>KYWpRv2kKGS9WZ`x{9L2DcTAQSIX7q=mR~ zcx2@B3lYZy79CQEx<6xcF7k|A*@Q+u%J_BL4y}Mm*s05xx4>ZIF>HA@r5$;{r_4Pg z<2Wg{H*a2v69*~6Lo}yA=p$2;+`Gf}9;F(k)-^WfLB4zW&pP&v>xW#q9KunBc!H3# znu&=?0rp@c&^~ml6tlyfhw^O#eo*ga7Tf()^muc&S5{S>hAS6tZ37NBQcU1ZB!PhJ z_u;yY#e|tC<#pH>D#Dk@L8?kDAsycBcQMD&tDaTST& zyoPvqT9~z~n&UPj_ADa+K_<+#*Z{WxMx3jPJn>f_kN4TOMnq%IS!pYX5^8v{*h^tZGgd~O89vlc}d`b(} zp$A#5AfpL9k3YS>)~v)kaiX2hj-JwZoG#_XOQuEo8{T3xjl}>F!PoUVS}VDSb?EP2 z7dv|*^*``4+JRK`=F3Pt`P+YC!T<9I{%Lb~{vVMmh1C{db7aD)8p*K-f-JT>i4OSI zH#D}RK;FD@8TQ(fk4^If#@K|;3aBm*iNyg7`z479UP7Av*z0Qz&!0bkkR^y;l3u<~ ze@2`)xMm?%#1G^g({pnXNDJyS?GKZ~W2(iGfdb4c0q>r@N-u_5sz=Vr+V+_xo!U=SekoS-%^#A<07Aey4rluyw?c0MQ zq9J_cC5J4QWCDEu$n!a7)4z3csMXY=^wp_*B)%ftsSRqQ3vKM@QM(fKA%)NI{awwr z0^_Adn^+MWAj$ZJbZKRF9c&!6k8B@-PD3KPii!ep5kv5>TUISt{p_<_4 zK`_6MbAR{2gFnV$W8B;6kOL!FZTR@5xh{#4tQ%PKZ)CxIKR=Tjre9t;+*EKHr4N<6 zhs^yI?)lRsE07foXeRGKF8X`Tlk(!?Ln3mrvZrt$egM{{KGY7ILx=v}-XNnTFaGRR zh#N?@B;xCK@87=`)g4dSD=dQ_z>(3yP@=Z8x5q=H2dS11o*|*_qT?MMIy@Tbo3KD= zcI{|@C-Cb9WEYYS4(UDi*n0^G&*X@u^>}r`8fm6Bc(HLfvQA>93ps6OdS~oELeGc-uUl1pE#Ei;Q@z z&-s5=Zx3v&b^*r~Sy)&W024Q`XT2;_t&CxYSuSuT?7AM{`M;o5)D^%k%EL#GlB(o^ z#X6|o1Xjc^?(MY(CKX_Z1QchpuP%-QH6t@FGJ!_DfTN15Z+rxf{J zynti1pe8J^)zt!QhOM5Y+2R1~h5~me Date: Mon, 13 Jul 2026 16:40:03 +0300 Subject: [PATCH 069/107] chore(deps): bump uuid from 1.23.4 to 1.23.5 in /python (#2021) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.23.4 to 1.23.5. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.23.4...v1.23.5) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.23.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 64c9566c7e..959d7895df 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -3948,7 +3948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4353,9 +4353,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.4" +version = "1.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" dependencies = [ "getrandom 0.4.2", "js-sys", From b85ab0801ee321b4cd7dcad5919fa95282dc0f9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:40:16 +0300 Subject: [PATCH 070/107] chore(deps): bump prost-types from 0.14.3 to 0.14.4 in /python (#2020) Bumps [prost-types](https://github.com/tokio-rs/prost) from 0.14.3 to 0.14.4. - [Release notes](https://github.com/tokio-rs/prost/releases) - [Changelog](https://github.com/tokio-rs/prost/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/prost/compare/v0.14.3...v0.14.4) --- updated-dependencies: - dependency-name: prost-types dependency-version: 0.14.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 959d7895df..530c8cbb05 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -3084,9 +3084,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] From cd5a88cd97b0a83eaa2e2ca85734473db652ef10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:40:26 +0300 Subject: [PATCH 071/107] chore(deps): bump uuid from 1.23.4 to 1.23.5 (#2019) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.23.4 to 1.23.5. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.23.4...v1.23.5) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.23.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43ea105778..bf66e95209 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7748,9 +7748,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" dependencies = [ "atomic", "getrandom 0.4.2", From 7a91d406ef6c34bf7e1fb7224877cadf1f9f581d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:40:39 +0300 Subject: [PATCH 072/107] chore(deps): bump lru from 0.18.0 to 0.18.1 (#2018) Bumps [lru](https://github.com/jeromefroe/lru-rs) from 0.18.0 to 0.18.1. - [Changelog](https://github.com/jeromefroe/lru-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/jeromefroe/lru-rs/compare/0.18.0...0.18.1) --- updated-dependencies: - dependency-name: lru dependency-version: 0.18.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf66e95209..16ae8682ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1159,7 +1159,7 @@ dependencies = [ "futures", "libc", "log", - "lru 0.18.0", + "lru 0.18.1", "memory-stats", "mimalloc", "parking_lot", @@ -4500,9 +4500,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" dependencies = [ "hashbrown 0.17.1", ] @@ -5748,7 +5748,7 @@ dependencies = [ "hashbrown 0.17.1", "itertools 0.14.0", "kasuari", - "lru 0.18.0", + "lru 0.18.1", "palette", "serde", "strum", @@ -6929,7 +6929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", From 3985f10742f4ece2e08d53a7ca74133e36cc9d98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:41:02 +0300 Subject: [PATCH 073/107] chore(deps): bump taiki-e/install-action from 2.83.0 to 2.83.2 (#2017) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.83.0 to 2.83.2. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/c7eb1735f09259a5035e8e5d44b1406b1cddc0fb...43aecc8d72668fbcfe75c31400bc4f890f1c5853) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.83.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rust.yml | 2 +- .github/workflows/tpch.yml | 2 +- .github/workflows/web-tui.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9818290539..c2679ed8dd 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,7 +235,7 @@ jobs: rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt - uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 + uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 with: tool: cargo-tomlfmt@0.2.1 diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index c5bc57159d..8e21eaf4b6 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -77,7 +77,7 @@ jobs: -p ballista-benchmarks - name: Install tpchgen-cli - uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 + uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 with: tool: tpchgen-cli@2.0.2 diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 4bca1c4117..f35c9231c8 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -64,12 +64,12 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk - uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 + uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 with: tool: trunk@0.21.14 - name: Install cargo-get - uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2.83.0 + uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 with: tool: cargo-get@1.4.0 From 5dacdf02cea825994ad711417914aa384cfd8808 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 13 Jul 2026 07:48:21 -0600 Subject: [PATCH 074/107] ci: install Rust via rustup in wheel build to fix ASF allowlist startup_failure (#2008) The `dtolnay/rust-toolchain@29eef336...` SHA pinned in #1748 is not on the ASF INFRA third-party actions allowlist, so the Python Release Build is rejected at startup (startup_failure) when an `*-rc*` tag is pushed. This only surfaces on release tags because the workflow runs solely on `*-rc*` tags and `python/**` PRs, so the pin was never exercised on main. Replace the `dtolnay/rust-toolchain` action with plain `rustup` shell commands, matching how rust.yml already installs the toolchain. This removes the allowlist dependency entirely so a future action bump cannot re-break the release build. --- .github/workflows/build.yml | 25 ++++++++++++++++++++----- .github/workflows/web-tui.yml | 8 ++++---- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91a47671af..a2507bc46b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,7 +49,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Protoc @@ -127,7 +130,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 with: # FIXME: temporary workaround, see: https://github.com/Swatinem/rust-cache/issues/341 @@ -176,7 +182,10 @@ jobs: steps: - uses: actions/checkout@v7.0.0 - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - run: rm LICENSE.txt @@ -218,7 +227,10 @@ jobs: steps: - uses: actions/checkout@v7.0.0 - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - run: rm LICENSE.txt @@ -260,7 +272,10 @@ jobs: steps: - uses: actions/checkout@v7.0.0 - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - run: rm LICENSE.txt diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index f35c9231c8..ba7b17e981 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -56,10 +56,10 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - toolchain: "stable" - targets: "wasm32-unknown-unknown" + run: | + rustup toolchain install stable + rustup default stable + rustup target add wasm32-unknown-unknown - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 From cf43961f8cf947308269e106e7833eda77d48c6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:08:16 +0100 Subject: [PATCH 075/107] chore(deps): bump rustls from 0.23.41 to 0.23.42 (#2035) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16ae8682ba..fca0e4a209 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6154,9 +6154,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", From de6d615190d57106a492d72cee7b6633599d285c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 15 Jul 2026 14:18:06 -0600 Subject: [PATCH 076/107] Update project description to remove Apache Arrow (#2044) --- python/Cargo.toml | 2 +- python/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Cargo.toml b/python/Cargo.toml index 97a2bead29..0712605831 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -21,7 +21,7 @@ version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" repository = "https://github.com/apache/datafusion-ballista" authors = ["Apache DataFusion "] -description = "Apache Arrow Ballista Python Client" +description = "Apache DataFusion Ballista Python Client" readme = "README.md" license = "Apache-2.0" edition = "2021" diff --git a/python/pyproject.toml b/python/pyproject.toml index b3818cf374..081c504b4d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,7 +21,7 @@ build-backend = "maturin" [project] name = "ballista" -description = "Python client for Apache Arrow Ballista Distributed SQL Query Engine" +description = "Python client for Apache DataFusion Ballista Distributed SQL Query Engine" readme = "README.md" license = {file = "LICENSE.txt"} requires-python = ">=3.10" From eff11b1653a4352400a0e8ff44c8312458ba713f Mon Sep 17 00:00:00 2001 From: alexander domenti <141058955+sandugood@users.noreply.github.com> Date: Thu, 16 Jul 2026 01:31:23 +0300 Subject: [PATCH 077/107] feat: make compression codec configurable (#2039) --- ballista/core/src/config.rs | 32 +++++++++++++++++++ .../src/execution_plans/shuffle_writer.rs | 19 +++++------ .../execution_plans/sort_shuffle/config.rs | 12 ++----- .../src/execution_plans/sort_shuffle/spill.rs | 16 +++++----- .../execution_plans/sort_shuffle/writer.rs | 14 +++++--- ballista/core/src/serde/mod.rs | 6 +--- ballista/core/src/utils.rs | 16 ++++++++-- ballista/scheduler/src/planner.rs | 1 - benchmarks/benches/sort_shuffle.rs | 3 +- benchmarks/src/bin/shuffle_bench.rs | 4 +-- 10 files changed, 77 insertions(+), 46 deletions(-) diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index 087a646b8e..103c59dda9 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -20,6 +20,8 @@ use crate::error::{BallistaError, Result}; use crate::execution_plans::DEFAULT_SHUFFLE_CHANNEL_CAPACITY; +use datafusion::arrow::ipc::CompressionType; +use datafusion::error::DataFusionError; use datafusion::{ arrow::datatypes::DataType, common::config_err, config::ConfigExtension, }; @@ -142,6 +144,9 @@ pub const BALLISTA_CHAOS_EXECUTION_FAULT_TYPE: &str = /// Configuration key for the optional RNG seed used by chaos-monkey execution. /// An empty string (default) means non-deterministic; set to a `u64` value for reproducibility. pub const BALLISTA_CHAOS_EXECUTION_SEED: &str = "ballista.testing.chaos_execution.seed"; +/// Configuration key for the compression codec used in the shuffle write process +/// Valid values are: none, lz4, zstd +pub const BALLISTA_SHUFFLE_COMPRESSION_CODEC: &str = "ballista.shuffle.compression.codec"; /// Result type for configuration parsing operations. pub type ParseResult = result::Result; @@ -334,6 +339,14 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| DataType::Utf8, Some("".to_string()), ), + ConfigEntry::new( + BALLISTA_SHUFFLE_COMPRESSION_CODEC.to_string(), + "Compression codec specification \ + used in the shuffle process. Possible values: \ + none, lz4, zstd. Defaults to lz4 to preserve current behaviour".to_string(), + DataType::Utf8, + Some("lz4".to_string()), + ), ]; entries .into_iter() @@ -585,6 +598,25 @@ impl BallistaConfig { self.get_bool_setting(BALLISTA_PROPAGATE_EMPTY_ENABLED) } + /// Returns compression codec that will be used during write stage of shuffle + pub fn shuffle_compression_codec( + &self, + ) -> std::result::Result, DataFusionError> { + match self + .get_string_setting(BALLISTA_SHUFFLE_COMPRESSION_CODEC) + .to_lowercase() + .trim() + { + "lz4" => Ok(Some(CompressionType::LZ4_FRAME)), + "zstd" => Ok(Some(CompressionType::ZSTD)), + "none" => Ok(None), + other => Err(DataFusionError::Configuration(format!( + "Got an invalid compression codec {other}: \ + Valid options are: lz4, zstd and none" + ))), + } + } + /// Returns the target post-coalesce partition byte size in bytes /// (Spark's `advisoryPartitionSizeInBytes`). pub fn coalesce_target_partition_bytes(&self) -> u64 { diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index 4a99197061..430680b9c7 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -20,9 +20,6 @@ //! partition is re-partitioned and streamed to disk in Arrow IPC format. Future stages of the query //! will use the ShuffleReaderExec to read these results. -use datafusion::arrow::ipc::CompressionType; -use datafusion::arrow::ipc::writer::IpcWriteOptions; - use datafusion::arrow::ipc::writer::StreamWriter; use std::fmt::Debug; use std::fs; @@ -41,6 +38,7 @@ use crate::utils; use crate::serde::protobuf::ShuffleWritePartition; use crate::serde::scheduler::PartitionStats; +use crate::utils::create_write_options; use datafusion::arrow::array::{ ArrayBuilder, ArrayRef, StringBuilder, StructBuilder, UInt32Builder, UInt64Builder, }; @@ -212,10 +210,9 @@ impl ShuffleWriterExec { async move { let now = Instant::now(); - let channel_capacity = context - .session_config() - .ballista_config() - .shuffle_writer_channel_capacity(); + let config = Arc::new(context.session_config().ballista_config()); + let compression_type = config.shuffle_compression_codec()?; + let channel_capacity = config.shuffle_writer_channel_capacity(); let mut stream = plan.execute(input_partition, context)?; match output_partitioning { @@ -240,6 +237,7 @@ impl ShuffleWriterExec { path.as_path(), &write_metrics.write_time, channel_capacity, + compression_type, ) .await .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; @@ -315,10 +313,9 @@ impl ShuffleWriterExec { debug!("Writing results to {p:?}"); - let options = IpcWriteOptions::default() - .try_with_compression(Some( - CompressionType::LZ4_FRAME, - ))?; + let options = + create_write_options(compression_type)?; + let file = BufWriter::new(File::create(p.clone())?); let mut writer = diff --git a/ballista/core/src/execution_plans/sort_shuffle/config.rs b/ballista/core/src/execution_plans/sort_shuffle/config.rs index 8c901f5dc1..5dd4b15095 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/config.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/config.rs @@ -17,15 +17,11 @@ //! Configuration for sort-based shuffle. -use datafusion::arrow::ipc::CompressionType; - /// Configuration for sort-based shuffle. #[derive(Debug, Clone)] pub struct SortShuffleConfig { /// Whether sort-based shuffle is enabled (default: false). pub enabled: bool, - /// Compression codec for shuffle data (default: LZ4_FRAME). - pub compression: CompressionType, /// Target batch size in rows when materializing buffered indices via /// `interleave_record_batch` (default: 8192). pub batch_size: usize, @@ -39,7 +35,6 @@ impl Default for SortShuffleConfig { fn default() -> Self { Self { enabled: false, - compression: CompressionType::LZ4_FRAME, batch_size: 8192, memory_limit_per_task_bytes: 256 * 1024 * 1024, } @@ -48,10 +43,9 @@ impl Default for SortShuffleConfig { impl SortShuffleConfig { /// Creates a new configuration with the default per-task memory limit. - pub fn new(enabled: bool, compression: CompressionType, batch_size: usize) -> Self { + pub fn new(enabled: bool, batch_size: usize) -> Self { Self { enabled, - compression, batch_size, memory_limit_per_task_bytes: Self::default().memory_limit_per_task_bytes, } @@ -72,16 +66,14 @@ mod tests { fn test_default_config() { let config = SortShuffleConfig::default(); assert!(!config.enabled); - assert!(matches!(config.compression, CompressionType::LZ4_FRAME)); assert_eq!(config.batch_size, 8192); assert_eq!(config.memory_limit_per_task_bytes, 256 * 1024 * 1024); } #[test] fn test_new() { - let config = SortShuffleConfig::new(true, CompressionType::LZ4_FRAME, 4096); + let config = SortShuffleConfig::new(true, 4096); assert!(config.enabled); - assert!(matches!(config.compression, CompressionType::LZ4_FRAME)); assert_eq!(config.batch_size, 4096); assert_eq!(config.memory_limit_per_task_bytes, 256 * 1024 * 1024); } diff --git a/ballista/core/src/execution_plans/sort_shuffle/spill.rs b/ballista/core/src/execution_plans/sort_shuffle/spill.rs index c1da095dd7..6f768d86f6 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/spill.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/spill.rs @@ -51,7 +51,7 @@ pub struct SpillManager { /// Active writers per partition, kept open for appending active_writers: HashMap>>, /// Compression codec for spill files - compression: CompressionType, + compression: Option, /// Total number of batches written across all spill files. One call to /// `spill_all_partitions` typically increments this multiple times (once /// per partition that had buffered rows). @@ -78,7 +78,7 @@ impl SpillManager { stage_id: usize, input_partition: usize, schema: SchemaRef, - compression: CompressionType, + compression: Option, ) -> Result { let mut spill_dir = PathBuf::from(work_dir); spill_dir.push(job_id.as_str()); @@ -119,8 +119,8 @@ impl SpillManager { let file = File::create(&spill_path).map_err(BallistaError::IoError)?; let buffered = BufWriter::new(file); - let options = IpcWriteOptions::default() - .try_with_compression(Some(self.compression))?; + let options = + IpcWriteOptions::default().try_with_compression(self.compression)?; let writer = StreamWriter::try_new_with_options(buffered, &self.schema, options)?; @@ -278,7 +278,7 @@ mod tests { 1, 0, schema.clone(), - CompressionType::LZ4_FRAME, + Some(CompressionType::LZ4_FRAME), )?; let b1 = create_test_batch(&schema, vec![1, 2, 3]); @@ -313,7 +313,7 @@ mod tests { 1, 0, schema.clone(), - CompressionType::LZ4_FRAME, + Some(CompressionType::LZ4_FRAME), )?; manager.spill(0, &create_test_batch(&schema, vec![1, 2]))?; @@ -350,7 +350,7 @@ mod tests { 1, 0, schema.clone(), - CompressionType::LZ4_FRAME, + Some(CompressionType::LZ4_FRAME), )?; manager.spill(0, &create_test_batch(&schema, vec![1, 2, 3]))?; @@ -396,7 +396,7 @@ mod tests { 1, 0, schema.clone(), - CompressionType::LZ4_FRAME, + Some(CompressionType::LZ4_FRAME), )?; manager.spill(0, &create_test_batch(&schema, vec![1, 2]))?; diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs b/ballista/core/src/execution_plans/sort_shuffle/writer.rs index 632b5aca07..3e6b310393 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs @@ -36,14 +36,17 @@ use super::partitioned_batch_iterator::PartitionedBatchIterator; use super::spill::SpillManager; use crate::JobId; use crate::execution_plans::create_shuffle_path; +use crate::extension::SessionConfigExt; use crate::serde::protobuf::ShuffleWritePartition; +use crate::utils::create_write_options; use datafusion::arrow::array::{ ArrayBuilder, ArrayRef, StringBuilder, StructBuilder, UInt32Builder, UInt64Builder, }; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::error::ArrowError; -use datafusion::arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use datafusion::arrow::ipc::CompressionType; +use datafusion::arrow::ipc::writer::StreamWriter; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::hash_utils::create_hashes; use datafusion::error::{DataFusionError, Result}; @@ -213,6 +216,8 @@ impl SortShuffleWriterExec { let now = Instant::now(); let mut stream = plan.execute(input_partition, context.clone())?; let schema = stream.schema(); + let ballista_config = Arc::new(context.session_config().ballista_config()); + let compression_type = ballista_config.shuffle_compression_codec()?; let Partitioning::Hash(exprs, num_output_partitions) = partitioning else { return Err(DataFusionError::Internal( @@ -226,7 +231,7 @@ impl SortShuffleWriterExec { stage_id, input_partition, schema.clone(), - config.compression, + compression_type, ) .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; @@ -318,6 +323,7 @@ impl SortShuffleWriterExec { &mut spill_manager, &schema, &config, + compression_type, )?; timer.done(); @@ -425,6 +431,7 @@ fn finalize_output( spill_manager: &mut SpillManager, schema: &SchemaRef, config: &SortShuffleConfig, + compression_type: Option, ) -> Result { let num_partitions = buffered.num_partitions(); let mut index = ShuffleIndex::new(num_partitions); @@ -444,8 +451,7 @@ fn finalize_output( let file = File::create(&data_path)?; let mut output = BufWriter::new(file); - let opts = - IpcWriteOptions::default().try_with_compression(Some(config.compression))?; + let opts = create_write_options(compression_type)?; // Leading schema-header stream (schema message + EOS, no batches) so the // reader can recover the schema even when the requested partition is empty. diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs index 0206afe5e8..9c19585151 100644 --- a/ballista/core/src/serde/mod.rs +++ b/ballista/core/src/serde/mod.rs @@ -423,11 +423,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec { } else { 8192 }; - let config = SortShuffleConfig::new( - true, - datafusion::arrow::ipc::CompressionType::LZ4_FRAME, - batch_size, - ); + let config = SortShuffleConfig::new(true, batch_size); Ok(Arc::new(SortShuffleWriterExec::try_new( sort_shuffle_writer.job_id.clone().into(), diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs index ffe7804c83..34b5b05f10 100644 --- a/ballista/core/src/utils.rs +++ b/ballista/core/src/utils.rs @@ -24,6 +24,7 @@ use datafusion::arrow::ipc::CompressionType; use datafusion::arrow::ipc::writer::IpcWriteOptions; use datafusion::arrow::ipc::writer::StreamWriter; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::error::DataFusionError; use datafusion::execution::context::{SessionConfig, SessionState}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::execution::session_state::SessionStateBuilder; @@ -177,6 +178,17 @@ pub fn default_config_producer() -> SessionConfig { SessionConfig::new_with_ballista() } +/// Creates [IpcWriteOptions] using the compression codec configured in the BallistaConfig +pub fn create_write_options( + compression_type: Option, +) -> std::result::Result { + IpcWriteOptions::default() + .try_with_compression(compression_type) + .map_err(|err| { + DataFusionError::Internal(format!("Failed to set compression codec: {err}")) + }) +} + /// Stream data to disk in Arrow IPC format. /// /// Batches are read from the async stream and forwarded through a bounded @@ -187,6 +199,7 @@ pub async fn write_stream_to_disk( path: &Path, disk_write_metric: &metrics::Time, channel_capacity: usize, + compression_type: Option, ) -> Result { let schema = stream.schema(); let path_owned = path.to_owned(); @@ -200,8 +213,7 @@ pub async fn write_stream_to_disk( BallistaError::IoError(e) })?); - let options = IpcWriteOptions::default() - .try_with_compression(Some(CompressionType::LZ4_FRAME))?; + let options = create_write_options(compression_type)?; let mut writer = StreamWriter::try_new_with_options(file, schema.as_ref(), options)?; diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 97cc1a7ac9..4448a8eb12 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -691,7 +691,6 @@ pub(crate) fn create_shuffle_writer_with_config( if let Some(Partitioning::Hash(exprs, partition_count)) = partitioning { let sort_config = SortShuffleConfig::new( true, - datafusion::arrow::ipc::CompressionType::LZ4_FRAME, ballista_config.shuffle_sort_based_batch_size(), ) .with_memory_limit_per_task_bytes( diff --git a/benchmarks/benches/sort_shuffle.rs b/benchmarks/benches/sort_shuffle.rs index 749b0f925a..3e135551fb 100644 --- a/benchmarks/benches/sort_shuffle.rs +++ b/benchmarks/benches/sort_shuffle.rs @@ -27,7 +27,6 @@ use datafusion::arrow::array::{ Int64Array, StringArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, }; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use datafusion::arrow::ipc::CompressionType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; @@ -236,7 +235,7 @@ fn run_sort_shuffle( }; let task_ctx = session_ctx.task_ctx(); - let config = SortShuffleConfig::new(true, CompressionType::LZ4_FRAME, 8192); + let config = SortShuffleConfig::new(true, 8192); let writer = SortShuffleWriterExec::try_new( "bench_job".into(), diff --git a/benchmarks/src/bin/shuffle_bench.rs b/benchmarks/src/bin/shuffle_bench.rs index 934f80a236..c53de9eaa9 100644 --- a/benchmarks/src/bin/shuffle_bench.rs +++ b/benchmarks/src/bin/shuffle_bench.rs @@ -43,7 +43,6 @@ use ballista_core::execution_plans::sort_shuffle::{ use ballista_core::utils; use clap::Parser; use datafusion::arrow::datatypes::{DataType, SchemaRef}; -use datafusion::arrow::ipc::CompressionType; use datafusion::execution::config::SessionConfig; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; @@ -286,8 +285,7 @@ async fn execute_shuffle_write( exec.metrics().unwrap_or_default() } WriterKind::Sort => { - let cfg = - SortShuffleConfig::new(true, CompressionType::LZ4_FRAME, args.batch_size); + let cfg = SortShuffleConfig::new(true, args.batch_size); let exec = SortShuffleWriterExec::try_new( format!("bench_job_{task_id}").into(), 1, From f45996cfbef6589ddafb965ffe81f21163f9ef20 Mon Sep 17 00:00:00 2001 From: Phillip LeBlanc <879445+phillipleblanc@users.noreply.github.com> Date: Thu, 16 Jul 2026 07:31:51 +0900 Subject: [PATCH 078/107] fix(scheduler): persist terminal status before cache eviction (#2037) Keep successful and aborted jobs in the active cache until terminal-state persistence completes, preventing status readers from observing stale state. Snapshot graphs before asynchronous work so graph locks are not held across persistence. Leave retry policy to JobState implementations because generic saves are not guaranteed to be idempotent. Exclude terminal entries from running-job metrics and evict after a failed save so shared state remains authoritative. Dispatch executor task cancellation before abort persistence so save failures cannot suppress cancellation. Add deterministic coverage for success, save failure, lock availability, metrics, and abort cancellation when persistence fails. --- ballista/scheduler/src/cluster/mod.rs | 3 + .../scheduler_server/query_stage_scheduler.rs | 50 +- ballista/scheduler/src/state/task_manager.rs | 494 ++++++++++++++++-- 3 files changed, 477 insertions(+), 70 deletions(-) diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index 6bd474f9a6..e59fe5d43f 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -317,6 +317,9 @@ pub trait JobState: Send + Sync { /// Persists the current state of an owned job. /// + /// Implementations are responsible for applying any backend-specific retry policy. + /// Callers must not assume that this operation is safe to retry. + /// /// Returns an error if the job is not owned by the caller. async fn save_job(&self, job_id: &JobId, graph: &ExecutionGraphBox) -> Result<()>; diff --git a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs index cd8e21db05..fe054aca85 100644 --- a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs +++ b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use std::time::Duration; +use ballista_core::JobId; use ballista_core::serde::protobuf::{FailedJob, JobStatus}; use log::{debug, error, info, trace, warn}; @@ -58,6 +59,22 @@ impl QueryStageSchedul config, } } + + async fn abort_job(&self, job_id: &JobId, failure_reason: String) -> Result<()> { + let executor_manager = self.state.executor_manager.clone(); + self.state + .task_manager + .abort_job(job_id, failure_reason, move |running_tasks| async move { + if running_tasks.is_empty() { + Ok(()) + } else { + executor_manager.cancel_running_tasks(running_tasks).await + } + }) + .await?; + Ok(()) + } + #[cfg(feature = "rest-api")] pub(crate) fn metrics_collector(&self) -> &dyn SchedulerMetricsCollector { self.metrics_collector.as_ref() @@ -234,24 +251,8 @@ impl .record_failed(&job_id, queued_at, failed_at); error!("Job failed: [{job_id}]"); - match self - .state - .task_manager - .abort_job(&job_id, fail_message) - .await - { - Ok((running_tasks, _pending_tasks)) => { - if !running_tasks.is_empty() { - event_sender - .post_event(QueryStageSchedulerEvent::CancelTasks( - running_tasks, - )) - .await?; - } - } - Err(e) => { - error!("Fail to invoke abort_job for job {job_id} due to {e:?}"); - } + if let Err(e) = self.abort_job(&job_id, fail_message).await { + error!("Fail to abort job {job_id} due to {e:?}"); } self.state.clean_up_failed_job(job_id); } @@ -265,17 +266,8 @@ impl self.metrics_collector.record_cancelled(&job_id); info!("Job cancelled: [{job_id}]"); - match self.state.task_manager.cancel_job(&job_id).await { - Ok((running_tasks, _pending_tasks)) => { - event_sender - .post_event(QueryStageSchedulerEvent::CancelTasks( - running_tasks, - )) - .await?; - } - Err(e) => { - error!("Fail to invoke cancel_job for job {job_id} due to {e:?}"); - } + if let Err(e) = self.abort_job(&job_id, "Cancelled".to_owned()).await { + error!("Fail to cancel job {job_id} due to {e:?}"); } self.state.clean_up_failed_job(job_id); } diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index 57bbd797e9..9285a2957a 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -52,6 +52,7 @@ use log::{debug, error, info, trace, warn}; use rand::distr::Alphanumeric; use rand::{RngExt, rng}; use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::ops::Deref; use std::sync::Arc; use std::time::Duration; @@ -341,7 +342,12 @@ impl TaskManager /// Get the number of running jobs. pub fn running_job_number(&self) -> usize { - self.active_job_cache.len() + self.active_job_cache + .iter() + .filter(|job_info| { + matches!(job_info.status, Some(job_status::Status::Running(_))) + }) + .count() } /// Generate an ExecutionGraph for the job and save it to the persistent state. @@ -655,67 +661,101 @@ impl TaskManager Ok(events) } + /// Save a terminal job snapshot and remove it from the active cache. + pub(crate) async fn persist_terminal_and_evict( + &self, + job_id: &JobId, + snapshot: &ExecutionGraphBox, + ) -> Result<()> { + if let Some(mut job_info) = self.active_job_cache.get_mut(job_id) { + job_info.status = snapshot.status().status.clone(); + } + + let save_result = self.state.save_job(job_id, snapshot).await; + if let Err(error) = &save_result { + warn!( + "Failed to persist terminal state for job {job_id}; evicting the local cache entry: {error}" + ); + } + let _ = self.remove_active_execution_graph(job_id); + save_result + } + /// Move a job from Active to Success and return the ids of its intermediate /// (non-final) stages so their shuffle data can be reclaimed immediately. /// Returns an empty vec if the job is not found or not successful. pub(crate) async fn succeed_job(&self, job_id: &JobId) -> Result> { debug!("Moving job {job_id} from Active to Success"); - if let Some(graph) = self.remove_active_execution_graph(job_id) { - let graph = graph.read().await; - if graph.is_successful() { - self.state.save_job(job_id, &graph).await?; - Ok(graph.intermediate_stage_ids()) - } else { - error!("Job {job_id} has not finished and cannot be completed"); - Ok(vec![]) - } + if let Some(graph) = self.get_active_execution_graph(job_id) { + let (snapshot, intermediate_stage_ids) = { + let graph = graph.read().await; + if !graph.is_successful() { + error!("Job {job_id} has not finished and cannot be completed"); + return Ok(vec![]); + } + + (graph.cloned(), graph.intermediate_stage_ids()) + }; + + self.persist_terminal_and_evict(job_id, &snapshot).await?; + Ok(intermediate_stage_ids) } else { warn!("Fail to find job {job_id} in the cache"); Ok(vec![]) } } - /// Cancel the job and return a Vec of running tasks need to cancel - pub(crate) async fn cancel_job( - &self, - job_id: &JobId, - ) -> Result<(Vec, usize)> { - self.abort_job(job_id, "Cancelled".to_owned()).await - } - - /// Abort the job and return a Vec of running tasks need to cancel - pub(crate) async fn abort_job( + /// Mark a job as failed, cancel its running tasks, and persist terminal state. + /// + /// Task cancellation is invoked before persistence so executor work is + /// stopped even when all terminal-state save attempts fail. + pub(crate) async fn abort_job( &self, job_id: &JobId, failure_reason: String, - ) -> Result<(Vec, usize)> { - let (tasks_to_cancel, pending_tasks) = if let Some(graph) = - self.remove_active_execution_graph(job_id) - { - let mut guard = graph.write().await; - - let pending_tasks = guard.available_tasks(); - let running_tasks = guard.abort_running(failure_reason); - - info!( - "Cancelling {} running tasks for job {}", - running_tasks.len(), - job_id - ); - - self.state.save_job(job_id, &guard).await?; - - (running_tasks, pending_tasks) - } else { + cancel_tasks: F, + ) -> Result + where + F: FnOnce(Vec) -> Fut, + Fut: Future>, + { + let Some(graph) = self.get_active_execution_graph(job_id) else { // TODO listen the job state update event and fix task cancelling warn!( "Fail to find job {job_id} in the cache, unable to cancel tasks for job, fail the job state only." ); - (vec![], 0) + return Ok(0); }; - Ok((tasks_to_cancel, pending_tasks)) + let (running_tasks, pending_tasks, snapshot) = { + let mut guard = graph.write().await; + let pending_tasks = guard.available_tasks(); + let running_tasks = guard.abort_running(failure_reason); + let snapshot = guard.cloned(); + if let Some(mut job_info) = self.active_job_cache.get_mut(job_id) { + job_info.status = snapshot.status().status.clone(); + } + (running_tasks, pending_tasks, snapshot) + }; + + info!( + "Cancelling {} running tasks for job {}", + running_tasks.len(), + job_id + ); + + let cancel_result = cancel_tasks(running_tasks).await; + let persist_result = self.persist_terminal_and_evict(job_id, &snapshot).await; + match (cancel_result, persist_result) { + (Ok(()), Ok(())) => Ok(pending_tasks), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(cancel_error), Err(persist_error)) => { + Err(BallistaError::General(format!( + "Failed to cancel tasks for job {job_id}: {cancel_error}; failed to persist terminal state: {persist_error}" + ))) + } + } } /// Mark a unscheduled job as failed. This will create a key under the FailedJobs keyspace @@ -1016,6 +1056,378 @@ impl From<&ExecutionGraphBox> for JobOverview { } } +#[cfg(test)] +mod tests { + use super::*; + use crate::cluster::JobStateEventStream; + use crate::cluster::memory::InMemoryJobState; + use crate::test_utils::{mock_completed_task, mock_executor, test_aggregation_plan}; + use ballista_core::serde::protobuf::job_status::Status; + use ballista_core::utils::{default_config_producer, default_session_builder}; + use datafusion_proto::protobuf::LogicalPlanNode; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::Notify; + use tokio::time::timeout; + + struct BlockingJobState { + inner: InMemoryJobState, + save_started: Notify, + allow_save: Notify, + failures_remaining: AtomicUsize, + save_attempts: AtomicUsize, + } + + impl BlockingJobState { + fn new() -> Self { + Self { + inner: InMemoryJobState::new( + "test-scheduler", + Arc::new(default_session_builder), + Arc::new(default_config_producer), + ), + save_started: Notify::new(), + allow_save: Notify::new(), + failures_remaining: AtomicUsize::new(0), + save_attempts: AtomicUsize::new(0), + } + } + } + + #[async_trait::async_trait] + impl JobState for BlockingJobState { + fn accept_job( + &self, + job_id: &JobId, + job_name: &str, + queued_at: u64, + ) -> Result<()> { + self.inner.accept_job(job_id, job_name, queued_at) + } + + fn pending_job_number(&self) -> usize { + self.inner.pending_job_number() + } + + async fn submit_job( + &self, + job_id: JobId, + graph: &ExecutionGraphBox, + subscriber: Option, + ) -> Result<()> { + self.inner.submit_job(job_id, graph, subscriber).await + } + + async fn get_jobs(&self) -> Result> { + self.inner.get_jobs().await + } + + async fn get_all_jobs(&self) -> Result> { + self.inner.get_all_jobs().await + } + + async fn get_job_status(&self, job_id: &JobId) -> Result> { + self.inner.get_job_status(job_id).await + } + + async fn get_execution_graph( + &self, + job_id: &JobId, + ) -> Result> { + self.inner.get_execution_graph(job_id).await + } + + async fn save_job( + &self, + job_id: &JobId, + graph: &ExecutionGraphBox, + ) -> Result<()> { + let attempt = self.save_attempts.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + self.save_started.notify_one(); + self.allow_save.notified().await; + } + if self + .failures_remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(BallistaError::General("injected save failure".to_string())); + } + self.inner.save_job(job_id, graph).await + } + + async fn fail_unscheduled_job( + &self, + job_id: &JobId, + reason: String, + ) -> Result<()> { + self.inner.fail_unscheduled_job(job_id, reason).await + } + + async fn remove_job(&self, job_id: &JobId) -> Result<()> { + self.inner.remove_job(job_id).await + } + + async fn try_acquire_job( + &self, + job_id: &JobId, + ) -> Result> { + self.inner.try_acquire_job(job_id).await + } + + async fn job_state_events(&self) -> Result { + self.inner.job_state_events().await + } + + async fn create_or_update_session( + &self, + session_id: &str, + config: &SessionConfig, + ) -> Result> { + self.inner + .create_or_update_session(session_id, config) + .await + } + + async fn remove_session(&self, session_id: &str) -> Result<()> { + self.inner.remove_session(session_id).await + } + + fn produce_config(&self) -> SessionConfig { + self.inner.produce_config() + } + } + + type TestTaskManager = TaskManager; + + async fn setup_job( + successful: bool, + ) -> Result<( + TestTaskManager, + Arc, + JobId, + Arc>, + )> { + let state = Arc::new(BlockingJobState::new()); + let job_state: Arc = state.clone(); + let manager = TaskManager::new( + job_state, + BallistaCodec::default(), + "test-scheduler".to_string(), + Arc::new(SchedulerConfig::default()), + ); + + let graph: ExecutionGraphBox = Box::new(test_aggregation_plan(2).await); + let job_id = graph.job_id().to_owned(); + state.accept_job(&job_id, graph.job_name(), 0)?; + state.submit_job(job_id.clone(), &graph, None).await?; + + manager + .active_job_cache + .insert(job_id.clone(), JobInfoCache::new(graph)); + let active_graph = manager + .get_active_execution_graph(&job_id) + .expect("job should be active"); + + if successful { + // Complete the graph after it is cached. `JobInfoCache::status` still + // contains Running until terminal persistence updates it. + let mut graph = active_graph.write().await; + let executor = mock_executor("executor-1".to_string()); + while let Some(task) = graph.pop_next_task(&executor.id)? { + graph.update_task_status( + &executor, + vec![mock_completed_task(task, &executor.id)], + 1, + 1, + )?; + } + graph.succeed_job()?; + } + + Ok((manager, state, job_id, active_graph)) + } + + #[tokio::test] + async fn successful_job_stays_visible_and_unlocked_until_persisted() -> Result<()> { + let (manager, state, job_id, active_graph) = setup_job(true).await?; + let manager_for_save = manager.clone(); + let job_id_for_save = job_id.clone(); + let save = + tokio::spawn( + async move { manager_for_save.succeed_job(&job_id_for_save).await }, + ); + + timeout(Duration::from_secs(1), state.save_started.notified()) + .await + .expect("terminal save should start"); + + let status = manager + .get_job_status(&job_id) + .await? + .expect("successful job should remain visible during persistence"); + assert!(matches!(status.status, Some(Status::Successful(_)))); + assert!(!manager.get_running_job_cache().contains_key(&job_id)); + assert_eq!(manager.running_job_number(), 0); + + let graph_guard = timeout(Duration::from_secs(1), active_graph.write()) + .await + .expect("graph lock must not be held across persistence"); + drop(graph_guard); + + state.allow_save.notify_one(); + save.await.expect("succeed_job task should not panic")?; + + assert!(manager.get_active_execution_graph(&job_id).is_none()); + let persisted = state + .get_job_status(&job_id) + .await? + .expect("successful status should be persisted"); + assert!(matches!(persisted.status, Some(Status::Successful(_)))); + Ok(()) + } + + #[tokio::test] + async fn failed_terminal_save_is_not_retried_and_evicts_local_cache() -> Result<()> { + let (manager, state, job_id, active_graph) = setup_job(true).await?; + state.failures_remaining.store(1, Ordering::SeqCst); + let manager_for_save = manager.clone(); + let job_id_for_save = job_id.clone(); + let save = + tokio::spawn( + async move { manager_for_save.succeed_job(&job_id_for_save).await }, + ); + + timeout(Duration::from_secs(1), state.save_started.notified()) + .await + .expect("terminal save should start"); + assert!(manager.get_active_execution_graph(&job_id).is_some()); + assert_eq!(manager.running_job_number(), 0); + state.allow_save.notify_one(); + assert!( + save.await + .expect("succeed_job task should not panic") + .is_err() + ); + + assert_eq!(state.save_attempts.load(Ordering::SeqCst), 1); + assert!(manager.get_active_execution_graph(&job_id).is_none()); + assert_eq!(manager.running_job_number(), 0); + let authoritative_status = manager + .get_job_status(&job_id) + .await? + .expect("shared status should remain available after local eviction"); + assert!(matches!( + authoritative_status.status, + Some(Status::Running(_)) + )); + + let graph_guard = timeout(Duration::from_secs(1), active_graph.write()) + .await + .expect("graph lock must be released after a failed save"); + drop(graph_guard); + Ok(()) + } + + #[tokio::test] + async fn aborted_tasks_are_cancelled_before_terminal_persistence() -> Result<()> { + let (manager, state, job_id, active_graph) = setup_job(false).await?; + let executor = mock_executor("executor-1".to_string()); + active_graph + .write() + .await + .pop_next_task(&executor.id)? + .expect("job should have a task to assign"); + + let cancelled_tasks = Arc::new(AtomicUsize::new(0)); + let cancelled_tasks_for_abort = cancelled_tasks.clone(); + let manager_for_abort = manager.clone(); + let job_id_for_abort = job_id.clone(); + let abort = tokio::spawn(async move { + manager_for_abort + .abort_job( + &job_id_for_abort, + "test failure".to_string(), + move |tasks| async move { + cancelled_tasks_for_abort.store(tasks.len(), Ordering::SeqCst); + Ok(()) + }, + ) + .await + }); + + timeout(Duration::from_secs(1), state.save_started.notified()) + .await + .expect("terminal save should start"); + assert_eq!(cancelled_tasks.load(Ordering::SeqCst), 1); + assert_eq!(state.save_attempts.load(Ordering::SeqCst), 1); + assert_eq!(manager.running_job_number(), 0); + + let status = manager + .get_job_status(&job_id) + .await? + .expect("failed job should remain visible during persistence"); + assert!(matches!(status.status, Some(Status::Failed(_)))); + let graph_guard = timeout(Duration::from_secs(1), active_graph.write()) + .await + .expect("graph lock must not be held across persistence"); + drop(graph_guard); + + state.allow_save.notify_one(); + abort.await.expect("abort_job task should not panic")?; + assert!(manager.get_active_execution_graph(&job_id).is_none()); + Ok(()) + } + + #[tokio::test] + async fn aborted_tasks_are_cancelled_when_terminal_save_fails() -> Result<()> { + let (manager, state, job_id, active_graph) = setup_job(false).await?; + let executor = mock_executor("executor-1".to_string()); + active_graph + .write() + .await + .pop_next_task(&executor.id)? + .expect("job should have a task to assign"); + state.failures_remaining.store(1, Ordering::SeqCst); + + let cancelled_tasks = Arc::new(AtomicUsize::new(0)); + let cancelled_tasks_for_abort = cancelled_tasks.clone(); + let manager_for_abort = manager.clone(); + let job_id_for_abort = job_id.clone(); + let abort = tokio::spawn(async move { + manager_for_abort + .abort_job( + &job_id_for_abort, + "test failure".to_string(), + move |tasks| async move { + cancelled_tasks_for_abort.store(tasks.len(), Ordering::SeqCst); + Ok(()) + }, + ) + .await + }); + timeout(Duration::from_secs(1), state.save_started.notified()) + .await + .expect("terminal save should start"); + assert_eq!(cancelled_tasks.load(Ordering::SeqCst), 1); + state.allow_save.notify_one(); + assert!( + abort + .await + .expect("abort_job task should not panic") + .is_err() + ); + + assert_eq!(cancelled_tasks.load(Ordering::SeqCst), 1); + assert_eq!(state.save_attempts.load(Ordering::SeqCst), 1); + assert!(manager.get_active_execution_graph(&job_id).is_none()); + assert_eq!(manager.running_job_number(), 0); + Ok(()) + } +} + #[cfg(all(test, feature = "disable-stage-plan-cache"))] mod prune_partition_tests { use crate::state::task_manager::JobInfoCache; From 30222651c11c2a9c042b053c83b693716f455759 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 08:26:52 +0100 Subject: [PATCH 079/107] chore(deps): bump uuid from 1.23.5 to 1.24.0 (#2054) --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fca0e4a209..be77bffab6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6929,7 +6929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -7748,9 +7748,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.5" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "atomic", "getrandom 0.4.2", From dc91baedb1ec80ade248884ffda7f72452467367 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 08:27:20 +0100 Subject: [PATCH 080/107] chore(deps): bump regex from 1.13.0 to 1.13.1 (#2053) --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be77bffab6..ab1ae2a2dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5931,9 +5931,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -5943,9 +5943,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", From 524f0842db6b43ec03569f3c2ccfe70bca569f92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 08:27:48 +0100 Subject: [PATCH 081/107] chore(deps): bump uuid from 1.23.5 to 1.24.0 in /python (#2052) --- python/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Cargo.lock b/python/Cargo.lock index 530c8cbb05..1abb188796 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4353,9 +4353,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.5" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.2", "js-sys", From 713599866f529834a56084cd4eca26a008539570 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 08:28:13 +0100 Subject: [PATCH 082/107] chore(deps): bump ctor from 1.0.8 to 1.0.9 (#2051) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab1ae2a2dd..bde0f41d7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2102,9 +2102,9 @@ dependencies = [ [[package]] name = "ctor" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb" +checksum = "a394189d59f9befacce833f337f7b1eca5e9a91221bcdd4d28e0114d96e597b3" dependencies = [ "link-section", "linktime-proc-macro", From b758d35e771071b05ebaec0b5514f90708e3b3b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 08:28:52 +0100 Subject: [PATCH 083/107] chore(deps): bump clap from 4.6.1 to 4.6.2 (#2050) --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bde0f41d7d..7e45d5831f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -123,7 +123,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1027,7 +1027,7 @@ version = "54.0.0" dependencies = [ "ballista", "ballista-core", - "clap 4.6.1", + "clap 4.6.2", "criterion", "datafusion", "datafusion-proto", @@ -1048,7 +1048,7 @@ version = "54.0.0" dependencies = [ "ballista", "chrono", - "clap 4.6.1", + "clap 4.6.2", "config", "console_error_panic_hook", "critical-section", @@ -1090,7 +1090,7 @@ dependencies = [ "aws-config", "aws-credential-types", "chrono", - "clap 4.6.1", + "clap 4.6.2", "datafusion", "datafusion-proto", "datafusion-proto-common", @@ -1152,7 +1152,7 @@ dependencies = [ "async-trait", "ballista-core", "bytesize", - "clap 4.6.1", + "clap 4.6.2", "dashmap", "datafusion", "datafusion-proto", @@ -1184,7 +1184,7 @@ dependencies = [ "async-trait", "axum", "ballista-core", - "clap 4.6.1", + "clap 4.6.2", "dashmap", "datafusion", "datafusion-proto", @@ -1657,9 +1657,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" dependencies = [ "clap_builder", "clap_derive", @@ -1667,9 +1667,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1933,7 +1933,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.6.1", + "clap 4.6.2", "criterion-plot", "itertools 0.13.0", "num-traits", @@ -2315,7 +2315,7 @@ dependencies = [ "aws-config", "aws-credential-types", "chrono", - "clap 4.6.1", + "clap 4.6.2", "datafusion", "datafusion-common", "dirs", @@ -3107,7 +3107,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3242,7 +3242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4715,7 +4715,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6149,7 +6149,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6208,7 +6208,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6677,7 +6677,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6718,7 +6718,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6932,7 +6932,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6945,7 +6945,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8056,7 +8056,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 4974e678c6438f6ae4294c0013b5b8c3aa2683ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:42:30 +0300 Subject: [PATCH 084/107] chore(deps): bump cryptography from 46.0.3 to 48.0.1 in /python (#2056) Bumps [cryptography](https://github.com/pyca/cryptography) from 46.0.3 to 48.0.1. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/46.0.3...48.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-version: 48.0.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/uv.lock | 107 +++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/python/uv.lock b/python/uv.lock index 9e5c92d547..d6d0872be9 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -270,67 +270,62 @@ wheels = [ [[package]] name = "cryptography" -version = "46.0.3" +version = "48.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/42/9c391dd801d6cf0d561b5890549d4b27bafcc53b39c31a817e69d87c625b/cryptography-46.0.3-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a", size = 7225004, upload-time = "2025-10-15T23:16:52.239Z" }, - { url = "https://files.pythonhosted.org/packages/1c/67/38769ca6b65f07461eb200e85fc1639b438bdc667be02cf7f2cd6a64601c/cryptography-46.0.3-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc", size = 4296667, upload-time = "2025-10-15T23:16:54.369Z" }, - { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" }, - { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" }, - { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" }, - { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" }, - { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" }, - { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" }, - { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" }, - { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" }, - { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" }, - { url = "https://files.pythonhosted.org/packages/96/92/8a6a9525893325fc057a01f654d7efc2c64b9de90413adcf605a85744ff4/cryptography-46.0.3-cp311-abi3-win32.whl", hash = "sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018", size = 3055988, upload-time = "2025-10-15T23:17:14.65Z" }, - { url = "https://files.pythonhosted.org/packages/7e/bf/80fbf45253ea585a1e492a6a17efcb93467701fa79e71550a430c5e60df0/cryptography-46.0.3-cp311-abi3-win_amd64.whl", hash = "sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb", size = 3514451, upload-time = "2025-10-15T23:17:16.142Z" }, - { url = "https://files.pythonhosted.org/packages/2e/af/9b302da4c87b0beb9db4e756386a7c6c5b8003cd0e742277888d352ae91d/cryptography-46.0.3-cp311-abi3-win_arm64.whl", hash = "sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c", size = 2928007, upload-time = "2025-10-15T23:17:18.04Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e2/a510aa736755bffa9d2f75029c229111a1d02f8ecd5de03078f4c18d91a3/cryptography-46.0.3-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:00a5e7e87938e5ff9ff5447ab086a5706a957137e6e433841e9d24f38a065217", size = 7158012, upload-time = "2025-10-15T23:17:19.982Z" }, - { url = "https://files.pythonhosted.org/packages/73/dc/9aa866fbdbb95b02e7f9d086f1fccfeebf8953509b87e3f28fff927ff8a0/cryptography-46.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8daeb2d2174beb4575b77482320303f3d39b8e81153da4f0fb08eb5fe86a6c5", size = 4288728, upload-time = "2025-10-15T23:17:21.527Z" }, - { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" }, - { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" }, - { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" }, - { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" }, - { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" }, - { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" }, - { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" }, - { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" }, - { url = "https://files.pythonhosted.org/packages/fd/30/27654c1dbaf7e4a3531fa1fc77986d04aefa4d6d78259a62c9dc13d7ad36/cryptography-46.0.3-cp314-cp314t-win32.whl", hash = "sha256:8a6e050cb6164d3f830453754094c086ff2d0b2f3a897a1d9820f6139a1f0914", size = 3022339, upload-time = "2025-10-15T23:17:40.888Z" }, - { url = "https://files.pythonhosted.org/packages/f6/30/640f34ccd4d2a1bc88367b54b926b781b5a018d65f404d409aba76a84b1c/cryptography-46.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:760f83faa07f8b64e9c33fc963d790a2edb24efb479e3520c14a45741cd9b2db", size = 3494315, upload-time = "2025-10-15T23:17:42.769Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8b/88cc7e3bd0a8e7b861f26981f7b820e1f46aa9d26cc482d0feba0ecb4919/cryptography-46.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:516ea134e703e9fe26bcd1277a4b59ad30586ea90c365a87781d7887a646fe21", size = 2919331, upload-time = "2025-10-15T23:17:44.468Z" }, - { url = "https://files.pythonhosted.org/packages/fd/23/45fe7f376a7df8daf6da3556603b36f53475a99ce4faacb6ba2cf3d82021/cryptography-46.0.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936", size = 7218248, upload-time = "2025-10-15T23:17:46.294Z" }, - { url = "https://files.pythonhosted.org/packages/27/32/b68d27471372737054cbd34c84981f9edbc24fe67ca225d389799614e27f/cryptography-46.0.3-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683", size = 4294089, upload-time = "2025-10-15T23:17:48.269Z" }, - { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" }, - { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" }, - { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" }, - { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" }, - { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" }, - { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" }, - { url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" }, - { url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" }, - { url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cd/1a8633802d766a0fa46f382a77e096d7e209e0817892929655fe0586ae32/cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32", size = 3689163, upload-time = "2025-10-15T23:18:13.821Z" }, - { url = "https://files.pythonhosted.org/packages/4c/59/6b26512964ace6480c3e54681a9859c974172fb141c38df11eadd8416947/cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c", size = 3429474, upload-time = "2025-10-15T23:18:15.477Z" }, - { url = "https://files.pythonhosted.org/packages/06/8a/e60e46adab4362a682cf142c7dcb5bf79b782ab2199b0dcb81f55970807f/cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea", size = 3698132, upload-time = "2025-10-15T23:18:17.056Z" }, - { url = "https://files.pythonhosted.org/packages/da/38/f59940ec4ee91e93d3311f7532671a5cef5570eb04a144bf203b58552d11/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b", size = 4243992, upload-time = "2025-10-15T23:18:18.695Z" }, - { url = "https://files.pythonhosted.org/packages/b0/0c/35b3d92ddebfdfda76bb485738306545817253d0a3ded0bfe80ef8e67aa5/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb", size = 4409944, upload-time = "2025-10-15T23:18:20.597Z" }, - { url = "https://files.pythonhosted.org/packages/99/55/181022996c4063fc0e7666a47049a1ca705abb9c8a13830f074edb347495/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:9394673a9f4de09e28b5356e7fff97d778f8abad85c9d5ac4a4b7e25a0de7717", size = 4242957, upload-time = "2025-10-15T23:18:22.18Z" }, - { url = "https://files.pythonhosted.org/packages/ba/af/72cd6ef29f9c5f731251acadaeb821559fe25f10852f44a63374c9ca08c1/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:94cd0549accc38d1494e1f8de71eca837d0509d0d44bf11d158524b0e12cebf9", size = 4409447, upload-time = "2025-10-15T23:18:24.209Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c3/e90f4a4feae6410f914f8ebac129b9ae7a8c92eb60a638012dde42030a9d/cryptography-46.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6b5063083824e5509fdba180721d55909ffacccc8adbec85268b48439423d78c", size = 3438528, upload-time = "2025-10-15T23:18:26.227Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, + { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, + { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, + { url = "https://files.pythonhosted.org/packages/89/c6/24266ac10c47f6cd2a865f4446062b466da1d1f10b27189eac00e61bf0c9/cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", size = 5300085, upload-time = "2026-06-09T22:31:58.703Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bb/cc4b78784f97efc8c5874c2a9743708d172be6663024b34a0467885ae0c8/cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", size = 4746137, upload-time = "2026-06-09T22:31:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/1f/52/0c44de3f5267f8fbe8e835138017522a333436166e406f0db9b9e6e3033f/cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", size = 4333867, upload-time = "2026-06-09T22:32:28.096Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/772d7adbfa931537bc401640b7cac9976bff689bda187833e5d63b428e49/cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", size = 4701805, upload-time = "2026-06-09T22:31:38.284Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a3/b06844f303873493c963caf581c04df31c7035e0c1b0f02c4814d319ec80/cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", size = 5258461, upload-time = "2026-06-09T22:31:04.187Z" }, + { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, + { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, + { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" }, + { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" }, + { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" }, + { url = "https://files.pythonhosted.org/packages/24/dc/d8ca05ffea724eec6d232ea6f18e74c269eb6bdfdcc9bfba689790d1325f/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:e361afba8918070d376df76f408a4f67fec0ee9cff81a99e48fe9a233ef59e17", size = 5290385, upload-time = "2026-06-09T22:31:15.212Z" }, + { url = "https://files.pythonhosted.org/packages/03/8c/3be6cb4da181f5bb6c19cf560c2359d60644a6b5fc5b57854e528f47b296/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d069066deead00ac7f090be101be875a06855908f7ec004c27b8fefb4acfb411", size = 4737082, upload-time = "2026-06-09T22:32:22.66Z" }, + { url = "https://files.pythonhosted.org/packages/aa/f6/d5f60a5a1434dbfd949e227fd0065d194c7e6b6ac526b17f5c06152b8231/cryptography-48.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:09f73a725d582cef64b91281a322cd798d14a33b2b6f2b7ad9531dc336d84c02", size = 4325328, upload-time = "2026-06-09T22:32:10.777Z" }, + { url = "https://files.pythonhosted.org/packages/17/b7/ba75dd947a14b6ad907b01ae8f6b5b348cdd1b48142f0063dee9e20c1d9d/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:15254441469dd6bf027039453288e2072124f8b6603563f5d759e1c9b69273fa", size = 4694530, upload-time = "2026-06-09T22:31:53.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/29/50d6b9e8aff12d8b67afaeb3569335e32dc83a5723e3bbded24fdac9f809/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:8ace4507d1e6533c125f4fac754f8bb8b6a74c08e92179dabd7e16571a3efbf3", size = 5245046, upload-time = "2026-06-09T22:31:25.774Z" }, + { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" }, + { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" }, + { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" }, + { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, + { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, + { url = "https://files.pythonhosted.org/packages/e5/0b/aa68b221dde92d09cb29a024ede17550ee21e77a404e59fc093c82bb51e1/cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", size = 5289970, upload-time = "2026-06-09T22:31:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/78/13/fba657f958d2af66ea959a4ba01212632089249d34af1ae48054136344d7/cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", size = 4731873, upload-time = "2026-06-09T22:31:22.253Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4c/9a964756d24a26b3e34dfcb16f961b89838786e6700b635b0d1e3adff4b6/cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", size = 4330804, upload-time = "2026-06-09T22:31:36.56Z" }, + { url = "https://files.pythonhosted.org/packages/4b/0f/a10f3a6eb12950a10e3a874070283aa2dd5875b2bfd15fad8a3e17b3f13e/cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46", size = 4696217, upload-time = "2026-06-09T22:31:13.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6f/5cd12f951165ea73ef85266775d97e4c763b2474ccfd816dd69d3a18d6f8/cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", size = 5245252, upload-time = "2026-06-09T22:32:02.193Z" }, + { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, + { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, + { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, + { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, + { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, + { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, ] [[package]] From 48b08b8847f576cf2e4b6b366d0eadb2e8d7e2c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:42:42 +0300 Subject: [PATCH 085/107] chore(deps): bump pyjwt from 2.12.0 to 2.13.0 in /python (#2055) Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.12.0 to 2.13.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.12.0...2.13.0) --- updated-dependencies: - dependency-name: pyjwt dependency-version: 2.13.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- python/uv.lock | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/uv.lock b/python/uv.lock index d6d0872be9..54493c14b0 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -685,11 +685,14 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.12.0" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a8/10/e8192be5f38f3e8e7e046716de4cae33d56fd5ae08927a823bb916be36c1/pyjwt-2.12.0.tar.gz", hash = "sha256:2f62390b667cd8257de560b850bb5a883102a388829274147f1d724453f8fb02", size = 102511, upload-time = "2026-03-12T17:15:30.831Z" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/70/70f895f404d363d291dcf62c12c85fdd47619ad9674ac0f53364d035925a/pyjwt-2.12.0-py3-none-any.whl", hash = "sha256:9bb459d1bdd0387967d287f5656bf7ec2b9a26645d1961628cda1764e087fd6e", size = 29700, upload-time = "2026-03-12T17:15:29.257Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] [package.optional-dependencies] From 874d14908fdfd26cd9922b75efaf77c41a4f2379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:42:51 +0300 Subject: [PATCH 086/107] chore(deps): bump actions/setup-node from 6.4.0 to 7.0.0 (#2034) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 6.4.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e...820762786026740c76f36085b0efc47a31fe5020) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 9df0f1e0af..718387978e 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -44,7 +44,7 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "20" - name: Prettier check From bcd9b56ce020edc12194e57699891bb8f6b4b9f8 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Thu, 16 Jul 2026 16:15:32 +0300 Subject: [PATCH 087/107] chore(ci): Add a CI workflow that checks that the used CI actions and their versions are allowed (#2058) --- .github/workflows/allowlist-check.yml | 41 +++++++++++++++++++++++++++ .github/workflows/build.yml | 12 ++++---- .github/workflows/dependencies.yml | 4 +-- .github/workflows/docker.yml | 4 +-- .github/workflows/rust.yml | 22 +++++++------- .github/workflows/tpch.yml | 4 +-- .github/workflows/web-tui.yml | 2 +- 7 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/allowlist-check.yml diff --git a/.github/workflows/allowlist-check.yml b/.github/workflows/allowlist-check.yml new file mode 100644 index 0000000000..7bd69829fa --- /dev/null +++ b/.github/workflows/allowlist-check.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +name: "ASF Allowlist Check" + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/**" + push: + # Always run on every commit on the main branch in case of expired allows + branches: + - main + +permissions: + contents: read + +jobs: + asf-allowlist-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: apache/infrastructure-actions/allowlist-check@main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2507bc46b..ed68844836 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Protoc uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 @@ -134,9 +134,9 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 with: - # FIXME: temporary workaround, see: https://github.com/Swatinem/rust-cache/issues/341 + # FIXME: temporary workaround, see: https://github.com/swatinem/rust-cache/issues/341 cache-bin: ${{ matrix.os != 'macos-latest' }} workspaces: | python -> target @@ -186,7 +186,7 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -231,7 +231,7 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -276,7 +276,7 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - run: rm LICENSE.txt - name: Download LICENSE.txt diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 1a816243f3..ef4c4412e7 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -51,7 +51,7 @@ jobs: uses: ./.github/actions/setup-macos-builder with: rust-version: stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Check dependencies run: | cd dev/msrvcheck @@ -68,7 +68,7 @@ jobs: uses: ./.github/actions/setup-macos-builder with: rust-version: stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Verify python/Cargo.lock is in sync with manifests run: | cd python diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 18bef7e7da..3d35d2f204 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,7 +29,7 @@ on: - "dev/build-ballista-docker.sh" - ".github/workflows/docker.yml" push: - branches: ["main"] + branches: [ "main" ] paths: - "ballista/**" - "ballista-cli/**" @@ -64,7 +64,7 @@ jobs: rustup update stable rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run script run: | git config --global --add safe.directory /__w/datafusion-ballista/datafusion-ballista diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c2679ed8dd..47ced05470 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -68,7 +68,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-macos-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: cargo check run: | # Adding `--locked` here to assert that the `Cargo.lock` file is up to @@ -88,7 +88,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run tests run: | export PATH=$PATH:$HOME/d/protoc/bin @@ -105,7 +105,7 @@ jobs: - uses: actions/checkout@v7.0.0 - name: Setup Rust toolchain uses: ./.github/actions/setup-macos-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run cargo doc run: ci/scripts/rust_docs.sh @@ -121,7 +121,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Try to compile when `--no-default-features` is selected run: | export PATH=$PATH:$HOME/d/protoc/bin @@ -141,7 +141,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run Ballista tests run: | export PATH=$PATH:$HOME/d/protoc/bin @@ -161,7 +161,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-macos-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run tests shell: bash run: | @@ -182,7 +182,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Verify that benchmark queries return expected results run: | cargo test --package ballista-benchmarks --profile release-nonlto --features=ci --locked -- --test-threads=1 @@ -197,7 +197,7 @@ jobs: rustup toolchain install stable rustup default stable rustup component add rustfmt - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run run: ci/scripts/rust_fmt.sh @@ -214,7 +214,7 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-macos-builder - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Run clippy run: ci/scripts/rust_clippy.sh @@ -233,7 +233,7 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install cargo-tomlfmt uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 with: @@ -262,7 +262,7 @@ jobs: run: | rustup toolchain install stable rustup default stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Check vendored DataFusion proto matches pinned crate run: ci/scripts/rust_proto_check.sh diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 8e21eaf4b6..580a125e52 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -26,7 +26,7 @@ concurrency: on: push: - branches: ["main"] + branches: [ "main" ] paths: - "ballista/**" - "benchmarks/**" @@ -67,7 +67,7 @@ jobs: with: rust-version: stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Build Ballista binaries run: | diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index ba7b17e981..00da126c79 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -61,7 +61,7 @@ jobs: rustup default stable rustup target add wasm32-unknown-unknown - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 + - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1 - name: Install Trunk uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 From 1c25d30d7e2d462ea7fefd555057f585feb94193 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Thu, 16 Jul 2026 16:42:55 +0300 Subject: [PATCH 088/107] refactor(tui): Drop the header banner/logo (#2057) The banner was using tui-big-text crate that seems to use too modern UTF-8 symbols which are not yet available for many operating systems and because of this the banner didn't render well. --- Cargo.lock | 96 +------------------ ballista-cli/Cargo.toml | 3 +- ballista-cli/Trunk.toml | 9 -- ballista-cli/ballista-web-tui.html | 14 --- ballista-cli/src/tui/infrastructure/theme.rs | 57 +++++------ .../src/tui/ui/header/ballista-logo.svg | 29 ------ ballista-cli/src/tui/ui/header/mod.rs | 52 ++-------- .../src/tui/ui/header/scheduler_state.rs | 2 +- 8 files changed, 36 insertions(+), 226 deletions(-) delete mode 100644 ballista-cli/src/tui/ui/header/ballista-logo.svg diff --git a/Cargo.lock b/Cargo.lock index 7e45d5831f..bc3b3181f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1074,7 +1074,6 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "tracing-web", - "tui-big-text", "tui-shimmer", "wasm-bindgen", "wasm-bindgen-futures", @@ -2119,38 +2118,14 @@ dependencies = [ "cmov", ] -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", + "darling_core", + "darling_macro", ] [[package]] @@ -2166,24 +2141,13 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.117", -] - [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.23.0", + "darling_core", "quote", "syn 2.0.117", ] @@ -3013,37 +2977,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -3387,12 +3320,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "font8x8" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -4104,7 +4031,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ - "darling 0.23.0", + "darling", "indoc", "proc-macro2", "quote", @@ -6500,7 +6427,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -7530,19 +7457,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tui-big-text" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e36b4d581e4e44186f87156c1c281f066ad792551580acc0e02aa7333f769ed" -dependencies = [ - "derive_builder", - "font8x8", - "itertools 0.15.0", - "ratatui-core", - "ratatui-widgets", -] - [[package]] name = "tui-shimmer" version = "0.1.3" diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml index e9aaa9863e..0344a3b5c9 100644 --- a/ballista-cli/Cargo.toml +++ b/ballista-cli/Cargo.toml @@ -59,7 +59,6 @@ tui-shimmer = { version = "0.1", optional = true } # Native-only TUI deps crossterm = { version = "0.29.0", features = ["event-stream"], optional = true } tracing-appender = { version = "0.2", optional = true } -tui-big-text = { version = "0.8.4", optional = true } # Web-only deps (WASM) critical-section = { version = "1.2.0", features = ["std"], optional = true } @@ -89,7 +88,7 @@ cli = [ tui = [ "dep:chrono", "dep:config", "dep:crossterm", "dep:dotparser", "dep:futures", "dep:percent-encoding", "dep:prometheus-parse", "dep:ratatui", "dep:reqwest", - "dep:serde", "dep:serde_json", "dep:tracing-appender", "dep:tui-big-text", + "dep:serde", "dep:serde_json", "dep:tracing-appender", "dep:tui-shimmer", "ratatui/crossterm", "ratatui/all-widgets", "ratatui/macros", "ratatui/layout-cache", "ratatui/underline-color", "ratatui/serde" ] diff --git a/ballista-cli/Trunk.toml b/ballista-cli/Trunk.toml index 61ace21658..37f83dd7e9 100644 --- a/ballista-cli/Trunk.toml +++ b/ballista-cli/Trunk.toml @@ -32,12 +32,3 @@ public_url = "./" [watch] ignore = [] - -[[hooks]] -stage = "build" -command = "sh" -command_arguments = ["-c", "cp -f ./src/tui/ui/header/ballista-logo.svg \"$TRUNK_STAGING_DIR/ballista-logo.svg\""] - -[hooks.windows] -command = "cmd" -command_arguments = ["/c", "copy .\\src\\tui\\ui\\header\\ballista-logo.svg \"%TRUNK_STAGING_DIR%\\ballista-logo.svg\""] \ No newline at end of file diff --git a/ballista-cli/ballista-web-tui.html b/ballista-cli/ballista-web-tui.html index e55be1653b..ea79918a22 100644 --- a/ballista-cli/ballista-web-tui.html +++ b/ballista-cli/ballista-web-tui.html @@ -27,23 +27,9 @@ margin: 0; background: #000; } - - #ballista-logo { - position: absolute; - top: 0; - left: 0; - width: 30%; - max-width: 500px; - max-height: 15vh; - object-fit: contain; - pointer-events: none; - z-index: 10; - } - -