diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index fcd9460025..b9067d7413 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -23,18 +23,43 @@ updates:
       interval: daily
     open-pull-requests-limit: 10
     target-branch: main
-    labels: [auto-dependencies]
+    labels: [ auto-dependencies ]
+    commit-message:
+      prefix: "chore(deps): "
     ignore:
       # arrow and datafusion are bumped manually
       - dependency-name: "arrow*"
-        update-types: ["version-update:semver-major"]
+        update-types: [ "version-update:semver-major" ]
       - dependency-name: "datafusion*"
-        update-types: ["version-update:semver-major"]
+        update-types: [ "version-update:semver-major" ]
       - dependency-name: "sqlparser"
-        update-types: ["version-update:semver-major"]
+        update-types: [ "version-update:semver-major" ]
+  - package-ecosystem: cargo
+    directory: "/python"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: main
+    labels: [ auto-dependencies ]
+    commit-message:
+      prefix: "chore(deps): "
+    ignore:
+      # arrow and datafusion are bumped manually
+      - dependency-name: "arrow*"
+        update-types: [ "version-update:semver-major" ]
+      - dependency-name: "datafusion*"
+        update-types: [ "version-update:semver-major" ]
+      - dependency-name: "sqlparser"
+        update-types: [ "version-update:semver-major" ]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 10
-    labels: [auto-dependencies]
+    labels: [ auto-dependencies ]
+    commit-message:
+      prefix: "chore(ci): "
+    groups:
+      github-codeql:
+        patterns:
+          - "github/codeql-action/*"
diff --git a/.github/workflows/allowlist-check.yml b/.github/workflows/allowlist-check.yml
new file mode 100644
index 0000000000..7bd69829fa
--- /dev/null
+++ b/.github/workflows/allowlist-check.yml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+name: "ASF Allowlist Check"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - ".github/**"
+  push:
+    # Always run on every commit on the main branch in case of expired allows
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  asf-allowlist-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: apache/infrastructure-actions/allowlist-check@main
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b4aea4d977..ed68844836 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -49,8 +49,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - name: Install Protoc
         uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
@@ -58,7 +61,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
 
@@ -77,7 +80,7 @@ jobs:
         with:
           python-version: "3.12"
 
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
 
@@ -100,7 +103,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7.0.0
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
       - name: Generate license file
@@ -127,10 +130,13 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
         with:
-          # FIXME: temporary workaround, see: https://github.com/Swatinem/rust-cache/issues/341
+          # FIXME: temporary workaround, see: https://github.com/swatinem/rust-cache/issues/341
           cache-bin: ${{ matrix.os != 'macos-latest' }}
           workspaces: |
             python -> target
@@ -148,7 +154,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
 
@@ -176,8 +182,11 @@ jobs:
     steps:
       - uses: actions/checkout@v7.0.0
 
-      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - run: rm LICENSE.txt
       - name: Download LICENSE.txt
@@ -194,7 +203,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
 
@@ -218,8 +227,11 @@ jobs:
     steps:
       - uses: actions/checkout@v7.0.0
 
-      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - run: rm LICENSE.txt
       - name: Download LICENSE.txt
@@ -236,7 +248,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
 
@@ -260,8 +272,11 @@ jobs:
     steps:
       - uses: actions/checkout@v7.0.0
 
-      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - run: rm LICENSE.txt
       - name: Download LICENSE.txt
@@ -278,7 +293,7 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
           enable-cache: true
 
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 639690037e..9ffe8e38a6 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -48,11 +48,11 @@ jobs:
           persist-credentials: false
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
+        uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4
         with:
           languages: actions
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
+        uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4
         with:
           category: "/language:actions"
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
index 65d267be98..ef4c4412e7 100644
--- a/.github/workflows/dependencies.yml
+++ b/.github/workflows/dependencies.yml
@@ -17,6 +17,9 @@
 
 name: Dependencies
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
@@ -48,8 +51,28 @@ jobs:
         uses: ./.github/actions/setup-macos-builder
         with:
           rust-version: stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Check dependencies
         run: |
           cd dev/msrvcheck
           cargo run
+
+  python-lock:
+    name: python Cargo.lock up to date
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 1
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-macos-builder
+        with:
+          rust-version: stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Verify python/Cargo.lock is in sync with manifests
+        run: |
+          cd python
+          if ! cargo metadata --locked --format-version 1 >/dev/null; then
+            echo "::error::python/Cargo.lock is out of date with the manifests. Run 'cd python && cargo update' and commit the updated Cargo.lock."
+            exit 1
+          fi
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 9df0f1e0af..718387978e 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -44,7 +44,7 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.1
-      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: "20"
       - name: Prettier check
diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml
index 106463a206..7b96019abc 100644
--- a/.github/workflows/dev_pr.yml
+++ b/.github/workflows/dev_pr.yml
@@ -38,7 +38,7 @@ jobs:
           github.event_name == 'pull_request_target' &&
             (github.event.action == 'opened' ||
              github.event.action == 'synchronize')
-        uses: actions/labeler@v6.1.0
+        uses: actions/labeler@v6.2.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/workflows/dev_pr/labeler.yml
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 18bef7e7da..86d6ae80ef 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -28,8 +28,10 @@ on:
       - "dev/docker/**"
       - "dev/build-ballista-docker.sh"
       - ".github/workflows/docker.yml"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   push:
-    branches: ["main"]
+    branches: [ "main" ]
     paths:
       - "ballista/**"
       - "ballista-cli/**"
@@ -40,6 +42,8 @@ on:
       - "dev/docker/**"
       - "dev/build-ballista-docker.sh"
       - ".github/workflows/docker.yml"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -64,7 +68,7 @@ jobs:
           rustup update stable
           rustup toolchain install stable
           rustup default stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run script
         run: |
           git config --global --add safe.directory /__w/datafusion-ballista/datafusion-ballista
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 2a77a53d5c..d28af79fb1 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -38,6 +38,8 @@ on:
       - ".github/actions/setup-windows-builder/**"
       - ".github/actions/setup-rust-runtime/**"
       - ".github/workflows/rust.yml"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   pull_request:
     paths:
       - "ballista/**"
@@ -53,6 +55,8 @@ on:
       - ".github/actions/setup-windows-builder/**"
       - ".github/actions/setup-rust-runtime/**"
       - ".github/workflows/rust.yml"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
@@ -68,7 +72,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-macos-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: cargo check
         run: |
           # Adding `--locked` here to assert that the `Cargo.lock` file is up to
@@ -88,7 +92,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run tests
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
@@ -105,7 +109,7 @@ jobs:
       - uses: actions/checkout@v7.0.0
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-macos-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run cargo doc
         run: ci/scripts/rust_docs.sh
 
@@ -121,7 +125,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Try to compile when `--no-default-features` is selected
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
@@ -141,7 +145,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run Ballista tests
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
@@ -161,7 +165,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-macos-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run tests
         shell: bash
         run: |
@@ -182,7 +186,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Verify that benchmark queries return expected results
         run: |
           cargo test --package ballista-benchmarks --profile release-nonlto --features=ci --locked -- --test-threads=1
@@ -197,7 +201,7 @@ jobs:
           rustup toolchain install stable
           rustup default stable
           rustup component add rustfmt
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run
         run: ci/scripts/rust_fmt.sh
 
@@ -214,7 +218,7 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-macos-builder
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Run clippy
         run: ci/scripts/rust_clippy.sh
 
@@ -233,9 +237,9 @@ jobs:
         run: |
           rustup toolchain install stable
           rustup default stable
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
       - name: Install cargo-tomlfmt
-        uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2.82.2
+        uses: taiki-e/install-action@ed67fa35ac944f3a9b33f12c4dd43b6f31a47e20 # v2.83.3
         with:
           tool: cargo-tomlfmt@0.2.1
 
@@ -251,6 +255,21 @@ jobs:
               exit 1
           fi
 
+  datafusion-proto-sync-check:
+    name: Check vendored DataFusion proto is in sync
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 1
+      - name: Setup Rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - name: Check vendored DataFusion proto matches pinned crate
+        run: ci/scripts/rust_proto_check.sh
+
 # Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it
 
 # coverage:
diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml
new file mode 100644
index 0000000000..ddfa4c802b
--- /dev/null
+++ b/.github/workflows/tpcds.yml
@@ -0,0 +1,184 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: TPC-DS SF1
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "ballista/**"
+      - "benchmarks/**"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - "rust-toolchain.toml"
+      - ".github/workflows/tpcds.yml"
+      - ".github/actions/setup-builder/**"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
+  pull_request:
+    paths:
+      - "ballista/**"
+      - "benchmarks/**"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - "rust-toolchain.toml"
+      - ".github/workflows/tpcds.yml"
+      - ".github/actions/setup-builder/**"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
+  workflow_dispatch:
+
+jobs:
+  tpcds-sf1:
+    name: TPC-DS SF1 (all queries, static planner)
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y netcat-traditional
+
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 1
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+
+      - name: Build Ballista binaries
+        run: |
+          cargo build --profile tpch-ci --locked \
+            -p ballista-scheduler \
+            -p ballista-executor \
+            -p ballista-benchmarks
+
+      - name: Install tpcgen-cli
+        run: |
+          cargo install --git https://github.com/clflushopt/tpchgen-rs \
+            --rev 573fd0018dd1b41e52880162f615e9903418c397 --locked tpcgen-cli
+
+      - name: Generate TPC-DS SF1 data
+        run: |
+          mkdir -p "$RUNNER_TEMP/tpcds-data"
+          tpcgen-cli tpcds parquet \
+            --scale-factor 1 \
+            --output-dir "$RUNNER_TEMP/tpcds-data"
+
+      - name: Run TPC-DS queries against Ballista cluster
+        env:
+          DATA_DIR: ${{ runner.temp }}/tpcds-data
+          WORK_DIR: ${{ runner.temp }}/work
+          SCHEDULER_LOG: ${{ runner.temp }}/scheduler.log
+          EXECUTOR_LOG: ${{ runner.temp }}/executor.log
+        run: |
+          set -euo pipefail
+
+          mkdir -p "$WORK_DIR"
+
+          ./target/tpch-ci/ballista-scheduler \
+            --bind-host 127.0.0.1 \
+            > "$SCHEDULER_LOG" 2>&1 &
+          SCHEDULER_PID=$!
+
+          ./target/tpch-ci/ballista-executor \
+            --bind-host 127.0.0.1 \
+            --bind-port 50051 \
+            --scheduler-host 127.0.0.1 \
+            --scheduler-connect-timeout-seconds 10 \
+            --concurrent-tasks 4 \
+            --memory-pool-size 2GB \
+            --work-dir "$WORK_DIR" \
+            > "$EXECUTOR_LOG" 2>&1 &
+          EXECUTOR_PID=$!
+
+          cleanup() {
+            echo "::group::scheduler log (tail)"
+            tail -n 200 "$SCHEDULER_LOG" || true
+            echo "::endgroup::"
+            echo "::group::executor log (tail)"
+            tail -n 200 "$EXECUTOR_LOG" || true
+            echo "::endgroup::"
+            kill "$SCHEDULER_PID" "$EXECUTOR_PID" 2>/dev/null || true
+            wait "$SCHEDULER_PID" "$EXECUTOR_PID" 2>/dev/null || true
+          }
+          trap cleanup EXIT
+
+          echo "Waiting for scheduler on 127.0.0.1:50050..."
+          for _ in $(seq 1 30); do
+            if nc -z 127.0.0.1 50050; then
+              break
+            fi
+            sleep 1
+          done
+          nc -z 127.0.0.1 50050 || { echo "scheduler did not start"; exit 1; }
+
+          echo "Waiting for executor on 127.0.0.1:50051..."
+          for _ in $(seq 1 30); do
+            if nc -z 127.0.0.1 50051; then
+              break
+            fi
+            sleep 1
+          done
+          nc -z 127.0.0.1 50051 || { echo "executor did not start"; exit 1; }
+
+          # Run the suite under the default (static) planner. The tpcds binary
+          # internally loops all non-skipped queries and exits non-zero on any
+          # failure, so a single invocation covers the whole suite.
+          #
+          # The adaptive planner (AQE on) is intentionally not run here yet: it
+          # currently fails many TPC-DS queries with an `EmptyExec invalid
+          # partition` assertion (issue #2047). Re-add an `AQE on` invocation
+          # once that is fixed.
+          run_suite() {
+            local label="$1"; shift
+            echo "::group::[$label] TPC-DS suite"
+            ./target/tpch-ci/tpcds \
+              --host 127.0.0.1 --port 50050 \
+              --path "$DATA_DIR" \
+              --partitions 16 \
+              --verify \
+              "$@"
+            echo "::endgroup::"
+          }
+
+          run_suite "static planner" \
+            -c datafusion.optimizer.prefer_hash_join=false
+
+      - name: Upload cluster logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tpcds-sf1-cluster-logs
+          retention-days: 14
+          path: |
+            ${{ runner.temp }}/scheduler.log
+            ${{ runner.temp }}/executor.log
+          if-no-files-found: ignore
diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml
index fac580fa05..f0cf83256e 100644
--- a/.github/workflows/tpch.yml
+++ b/.github/workflows/tpch.yml
@@ -26,7 +26,7 @@ concurrency:
 
 on:
   push:
-    branches: ["main"]
+    branches: [ "main" ]
     paths:
       - "ballista/**"
       - "benchmarks/**"
@@ -35,6 +35,8 @@ on:
       - "rust-toolchain.toml"
       - ".github/workflows/tpch.yml"
       - ".github/actions/setup-builder/**"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   pull_request:
     paths:
       - "ballista/**"
@@ -44,11 +46,13 @@ on:
       - "rust-toolchain.toml"
       - ".github/workflows/tpch.yml"
       - ".github/actions/setup-builder/**"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   workflow_dispatch:
 
 jobs:
   tpch-sf10:
-    name: TPC-H SF10 (all queries)
+    name: TPC-H SF10 (all queries, AQE off + on)
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
@@ -67,17 +71,17 @@ jobs:
         with:
           rust-version: stable
 
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - name: Build Ballista binaries
         run: |
-          cargo build --profile release-nonlto --locked \
+          cargo build --profile tpch-ci --locked \
             -p ballista-scheduler \
             -p ballista-executor \
             -p ballista-benchmarks
 
       - name: Install tpchgen-cli
-        uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2.82.2
+        uses: taiki-e/install-action@ed67fa35ac944f3a9b33f12c4dd43b6f31a47e20 # v2.83.3
         with:
           tool: tpchgen-cli@2.0.2
 
@@ -101,17 +105,17 @@ jobs:
 
           mkdir -p "$WORK_DIR"
 
-          ./target/release-nonlto/ballista-scheduler \
+          ./target/tpch-ci/ballista-scheduler \
             --bind-host 127.0.0.1 \
             > "$SCHEDULER_LOG" 2>&1 &
           SCHEDULER_PID=$!
 
-          ./target/release-nonlto/ballista-executor \
+          ./target/tpch-ci/ballista-executor \
             --bind-host 127.0.0.1 \
             --bind-port 50051 \
             --scheduler-host 127.0.0.1 \
             --scheduler-connect-timeout-seconds 10 \
-            --concurrent-tasks 4 \
+            --vcores 4 \
             --memory-pool-size 2GB \
             --work-dir "$WORK_DIR" \
             > "$EXECUTOR_LOG" 2>&1 &
@@ -147,19 +151,34 @@ jobs:
           done
           nc -z 127.0.0.1 50051 || { echo "executor did not start"; exit 1; }
 
-          # q16 omitted: still unsupported (matches benchmarks/run.sh).
-          for q in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 17 18 19 20 21 22; do
-            echo "::group::Query $q"
-            ./target/release-nonlto/tpch benchmark ballista \
-              --host 127.0.0.1 --port 50050 \
-              --query "$q" \
-              --path "$DATA_DIR" \
-              --format parquet \
-              --partitions 16 \
-              --iterations 1 \
-              -c datafusion.optimizer.prefer_hash_join=false
-            echo "::endgroup::"
-          done
+          # Run the suite once per planner mode against the same data and
+          # cluster. AQE on/off is a per-query session setting, so the scheduler
+          # picks the planner per job; no need to regenerate data or restart the
+          # cluster.
+          run_suite() {
+            local label="$1"; shift
+            # q16 omitted: still unsupported (matches benchmarks/run.sh).
+            for q in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 17 18 19 20 21 22; do
+              echo "::group::[$label] Query $q"
+              ./target/tpch-ci/tpch benchmark ballista \
+                --host 127.0.0.1 --port 50050 \
+                --query "$q" \
+                --path "$DATA_DIR" \
+                --format parquet \
+                --partitions 16 \
+                --iterations 1 \
+                "$@"
+              echo "::endgroup::"
+            done
+          }
+
+          run_suite "AQE off" \
+            --verify \
+            -c datafusion.optimizer.prefer_hash_join=false
+          run_suite "AQE on" \
+            --verify \
+            -c datafusion.optimizer.prefer_hash_join=false \
+            -c ballista.planner.adaptive.enabled=true
 
       - name: Upload cluster logs on failure
         if: failure()
diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml
index 151e360f22..5074a63031 100644
--- a/.github/workflows/web-tui.yml
+++ b/.github/workflows/web-tui.yml
@@ -35,6 +35,8 @@ on:
       - "rust-toolchain.toml"
       - ".github/workflows/web-tui.yml"
       - ".github/actions/setup-builder/**"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   pull_request:
     paths:
       - "ballista/**"
@@ -44,6 +46,8 @@ on:
       - "rust-toolchain.toml"
       - ".github/workflows/web-tui.yml"
       - ".github/actions/setup-builder/**"
+      # docs-only changes cannot affect this job; exclusions are applied last
+      - "!**/*.md"
   workflow_dispatch:
 
 jobs:
@@ -56,24 +60,49 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Rust toolchain
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-        with:
-          toolchain: "stable"
-          targets: "wasm32-unknown-unknown"
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+          rustup target add wasm32-unknown-unknown
 
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
+      - uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 #2.9.1
 
       - name: Install Trunk
-        uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2.82.2
+        uses: taiki-e/install-action@ed67fa35ac944f3a9b33f12c4dd43b6f31a47e20 # v2.83.3
         with:
           tool: trunk@0.21.14
 
+      - name: Install cargo-get
+        uses: taiki-e/install-action@ed67fa35ac944f3a9b33f12c4dd43b6f31a47e20 # v2.83.3
+        with:
+          tool: cargo-get@1.4.0
+
+      - name: Get package.version
+        id: cargo_metadata
+        run: |
+          BALLISTA_VERSION=$(cargo get --entry ballista/core/ package.version)
+          echo "ballista_version=${BALLISTA_VERSION}" >> "$GITHUB_OUTPUT"
+
       - name: Build
         working-directory: ballista-cli
         run: |
           trunk build --cargo-profile release-nonlto --minify --locked \
             --no-default-features --features web
 
+      - name: Deploy at nightlies.a.o
+        if: ${{ github.event == 'push' && github.ref == 'refs/heads/main' }}
+        run: |
+          TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}"
+          REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/"
+
+          echo "${{ secrets.NIGHTLIES_RSYNC_KEY }}" | ssh-add -
+
+          ssh-keyscan -H "${{ secrets.NIGHTLIES_RSYNC_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
+
+          ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }} ${{ secrets.NIGHTLIES_RSYNC_HOST }} "mkdir -p ${REMOTE_TARGET_DIR}"
+
+          rsync --times --compress --delete --verbose -e "ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }}" ./target/web-tui/* ${{ secrets.NIGHTLIES_RSYNC_HOST }}:${REMOTE_TARGET_DIR}
+
       - name: Upload WASM32 application
         uses: actions/upload-artifact@v7
         with:
diff --git a/.gitignore b/.gitignore
index 42f5e3ebe2..f99b29e5f1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -116,3 +116,4 @@ CLAUDE.md
 .worktrees/
 # ignore insta captures
 *-snap
+.local/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dfab6fb9f6..c8d0c31fa0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,1630 +17,7 @@
   under the License.
 -->
 
-# Changelog
+# Apache DataFusion Ballista Changelog
 
-## [53.0.0](https://github.com/apache/datafusion-ballista/tree/53.0.0) (2026-05-19)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/52.0.0...53.0.0)
-
-**Implemented enhancements:**
-
-- [REST]: Cancelling a completed/failed job should return "false" [#1494](https://github.com/apache/datafusion-ballista/pull/1494) (martin-g)
-- feat: Add plain "status" field to the JobResponse [#1497](https://github.com/apache/datafusion-ballista/pull/1497) (martin-g)
-- feat: Expose Logical and Physical plan details in the REST API [#1498](https://github.com/apache/datafusion-ballista/pull/1498) (milenkovicm)
-- feat: (remote) shuffle reader cleanup [#1503](https://github.com/apache/datafusion-ballista/pull/1503) (milenkovicm)
-- feat: enable scheduler rest api by default [#1506](https://github.com/apache/datafusion-ballista/pull/1506) (milenkovicm)
-- feat: [REST] Return the job's start and end times [#1511](https://github.com/apache/datafusion-ballista/pull/1511) (martin-g)
-- feat: remove hash policy task distribution as not used [#1526](https://github.com/apache/datafusion-ballista/pull/1526) (eachsaj)
-- feat: remove full path from partition locations [#1527](https://github.com/apache/datafusion-ballista/pull/1527) (milenkovicm)
-- feat: add config to ExecutorEngine::create_query_stage_exec [#1542](https://github.com/apache/datafusion-ballista/pull/1542) (milenkovicm)
-- feat: Improve REST API adding task related info to job stages [#1543](https://github.com/apache/datafusion-ballista/pull/1543) (milenkovicm)
-- feat: Ballista Text User Interface [#1436](https://github.com/apache/datafusion-ballista/pull/1436) (martin-g)
-- feat: jupyter notebook support [#1513](https://github.com/apache/datafusion-ballista/pull/1513) (sandugood)
-- feat: ExecutionEngine::create_query_stage_exec accepts partition_id [#1556](https://github.com/apache/datafusion-ballista/pull/1556) (milenkovicm)
-- feat: make ballista client retry policy configurable  [#1577](https://github.com/apache/datafusion-ballista/pull/1577) (eachsaj)
-- feat: Executor system & process metrics reporting [#1547](https://github.com/apache/datafusion-ballista/pull/1547) (sandugood)
-- feat: Scheduler config update [#1597](https://github.com/apache/datafusion-ballista/pull/1597) (sandugood)
-- feat: Support `EXPLAIN ANALYZE` in Ballista [#1567](https://github.com/apache/datafusion-ballista/pull/1567) (danielhumanmod)
-- feat: Add standalone shuffle writer benchmark that shuffles real Parquet input [#1600](https://github.com/apache/datafusion-ballista/pull/1600) (andygrove)
-- feat: defer sort-shuffle materialization with interleave_record_batch [#1598](https://github.com/apache/datafusion-ballista/pull/1598) (andygrove)
-- feat(executor): bound executor memory via --memory-pool-size [#1624](https://github.com/apache/datafusion-ballista/pull/1624) (andygrove)
-- feat(bench): show TPC-H query timings in seconds and add total time [#1641](https://github.com/apache/datafusion-ballista/pull/1641) (andygrove)
-- feat(aqe): Support sort-based shuffle writer in AQE [#1640](https://github.com/apache/datafusion-ballista/pull/1640) (danielhumanmod)
-- feat: rest api supports plan tree rendering [#1650](https://github.com/apache/datafusion-ballista/pull/1650) (sandugood)
-- feat: Cache ballista clients on executor [#1578](https://github.com/apache/datafusion-ballista/pull/1578) (milenkovicm)
-- feat(aqe): Lazy stage evaluation in AQE [#1649](https://github.com/apache/datafusion-ballista/pull/1649) (milenkovicm)
-- feat: move shuffle writer disk I/O off tokio worker threads [#1537](https://github.com/apache/datafusion-ballista/pull/1537) (hcrosse)
-- feat(scheduler): broadcast-style hash join for small-side joins [#1647](https://github.com/apache/datafusion-ballista/pull/1647) (andygrove)
-- feat: default to sort-merge join [#1651](https://github.com/apache/datafusion-ballista/pull/1651) (andygrove)
-- feat: TUI shows running job information [#1717](https://github.com/apache/datafusion-ballista/pull/1717) (milenkovicm)
-- feat(aqe): CoalescePartitionsRule — shuffle-partition coalescing on resolved stats [#1684](https://github.com/apache/datafusion-ballista/pull/1684) (metegenez)
-- feat: TUI make task popup scrollable [#1725](https://github.com/apache/datafusion-ballista/pull/1725) (milenkovicm)
-- feat(tui): Use separate areas for the table and its associated scrollbar [#1729](https://github.com/apache/datafusion-ballista/pull/1729) (martin-g)
-
-**Fixed bugs:**
-
-- fix: remove unwrap from executor and improve task error handling [#1540](https://github.com/apache/datafusion-ballista/pull/1540) (eachsaj)
-- fix: handle None task slot in update_task_info after executor lost  [#1523](https://github.com/apache/datafusion-ballista/pull/1523) (milenkovicm)
-- fix: [Python] serialize optimized logical plan to fix subquery support [#1586](https://github.com/apache/datafusion-ballista/pull/1586) (andygrove)
-- fix: allow S3 access without explicit credentials [#1584](https://github.com/apache/datafusion-ballista/pull/1584) (andygrove)
-- fix: route collect/show/to_pandas through Ballista cluster [#1585](https://github.com/apache/datafusion-ballista/pull/1585) (andygrove)
-- fix: propagate session config from Python client to Ballista cluster [#1592](https://github.com/apache/datafusion-ballista/pull/1592) (andygrove)
-- fix(python): ignore ballista-namespaced cluster_config keys locally [#1613](https://github.com/apache/datafusion-ballista/pull/1613) (andygrove)
-- fix: `df.write_` fix as it was broken after update [#1625](https://github.com/apache/datafusion-ballista/pull/1625) (milenkovicm)
-- fix(rest): remove unwrap and return 404 if executor does not exist [#1628](https://github.com/apache/datafusion-ballista/pull/1628) (milenkovicm)
-- fix(metrics): avoid stage metrics inflation by tracking partition snapshots [#1652](https://github.com/apache/datafusion-ballista/pull/1652) (danielhumanmod)
-- fix: compilation issue after merge [#1658](https://github.com/apache/datafusion-ballista/pull/1658) (milenkovicm)
-- fix: rest api calculates stage running time correctly [#1675](https://github.com/apache/datafusion-ballista/pull/1675) (milenkovicm)
-- fix: REST API does not show running jobs [#1703](https://github.com/apache/datafusion-ballista/pull/1703) (gittihub-jpg)
-- fix: no executor warning, correct prometheus feature name in TUI [#1698](https://github.com/apache/datafusion-ballista/pull/1698) (killzoner)
-
-**Documentation updates:**
-
-- docs: document hash-based and sort-based shuffle implementations [#1595](https://github.com/apache/datafusion-ballista/pull/1595) (andygrove)
-- docs: improve Python documentation structure [#1579](https://github.com/apache/datafusion-ballista/pull/1579) (andygrove)
-- docs: add Python client build-from-source section to contributor guide [#1620](https://github.com/apache/datafusion-ballista/pull/1620) (andygrove)
-- chore: remove NYC Taxi benchmark [#1644](https://github.com/apache/datafusion-ballista/pull/1644) (andygrove)
-- docs: document experimental Adaptive Query Execution in tuning guide [#1645](https://github.com/apache/datafusion-ballista/pull/1645) (andygrove)
-- build(bench): rework docker-compose TPC-H stack [#1646](https://github.com/apache/datafusion-ballista/pull/1646) (andygrove)
-- docs: add Ballista TUI documentation [#1593](https://github.com/apache/datafusion-ballista/pull/1593) (goingforstudying-ctrl)
-- [TUI] Show executor's details in a popup [#1670](https://github.com/apache/datafusion-ballista/pull/1670) (martin-g)
-- [TUI] Add a config setting for rendering job stage's plan as a tree [#1704](https://github.com/apache/datafusion-ballista/pull/1704) (martin-g)
-- [TUI] Add support for horizontal scrolling to the job/stage plan popups [#1711](https://github.com/apache/datafusion-ballista/pull/1711) (martin-g)
-- [TUI] Add screenshots of the TUI application in README/cli.md [#1714](https://github.com/apache/datafusion-ballista/pull/1714) (martin-g)
-
-**Merged pull requests:**
-
-- CI: Add CodeQL workflow for GitHub Actions security scanning [#1484](https://github.com/apache/datafusion-ballista/pull/1484) (kevinjqliu)
-- ci: Harden labeler workflow, remove unnecessary checkout from pull_request_target job [#1487](https://github.com/apache/datafusion-ballista/pull/1487) (kevinjqliu)
-- ci: add take and stale [#1488](https://github.com/apache/datafusion-ballista/pull/1488) (kevinjqliu)
-- [REST]: Cancelling a completed/failed job should return "false" [#1494](https://github.com/apache/datafusion-ballista/pull/1494) (martin-g)
-- feat: Add plain "status" field to the JobResponse [#1497](https://github.com/apache/datafusion-ballista/pull/1497) (martin-g)
-- feat: Expose Logical and Physical plan details in the REST API [#1498](https://github.com/apache/datafusion-ballista/pull/1498) (milenkovicm)
-- chore(deps): cargo update deps  [#1502](https://github.com/apache/datafusion-ballista/pull/1502) (milenkovicm)
-- feat: (remote) shuffle reader cleanup [#1503](https://github.com/apache/datafusion-ballista/pull/1503) (milenkovicm)
-- minor: task scheduling policy config  cleanup [#1507](https://github.com/apache/datafusion-ballista/pull/1507) (milenkovicm)
-- minor: address task policy config comments [#1508](https://github.com/apache/datafusion-ballista/pull/1508) (milenkovicm)
-- feat: enable scheduler rest api by default [#1506](https://github.com/apache/datafusion-ballista/pull/1506) (milenkovicm)
-- feat: [REST] Return the job's start and end times [#1511](https://github.com/apache/datafusion-ballista/pull/1511) (martin-g)
-- minor: [REST] add datafusion version info [#1512](https://github.com/apache/datafusion-ballista/pull/1512) (milenkovicm)
-- ci: pin third-party actions to Apache-approved SHAs [#1516](https://github.com/apache/datafusion-ballista/pull/1516) (kevinjqliu)
-- chore(deps): update to datafusion v.53 [#1486](https://github.com/apache/datafusion-ballista/pull/1486) (milenkovicm)
-- feat: remove hash policy task distribution as not used [#1526](https://github.com/apache/datafusion-ballista/pull/1526) (eachsaj)
-- chore: update datafusion proto [#1528](https://github.com/apache/datafusion-ballista/pull/1528) (milenkovicm)
-- minor: cleanup scheduler clap configuration  [#1529](https://github.com/apache/datafusion-ballista/pull/1529) (milenkovicm)
-- fix: remove unwrap from executor and improve task error handling [#1540](https://github.com/apache/datafusion-ballista/pull/1540) (eachsaj)
-- feat: remove full path from partition locations [#1527](https://github.com/apache/datafusion-ballista/pull/1527) (milenkovicm)
-- fix: handle None task slot in update_task_info after executor lost  [#1523](https://github.com/apache/datafusion-ballista/pull/1523) (milenkovicm)
-- feat: add config to ExecutorEngine::create_query_stage_exec [#1542](https://github.com/apache/datafusion-ballista/pull/1542) (milenkovicm)
-- feat: Improve REST API adding task related info to job stages [#1543](https://github.com/apache/datafusion-ballista/pull/1543) (milenkovicm)
-- feat: Ballista Text User Interface [#1436](https://github.com/apache/datafusion-ballista/pull/1436) (martin-g)
-- feat: jupyter notebook support [#1513](https://github.com/apache/datafusion-ballista/pull/1513) (sandugood)
-- feat: ExecutionEngine::create_query_stage_exec accepts partition_id [#1556](https://github.com/apache/datafusion-ballista/pull/1556) (milenkovicm)
-- chore: Fix Clippy issues with Rust 1.95.0 [#1558](https://github.com/apache/datafusion-ballista/pull/1558) (martin-g)
-- chore: update deps [#1561](https://github.com/apache/datafusion-ballista/pull/1561) (milenkovicm)
-- [TUI] Change the key binding for job's plans [#1573](https://github.com/apache/datafusion-ballista/pull/1573) (martin-g)
-- feat: make ballista client retry policy configurable  [#1577](https://github.com/apache/datafusion-ballista/pull/1577) (eachsaj)
-- feat: Executor system & process metrics reporting [#1547](https://github.com/apache/datafusion-ballista/pull/1547) (sandugood)
-- fix: [Python] serialize optimized logical plan to fix subquery support [#1586](https://github.com/apache/datafusion-ballista/pull/1586) (andygrove)
-- fix: allow S3 access without explicit credentials [#1584](https://github.com/apache/datafusion-ballista/pull/1584) (andygrove)
-- fix: route collect/show/to_pandas through Ballista cluster [#1585](https://github.com/apache/datafusion-ballista/pull/1585) (andygrove)
-- fix: propagate session config from Python client to Ballista cluster [#1592](https://github.com/apache/datafusion-ballista/pull/1592) (andygrove)
-- minor: make config naming convention consistent [#1580](https://github.com/apache/datafusion-ballista/pull/1580) (milenkovicm)
-- chore: Remove unnecessary optimizer rules due to datafusion upgrade to v53 [#1594](https://github.com/apache/datafusion-ballista/pull/1594) (sandugood)
-- docs: document hash-based and sort-based shuffle implementations [#1595](https://github.com/apache/datafusion-ballista/pull/1595) (andygrove)
-- feat: Scheduler config update [#1597](https://github.com/apache/datafusion-ballista/pull/1597) (sandugood)
-- feat: Support `EXPLAIN ANALYZE` in Ballista [#1567](https://github.com/apache/datafusion-ballista/pull/1567) (danielhumanmod)
-- docs: improve Python documentation structure [#1579](https://github.com/apache/datafusion-ballista/pull/1579) (andygrove)
-- feat: Add standalone shuffle writer benchmark that shuffles real Parquet input [#1600](https://github.com/apache/datafusion-ballista/pull/1600) (andygrove)
-- feat: defer sort-shuffle materialization with interleave_record_batch [#1598](https://github.com/apache/datafusion-ballista/pull/1598) (andygrove)
-- ci: drop Intel macOS Python wheel build [#1612](https://github.com/apache/datafusion-ballista/pull/1612) (andygrove)
-- fix(python): ignore ballista-namespaced cluster_config keys locally [#1613](https://github.com/apache/datafusion-ballista/pull/1613) (andygrove)
-- chore: update python deps to ballista and datafusion 52 [#1590](https://github.com/apache/datafusion-ballista/pull/1590) (andygrove)
-- feat(sort-shuffle): byte-copy spill files and enable block-IO transport [#1615](https://github.com/apache/datafusion-ballista/pull/1615) (andygrove)
-- Use BallistaSessionContext instead of BallistaBuilder in tpch.py [#1621](https://github.com/apache/datafusion-ballista/pull/1621) (martin-g)
-- docs: add Python client build-from-source section to contributor guide [#1620](https://github.com/apache/datafusion-ballista/pull/1620) (andygrove)
-- feat(executor): bound executor memory via --memory-pool-size [#1624](https://github.com/apache/datafusion-ballista/pull/1624) (andygrove)
-- fix: `df.write_` fix as it was broken after update [#1625](https://github.com/apache/datafusion-ballista/pull/1625) (milenkovicm)
-- feat(sort-shuffle): enable sort-based shuffle by default [#1623](https://github.com/apache/datafusion-ballista/pull/1623) (andygrove)
-- fix(rest): remove unwrap and return 404 if executor does not exist [#1628](https://github.com/apache/datafusion-ballista/pull/1628) (milenkovicm)
-- perf(sort-shuffle): fix performance regression caused by datafusion upgrade [#1626](https://github.com/apache/datafusion-ballista/pull/1626) (andygrove)
-- fix(sort-shuffle): bound writer memory with per-task spill threshold [#1636](https://github.com/apache/datafusion-ballista/pull/1636) (andygrove)
-- chore: remove NYC Taxi benchmark [#1644](https://github.com/apache/datafusion-ballista/pull/1644) (andygrove)
-- docs: document experimental Adaptive Query Execution in tuning guide [#1645](https://github.com/apache/datafusion-ballista/pull/1645) (andygrove)
-- feat(bench): show TPC-H query timings in seconds and add total time [#1641](https://github.com/apache/datafusion-ballista/pull/1641) (andygrove)
-- build(bench): rework docker-compose TPC-H stack [#1646](https://github.com/apache/datafusion-ballista/pull/1646) (andygrove)
-- feat(aqe): Support sort-based shuffle writer in AQE [#1640](https://github.com/apache/datafusion-ballista/pull/1640) (danielhumanmod)
-- fix(metrics): avoid stage metrics inflation by tracking partition snapshots [#1652](https://github.com/apache/datafusion-ballista/pull/1652) (danielhumanmod)
-- feat: rest api supports plan tree rendering [#1650](https://github.com/apache/datafusion-ballista/pull/1650) (sandugood)
-- feat: Cache ballista clients on executor [#1578](https://github.com/apache/datafusion-ballista/pull/1578) (milenkovicm)
-- fix: compilation issue after merge [#1658](https://github.com/apache/datafusion-ballista/pull/1658) (milenkovicm)
-- [TUI] Show job's stages and their tasks [#1574](https://github.com/apache/datafusion-ballista/pull/1574) (martin-g)
-- [TUI] Configurable tick interval [#1669](https://github.com/apache/datafusion-ballista/pull/1669) (martin-g)
-- feat(aqe): Lazy stage evaluation in AQE [#1649](https://github.com/apache/datafusion-ballista/pull/1649) (milenkovicm)
-- docs: add Ballista TUI documentation [#1593](https://github.com/apache/datafusion-ballista/pull/1593) (goingforstudying-ctrl)
-- feat: move shuffle writer disk I/O off tokio worker threads [#1537](https://github.com/apache/datafusion-ballista/pull/1537) (hcrosse)
-- feat(scheduler): broadcast-style hash join for small-side joins [#1647](https://github.com/apache/datafusion-ballista/pull/1647) (andygrove)
-- feat: default to sort-merge join [#1651](https://github.com/apache/datafusion-ballista/pull/1651) (andygrove)
-- minor: add pending stage indicator for `AdaptiveDatafusionExec` [#1672](https://github.com/apache/datafusion-ballista/pull/1672) (milenkovicm)
-- fix: rest api calculates stage running time correctly [#1675](https://github.com/apache/datafusion-ballista/pull/1675) (milenkovicm)
-- minor: change parameter ordering in `AdaptivePlanner::try_new_with_optimizers` [#1687](https://github.com/apache/datafusion-ballista/pull/1687) (milenkovicm)
-- [TUI] Show executor's details in a popup [#1670](https://github.com/apache/datafusion-ballista/pull/1670) (martin-g)
-- Fix REST API panic on job list/detail when end_time < start_time [#1693](https://github.com/apache/datafusion-ballista/pull/1693) (abhinavgautam01)
-- Right align all numeric columns in the TUI tables [#1695](https://github.com/apache/datafusion-ballista/pull/1695) (martin-g)
-- fix(join-selection): guard CollectLeft swap when right has multiple partitions [#1691](https://github.com/apache/datafusion-ballista/pull/1691) (andygrove)
-- Minor improvements for #1675 [#1686](https://github.com/apache/datafusion-ballista/pull/1686) (martin-g)
-- [CLI/TUI] Use only tracing crate for logging in CLI and TUI [#1697](https://github.com/apache/datafusion-ballista/pull/1697) (martin-g)
-- minor: change log level for few statements [#1706](https://github.com/apache/datafusion-ballista/pull/1706) (milenkovicm)
-- Saturate scheduler job elapsed time [#1708](https://github.com/apache/datafusion-ballista/pull/1708) (MukundaKatta)
-- [TUI] Add a config setting for rendering job stage's plan as a tree [#1704](https://github.com/apache/datafusion-ballista/pull/1704) (martin-g)
-- [TUI] Add support for horizontal scrolling to the job/stage plan popups [#1711](https://github.com/apache/datafusion-ballista/pull/1711) (martin-g)
-- [TUI] Executor's id is not a numeric column. It should be center aligned [#1713](https://github.com/apache/datafusion-ballista/pull/1713) (martin-g)
-- [TUI] Add screenshots of the TUI application in README/cli.md [#1714](https://github.com/apache/datafusion-ballista/pull/1714) (martin-g)
-- Make use of Swatinem/rust-cache to make the CI workflows faster [#1705](https://github.com/apache/datafusion-ballista/pull/1705) (martin-g)
-- Merge Executor's brief and extended properties [#1716](https://github.com/apache/datafusion-ballista/pull/1716) (martin-g)
-- fix: REST API does not show running jobs [#1703](https://github.com/apache/datafusion-ballista/pull/1703) (gittihub-jpg)
-- [INFRA] Set up default rulesets for default and release branches [#1715](https://github.com/apache/datafusion-ballista/pull/1715) (asf-gitbox-commits)
-- feat: TUI shows running job information [#1717](https://github.com/apache/datafusion-ballista/pull/1717) (milenkovicm)
-- feat(aqe): CoalescePartitionsRule — shuffle-partition coalescing on resolved stats [#1684](https://github.com/apache/datafusion-ballista/pull/1684) (metegenez)
-- feat: TUI make task popup scrollable [#1725](https://github.com/apache/datafusion-ballista/pull/1725) (milenkovicm)
-- ci: add TPC-H SF10 workflow [#1688](https://github.com/apache/datafusion-ballista/pull/1688) (andygrove)
-- feat(tui): Use separate areas for the table and its associated scrollbar [#1729](https://github.com/apache/datafusion-ballista/pull/1729) (martin-g)
-- minor: [TUI] Extract a helper method for the table/scrollbar area splitter [#1730](https://github.com/apache/datafusion-ballista/pull/1730) (martin-g)
-- minor: [TUI] Sort the metrics before rendering them [#1731](https://github.com/apache/datafusion-ballista/pull/1731) (martin-g)
-- chore: Do not run -cli tests twice [#1732](https://github.com/apache/datafusion-ballista/pull/1732) (martin-g)
-- fix: no executor warning, correct prometheus feature name in TUI [#1698](https://github.com/apache/datafusion-ballista/pull/1698) (killzoner)
-
-## [52.0.0](https://github.com/apache/datafusion-ballista/tree/52.0.0) (2026-03-02)
-
-**Performance related:**
-
-- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove)
-
-**Implemented enhancements:**
-
-- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner)
-- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove)
-- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove)
-- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm)
-- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod)
-- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg)
-- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento)
-- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc)
-- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove)
-- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm)
-- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento)
-- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner)
-- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm)
-- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm)
-- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm)
-
-**Fixed bugs:**
-
-- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm)
-- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner)
-- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm)
-- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g)
-
-**Documentation updates:**
-
-- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove)
-- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove)
-- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove)
-- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento)
-
-**Merged pull requests:**
-
-- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner)
-- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove)
-- restrict python CI to python directory [#1383](https://github.com/apache/datafusion-ballista/pull/1383) (Huy1Ng)
-- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove)
-- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove)
-- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove)
-- doc: Add Ballista extensions example to the docs. [#1382](https://github.com/apache/datafusion-ballista/pull/1382) (LouisBurke)
-- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove)
-- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm)
-- doc: add more details for protobuf extension [#1393](https://github.com/apache/datafusion-ballista/pull/1393) (LouisBurke)
-- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod)
-- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove)
-- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg)
-- chore: update datafusion to 52 [#1394](https://github.com/apache/datafusion-ballista/pull/1394) (killzoner)
-- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento)
-- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm)
-- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc)
-- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento)
-- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner)
-- Improve sort-based shuffle: single spill file per partition and batch coalescing [#1431](https://github.com/apache/datafusion-ballista/pull/1431) (andygrove)
-- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove)
-- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm)
-- chore: add confirmation before tarball is released [#1445](https://github.com/apache/datafusion-ballista/pull/1445) (milenkovicm)
-- minor: add test to cover IPC arrow file read [#1450](https://github.com/apache/datafusion-ballista/pull/1450) (milenkovicm)
-- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento)
-- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner)
-- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm)
-- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm)
-- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm)
-- minor: improve log statements [#1482](https://github.com/apache/datafusion-ballista/pull/1482) (milenkovicm)
-- chore: update datafusion to 52.2 and other deps to latest [#1483](https://github.com/apache/datafusion-ballista/pull/1483) (milenkovicm)
-- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g)
-- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm)
-
-## [51.0.0](https://github.com/apache/datafusion-ballista/tree/51.0.0) (2026-01-11)
-
-**Implemented enhancements:**
-
-- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod)
-- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner)
-- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang)
-- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento)
-- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento)
-
-**Merged pull requests:**
-
-- minor: minor changes to release script/docs [#1342](https://github.com/apache/datafusion-ballista/pull/1342) (andygrove)
-- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod)
-- chore: update datafusion to 51.0 [#1345](https://github.com/apache/datafusion-ballista/pull/1345) (danielhumanmod)
-- doc: Add a note that datafusion may need to be downgraded after installing it [#1348](https://github.com/apache/datafusion-ballista/pull/1348) (martin-g)
-- minor: Make `DisplayAs` consistent and more readable for ShuffleExec [#1347](https://github.com/apache/datafusion-ballista/pull/1347) (milenkovicm)
-- minor: remove unnecessary clone functions  [#1352](https://github.com/apache/datafusion-ballista/pull/1352) (mmooyyii)
-- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner)
-- chore(doc): Clean up deployment docs. [#1354](https://github.com/apache/datafusion-ballista/pull/1354) (LouisBurke)
-- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang)
-- doc: Fix plan translation example to use correct aggregation and column [#1362](https://github.com/apache/datafusion-ballista/pull/1362) (mattcuento)
-- chore: update ballista version to 51.0.0 (from 50.0.0) [#1363](https://github.com/apache/datafusion-ballista/pull/1363) (milenkovicm)
-- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento)
-- chore: Add missing public API documentation/comments  [#1364](https://github.com/apache/datafusion-ballista/pull/1364) (killzoner)
-- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento)
-
-## [50.0.0](https://github.com/apache/datafusion-ballista/tree/50.0.0) (2025-11-04)
-
-**Implemented enhancements:**
-
-- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan)
-
-**Fixed bugs:**
-
-- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear)
-- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel)
-- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm)
-
-**Merged pull requests:**
-
-- infra: macos-13 is deprecated [#1324](https://github.com/apache/datafusion-ballista/pull/1324) (kevinjqliu)
-- chore: update to datafusion v50 [#1320](https://github.com/apache/datafusion-ballista/pull/1320) (milenkovicm)
-- chore: Update datafusion to 50.2 [#1326](https://github.com/apache/datafusion-ballista/pull/1326) (milenkovicm)
-- document private items and update docs CI [#1327](https://github.com/apache/datafusion-ballista/pull/1327) (killzoner)
-- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear)
-- add msrvcheck [#1328](https://github.com/apache/datafusion-ballista/pull/1328) (killzoner)
-- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel)
-- chore: Pinning versions of external actions. [#1334](https://github.com/apache/datafusion-ballista/pull/1334) (samueleresca)
-- chore: update python deps to 49 [#1335](https://github.com/apache/datafusion-ballista/pull/1335) (milenkovicm)
-- chore: update datafusion to 50.3 [#1336](https://github.com/apache/datafusion-ballista/pull/1336) (milenkovicm)
-- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm)
-- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan)
-- minor: change log level [#1340](https://github.com/apache/datafusion-ballista/pull/1340) (milenkovicm)
-
-## [49.0.0](https://github.com/apache/datafusion-ballista/tree/49.0.0) (2025-09-12)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/48.0.0...49.0.0)
-
-**Implemented enhancements:**
-
-- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm)
-- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay)
-
-**Fixed bugs:**
-
-- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm)
-- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez)
-- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm)
-- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm)
-- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm)
-- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved  [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm)
-
-**Merged pull requests:**
-
-- chore: update datafusion to 49 [#1285](https://github.com/apache/datafusion-ballista/pull/1285) (milenkovicm)
-- chore: Improve GitHub actions/python workflows [#1289](https://github.com/apache/datafusion-ballista/pull/1289) (Huy1Ng)
-- chore: update datafusion to 49.0.2 [#1298](https://github.com/apache/datafusion-ballista/pull/1298) (milenkovicm)
-- minor: make shuffle exec display consistent [#1299](https://github.com/apache/datafusion-ballista/pull/1299) (milenkovicm)
-- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm)
-- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez)
-- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm)
-- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm)
-- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm)
-- chore: notice and cargo deps cleanup [#1295](https://github.com/apache/datafusion-ballista/pull/1295) (milenkovicm)
-- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 [#1303](https://github.com/apache/datafusion-ballista/pull/1303) (dependabot[bot])
-- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 in /python [#1308](https://github.com/apache/datafusion-ballista/pull/1308) (dependabot[bot])
-- refactor: expand ClusterEventSender visibility to public [#1312](https://github.com/apache/datafusion-ballista/pull/1312) (Th824)
-- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved  [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm)
-- minor: enable json write test [#1311](https://github.com/apache/datafusion-ballista/pull/1311) (milenkovicm)
-- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm)
-- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay)
-- feat: Improve Remote Shuffle Read Speed and Resource Utilisation  [#1318](https://github.com/apache/datafusion-ballista/pull/1318) (milenkovicm)
-- fix: Issue with `JoinSelection` and `CrossJoinExec` when stages have been resoled [#1322](https://github.com/apache/datafusion-ballista/pull/1322) (ZihuanLing)
-
-## [48.0.0](https://github.com/apache/datafusion-ballista/tree/48.0.0) (2025-07-30)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/47.0.0...48.0.0)
-
-**Fixed bugs:**
-
-- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG)
-- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm)
-- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm)
-
-**Documentation updates:**
-
-- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm)
-- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone)
-
-**Merged pull requests:**
-
-- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm)
-- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone)
-- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG)
-- improve rust workflows without cache [#1275](https://github.com/apache/datafusion-ballista/pull/1275) (Huy1Ng)
-- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm)
-- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm)
-- chore: update python module to latest ballista release (v.47) [#1279](https://github.com/apache/datafusion-ballista/pull/1279) (milenkovicm)
-- chore: update datafusion to 48.0.1 [#1284](https://github.com/apache/datafusion-ballista/pull/1284) (milenkovicm)
-
-## [47.0.0](https://github.com/apache/datafusion-ballista/tree/47.0.0) (2025-06-15)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/46.0.0...47.0.0)
-
-**Implemented enhancements:**
-
-- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm)
-- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm)
-- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm)
-- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm)
-- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm)
-- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm)
-
-**Fixed bugs:**
-
-- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm)
-- fix: fix tests failing on windows [#1273](https://github.com/apache/datafusion-ballista/pull/1273) (Huy1Ng)
-
-**Merged pull requests:**
-
-- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm)
-- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm)
-- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm)
-- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm)
-- minor: release docker on when release has been tagged [#1264](https://github.com/apache/datafusion-ballista/pull/1264) (milenkovicm)
-- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm)
-- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm)
-- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm)
-- chore(deps): update to datafusion 47.0.0 [#1250](https://github.com/apache/datafusion-ballista/pull/1250) (milenkovicm)
-
-## [46.0.0](https://github.com/apache/datafusion-ballista/tree/46.0.0) (2025-05-05)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/45.0.0...46.0.0)
-
-**Implemented enhancements:**
-
-- feat: make task distribution policies pluggable  [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm)
-- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm)
-- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm)
-- feat: make execution_graph.stages()  public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm)
-
-**Fixed bugs:**
-
-- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii)
-
-**Merged pull requests:**
-
-- chore: Remove some arrow references [#1232](https://github.com/apache/datafusion-ballista/pull/1232) (andygrove)
-- chore: remove unused executor configuration option [#1229](https://github.com/apache/datafusion-ballista/pull/1229) (milenkovicm)
-- chore: return `404` for api requests if path does not exist [#1224](https://github.com/apache/datafusion-ballista/pull/1224) (milenkovicm)
-- chore(ci): replace `actions-rs` which are deprecated  [#1222](https://github.com/apache/datafusion-ballista/pull/1222) (milenkovicm)
-- minor: Decouple `ExecutionGraph` and `DistributedPlanner` [#1221](https://github.com/apache/datafusion-ballista/pull/1221) (milenkovicm)
-- chore: update datafusion to 46 [#1201](https://github.com/apache/datafusion-ballista/pull/1201) (milenkovicm)
-- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 [#1233](https://github.com/apache/datafusion-ballista/pull/1233) (dependabot[bot])
-- chore(deps): bump tokio from 1.44.1 to 1.44.2 [#1234](https://github.com/apache/datafusion-ballista/pull/1234) (dependabot[bot])
-- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii)
-- chore: update python deps to 45 [#1240](https://github.com/apache/datafusion-ballista/pull/1240) (milenkovicm)
-- Add S3 object store support to executor and scheduler [#1230](https://github.com/apache/datafusion-ballista/pull/1230) (milenkovicm)
-- feat: make task distribution policies pluggable  [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm)
-- chore: reduce log levels for few log statements [#1237](https://github.com/apache/datafusion-ballista/pull/1237) (milenkovicm)
-- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 in /python [#1244](https://github.com/apache/datafusion-ballista/pull/1244) (dependabot[bot])
-- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm)
-- minor: `executor_shutdown_while_running` test has race condition [#1248](https://github.com/apache/datafusion-ballista/pull/1248) (milenkovicm)
-- bug: build fails with `--no-default-features` [#1255](https://github.com/apache/datafusion-ballista/pull/1255) (milenkovicm)
-- doc: update architectural diagram [#1253](https://github.com/apache/datafusion-ballista/pull/1253) (milenkovicm)
-- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm)
-- chore: add read/write roundtrip tests [#1249](https://github.com/apache/datafusion-ballista/pull/1249) (milenkovicm)
-- minor: change log level for object store creation [#1247](https://github.com/apache/datafusion-ballista/pull/1247) (milenkovicm)
-- feat: make execution_graph.stages()  public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm)
-
-## [45.0.0](https://github.com/apache/datafusion-ballista/tree/45.0.0) (2025-03-30)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/44.0.0...45.0.0)
-
-**Implemented enhancements:**
-
-- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm)
-- feat: publish docker containers for executor and scheduler  [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm)
-
-**Merged pull requests:**
-
-- chore: minor release script fix [#1192](https://github.com/apache/datafusion-ballista/pull/1192) (andygrove)
-- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm)
-- chore: update datafusion to v45 [#1176](https://github.com/apache/datafusion-ballista/pull/1176) (milenkovicm)
-- chore: Update changelog for 44.0.0 [#1191](https://github.com/apache/datafusion-ballista/pull/1191) (andygrove)
-- minor: fix repo and homepage url in `cargo.toml` [#1196](https://github.com/apache/datafusion-ballista/pull/1196) (milenkovicm)
-- chore(deps): bump ring from 0.17.11 to 0.17.13 [#1199](https://github.com/apache/datafusion-ballista/pull/1199) (dependabot[bot])
-- feat: publish docker containers for executor and scheduler  [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm)
-- minor: make `graphviz-rust` dependency optional [#1203](https://github.com/apache/datafusion-ballista/pull/1203) (milenkovicm)
-- doc: update docker related documentation [#1204](https://github.com/apache/datafusion-ballista/pull/1204) (milenkovicm)
-- chore: update python dependencies [#1197](https://github.com/apache/datafusion-ballista/pull/1197) (milenkovicm)
-- chore(deps): bump ring from 0.17.8 to 0.17.14 in /python [#1206](https://github.com/apache/datafusion-ballista/pull/1206) (dependabot[bot])
-- doc: remove arrow from doc title [#1207](https://github.com/apache/datafusion-ballista/pull/1207) (milenkovicm)
-- Fix unit tests in tpch.rs [#1195](https://github.com/apache/datafusion-ballista/pull/1195) (vmingchen)
-- documentation :: quick-start.md sample source code correction [#1213](https://github.com/apache/datafusion-ballista/pull/1213) (nj7)
-- doc: fix quick-start executor command [#1217](https://github.com/apache/datafusion-ballista/pull/1217) (westhide)
-
-## [44.0.0](https://github.com/apache/datafusion-ballista/tree/44.0.0) (2025-03-01)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/43.0.0...44.0.0)
-
-**Implemented enhancements:**
-
-- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm)
-- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm)
-- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm)
-- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm)
-
-**Fixed bugs:**
-
-- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm)
-- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm)
-
-**Documentation updates:**
-
-- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm)
-
-**Merged pull requests:**
-
-- chore: planner cleanup and refactor [#1160](https://github.com/apache/datafusion-ballista/pull/1160) (milenkovicm)
-- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm)
-- chore: fix executor build issue on release [#1167](https://github.com/apache/datafusion-ballista/pull/1167) (milenkovicm)
-- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm)
-- chore: update to DF.44 [#1153](https://github.com/apache/datafusion-ballista/pull/1153) (milenkovicm)
-- chore: publicly expose datafusion in ballista client [#1169](https://github.com/apache/datafusion-ballista/pull/1169) (milenkovicm)
-- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm)
-- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm)
-- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm)
-- chore: generate change log for 44.0.0 [#1173](https://github.com/apache/datafusion-ballista/pull/1173) (milenkovicm)
-- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm)
-- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm)
-- chore: fix tpch data generator [#1186](https://github.com/apache/datafusion-ballista/pull/1186) (milenkovicm)
-- chore: fix clippy after rust 1.85 update [#1188](https://github.com/apache/datafusion-ballista/pull/1188) (milenkovicm)
-- chore: commit `Cargo.lock` file to make builds more predictable [#1190](https://github.com/apache/datafusion-ballista/pull/1190) (milenkovicm)
-
-## [43.0.0](<https://github.com/apache/datafusion-ballista/tree/43.0.0-rc2> (2025-01-07)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.12.0...43.0.0-rc2)
-
-**Implemented enhancements:**
-
-- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov)
-- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove)
-- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove)
-- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm)
-- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm)
-- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm)
-
-**Documentation updates:**
-
-- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd)
-- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove)
-- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins)
-- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov)
-- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins)
-- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski)
-- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove)
-- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove)
-- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove)
-- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove)
-- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm)
-- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm)
-- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm)
-- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4)
-- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm)
-- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove)
-
-**Merged pull requests:**
-
-- PyBallista - Python SQL client for Ballista [#970](https://github.com/apache/datafusion-ballista/pull/970) (andygrove)
-- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd)
-- [Python] Add `read_csv` and `read_parquet` methods [#976](https://github.com/apache/datafusion-ballista/pull/976) (andygrove)
-- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove)
-- [Python] Add more methods to SessionContext [#978](https://github.com/apache/datafusion-ballista/pull/978) (andygrove)
-- [Python] Add `execute_logical_plan` to context [#972](https://github.com/apache/datafusion-ballista/pull/972) (andygrove)
-- Use correct product name in docs [#975](https://github.com/apache/datafusion-ballista/pull/975) (andygrove)
-- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins)
-- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov)
-- Upgrade Rust version to 1.72 to keep the same as DataFusion v35 [#982](https://github.com/apache/datafusion-ballista/pull/982) (haoxins)
-- build: Fix the ballista-cli Dockerfile [#981](https://github.com/apache/datafusion-ballista/pull/981) (haoxins)
-- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov)
-- Remove some hard-coded gRPC max message sizes [#984](https://github.com/apache/datafusion-ballista/pull/984) (andygrove)
-- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins)
-- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski)
-- Fix ExecutorLost event debug info [#988](https://github.com/apache/datafusion-ballista/pull/988) (lewiszlw)
-- Fix shuffle writer test [#998](https://github.com/apache/datafusion-ballista/pull/998) (Jefffrey)
-- Bump graphviz-rust from 0.6.1 to 0.8.0 [#999](https://github.com/apache/datafusion-ballista/pull/999) (Jefffrey)
-- Add rust-toolchain.toml for clarity [#1014](https://github.com/apache/datafusion-ballista/pull/1014) (scnerd)
-- Fix executor metadata decode bug [#1004](https://github.com/apache/datafusion-ballista/pull/1004) (lewiszlw)
-- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove)
-- Fix Ballista rust.yml github workflow [#1026](https://github.com/apache/datafusion-ballista/pull/1026) (RaphaelMarinier)
-- Bump datafusion to 36.0.0 and make ballista compatible with it. [#1027](https://github.com/apache/datafusion-ballista/pull/1027) (RaphaelMarinier)
-- Make Ballista compatible with Datafusion 37.0.0 (from 36.0.0) [#1031](https://github.com/apache/datafusion-ballista/pull/1031) (RaphaelMarinier)
-- Fixes Setting Job Name Not Reflected in Ballista UI [#1039](https://github.com/apache/datafusion-ballista/pull/1039) (athultr1997)
-- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove)
-- [Docs] fix good_first_issue link in the contribution md doc [#1022](https://github.com/apache/datafusion-ballista/pull/1022) (Almaz-KG)
-- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove)
-- Fix job hangs when partition count of plan is zero [#1024](https://github.com/apache/datafusion-ballista/pull/1024) (lewiszlw)
-- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove)
-- Fix cargo build [#1045](https://github.com/apache/datafusion-ballista/pull/1045) (andygrove)
-- fix docker build in CI [#1046](https://github.com/apache/datafusion-ballista/pull/1046) (andygrove)
-- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove)
-- Bump actions/setup-node from 3 to 4 [#909](https://github.com/apache/datafusion-ballista/pull/909) (dependabot[bot])
-- Bump actions/cache from 3 to 4 [#958](https://github.com/apache/datafusion-ballista/pull/958) (dependabot[bot])
-- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove)
-- Update datafusion protobuf definitions [#1057](https://github.com/apache/datafusion-ballista/pull/1057) (palaska)
-- Fix regression with TPC-H benchmark [#1060](https://github.com/apache/datafusion-ballista/pull/1060) (andygrove)
-- Upgrade to Datafusion 41 [#1062](https://github.com/apache/datafusion-ballista/pull/1062) (palaska)
-- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm)
-- Remove plugin subsystem [#1070](https://github.com/apache/datafusion-ballista/pull/1070) (milenkovicm)
-- Remove CI folder [#1074](https://github.com/apache/datafusion-ballista/pull/1074) (milenkovicm)
-- Code cleanup, move examples, remove unused files [#1075](https://github.com/apache/datafusion-ballista/pull/1075) (milenkovicm)
-- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm)
-- Remove key-value stores for scheduler persistence [#1077](https://github.com/apache/datafusion-ballista/pull/1077) (milenkovicm)
-- Remove cache functionality [#1076](https://github.com/apache/datafusion-ballista/pull/1076) (milenkovicm)
-- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm)
-- Reorganise and remove dependencies [#1078](https://github.com/apache/datafusion-ballista/pull/1078) (milenkovicm)
-- Promote keda and flight-sql to optional features [#1079](https://github.com/apache/datafusion-ballista/pull/1079) (milenkovicm)
-- Update to datafusion 42 ... [#1080](https://github.com/apache/datafusion-ballista/pull/1080) (milenkovicm)
-- #1086 solve examples errors [#1087](https://github.com/apache/datafusion-ballista/pull/1087) (tbar4)
-- fix issue with not building python package ... [#1085](https://github.com/apache/datafusion-ballista/pull/1085) (milenkovicm)
-- another round of code cleanup ... [#1089](https://github.com/apache/datafusion-ballista/pull/1089) (milenkovicm)
-- Make rest-api optional feature ... [#1084](https://github.com/apache/datafusion-ballista/pull/1084) (milenkovicm)
-- fix clippy issues after updating to rust 1.82 [#1090](https://github.com/apache/datafusion-ballista/pull/1090) (milenkovicm)
-- Replace BallistaContext with SessionContext [#1088](https://github.com/apache/datafusion-ballista/pull/1088) (milenkovicm)
-- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4)
-- Ergonomic way to setup/configure `SessionContextExt` [#1096](https://github.com/apache/datafusion-ballista/pull/1096) (milenkovicm)
-- Executor configuration extended .. [#1099](https://github.com/apache/datafusion-ballista/pull/1099) (milenkovicm)
-- fix issue with executor registration ... [#1101](https://github.com/apache/datafusion-ballista/pull/1101) (milenkovicm)
-- Deprecate `BallistaContext` [#1103](https://github.com/apache/datafusion-ballista/pull/1103) (milenkovicm)
-- fix imports after un-rebased PR [#1106](https://github.com/apache/datafusion-ballista/pull/1106) (milenkovicm)
-- Ballista proto cleanup [#1110](https://github.com/apache/datafusion-ballista/pull/1110) (milenkovicm)
-- Update and move deps to workspace [#1109](https://github.com/apache/datafusion-ballista/pull/1109) (milenkovicm)
-- Trim down `BallistaConfig` [#1108](https://github.com/apache/datafusion-ballista/pull/1108) (milenkovicm)
-- Remove build-in object store registry [#1114](https://github.com/apache/datafusion-ballista/pull/1114) (milenkovicm)
-- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm)
-- support window functions [#1112](https://github.com/apache/datafusion-ballista/pull/1112) (onursatici)
-- added a BallistaContext to ballista to allow for Remote or standalone [#1100](https://github.com/apache/datafusion-ballista/pull/1100) (tbar4)
-- Decommission `BallistaContext` [#1119](https://github.com/apache/datafusion-ballista/pull/1119) (milenkovicm)
-- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove)
-- Make easier to create custom schedulers and executors [#1118](https://github.com/apache/datafusion-ballista/pull/1118) (milenkovicm)
-- refactor: Move BallistaRegistry to better location [#1126](https://github.com/apache/datafusion-ballista/pull/1126) (milenkovicm)
-- refactor: BallistaLogicalExtensionCodec refactoring and improvements [#1127](https://github.com/apache/datafusion-ballista/pull/1127) (milenkovicm)
-- refactor: consolidate ballista tests [#1129](https://github.com/apache/datafusion-ballista/pull/1129) (milenkovicm)
-- refactor: SessionStateExt and SessionConfigExt [#1130](https://github.com/apache/datafusion-ballista/pull/1130) (milenkovicm)
-- chore: dependancy updates [#1131](https://github.com/apache/datafusion-ballista/pull/1131) (milenkovicm)
-- chore: fix warning mimaloc warning when building [#1137](https://github.com/apache/datafusion-ballista/pull/1137) (milenkovicm)
-- refactor: SessionBuilder to return Result<\_> [#1138](https://github.com/apache/datafusion-ballista/pull/1138) (milenkovicm)
-- chore: remove unused cache\_ options from executor [#1140](https://github.com/apache/datafusion-ballista/pull/1140) (milenkovicm)
-- updated maturin version and ccargo build to build yml [#1136](https://github.com/apache/datafusion-ballista/pull/1136) (tbar4)
-- chore: Fix clippy issues after rust update (1.83.0) [#1143](https://github.com/apache/datafusion-ballista/pull/1143) (milenkovicm)
-- Fix documentation example which still uses BallistaContext [#1145](https://github.com/apache/datafusion-ballista/pull/1145) (milenkovicm)
-- Ballista proto cleanup [#1146](https://github.com/apache/datafusion-ballista/pull/1146) (milenkovicm)
-- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm)
-- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm)
-- chore: dependency cleanup [#1150](https://github.com/apache/datafusion-ballista/pull/1150) (milenkovicm)
-- Update DataFusion to 43 [#1125](https://github.com/apache/datafusion-ballista/pull/1125) (Dandandan)
-- Reinstantiate join order optimization [#1122](https://github.com/apache/datafusion-ballista/pull/1122) (Dandandan)
-- add partitioning scheme for unresolved shuffle and shuffle reader exec [#1144](https://github.com/apache/datafusion-ballista/pull/1144) (onursatici)
-- chore: update py-df to 43.1 [#1152](https://github.com/apache/datafusion-ballista/pull/1152) (milenkovicm)
-- chore: no need to run python test in rust [#1154](https://github.com/apache/datafusion-ballista/pull/1154) (milenkovicm)
-- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm)
-
-## [0.12.0](https://github.com/apache/datafusion-ballista/tree/0.12.0) (2024-01-14)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.11.0...0.12.0)
-
-**Documentation updates:**
-
-- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins)
-
-**Merged pull requests:**
-
-- [minor] remove outdate todo [#683](https://github.com/apache/datafusion-ballista/pull/683) (Ted-Jiang)
-- Add executor terminating status for graceful shutdown [#667](https://github.com/apache/datafusion-ballista/pull/667) (thinkharderdev)
-- Allow `BallistaContext::read_*` methods to read multiple paths. [#679](https://github.com/apache/datafusion-ballista/pull/679) (luckylsk34)
-- Update scheduler.md [#657](https://github.com/apache/datafusion-ballista/pull/657) (psvri)
-- Mark `SchedulerState` as pub [#688](https://github.com/apache/datafusion-ballista/pull/688) (Dandandan)
-- Update graphviz-rust requirement from 0.5.0 to 0.6.1 [#651](https://github.com/apache/datafusion-ballista/pull/651) (dependabot[bot])
-- Upgrade DataFusion to 19.0.0 [#691](https://github.com/apache/datafusion-ballista/pull/691) (r4ntix)
-- Update release docs [#692](https://github.com/apache/datafusion-ballista/pull/692) (andygrove)
-- Mark `SchedulerServer::with_task_launcher` as pub [#695](https://github.com/apache/datafusion-ballista/pull/695) (Dandandan)
-- Make task_manager pub [#698](https://github.com/apache/datafusion-ballista/pull/698) (Dandandan)
-- Add ExecutionEngine abstraction [#687](https://github.com/apache/datafusion-ballista/pull/687) (andygrove)
-- Allow accessing s3 locations in client mode [#700](https://github.com/apache/datafusion-ballista/pull/700) (luckylsk34)
-- git clone branch incorrect [#699](https://github.com/apache/datafusion-ballista/pull/699) (BubbaJoe)
-- Fix for error message during testing [#707](https://github.com/apache/datafusion-ballista/pull/707) (yahoNanJing)
-- Upgrade datafusion to 20.0.0 & sqlparser to to 0.32.0 [#711](https://github.com/apache/datafusion-ballista/pull/711) (r4ntix)
-- Update README.md [#729](https://github.com/apache/datafusion-ballista/pull/729) (jiangzhx)
-- Update link to scheduler proto file in dev docs [#713](https://github.com/apache/datafusion-ballista/pull/713) (JAicewizard)
-- Fix `show tables` fails [#715](https://github.com/apache/datafusion-ballista/pull/715) (r4ntix)
-- Remove redundant fields in ExecutorManager [#728](https://github.com/apache/datafusion-ballista/pull/728) (yahoNanJing)
-- Fix parameter '--config-backend' to '--cluster-backend' [#720](https://github.com/apache/datafusion-ballista/pull/720) (paolorechia)
-- Upgrade DataFusion to 21.0.0 [#727](https://github.com/apache/datafusion-ballista/pull/727) (r4ntix)
-- [minor] remove useless brackets [#739](https://github.com/apache/datafusion-ballista/pull/739) (Ted-Jiang)
-- Only decode plan in `LaunchMultiTaskParams` once [#743](https://github.com/apache/datafusion-ballista/pull/743) (Dandandan)
-- Upgrade DataFusion to 22.0.0 [#740](https://github.com/apache/datafusion-ballista/pull/740) (r4ntix)
-- [feature] support shuffle read with retry when facing IO error. [#738](https://github.com/apache/datafusion-ballista/pull/738) (Ted-Jiang)
-- [log] Print long running task status. [#750](https://github.com/apache/datafusion-ballista/pull/750) (Ted-Jiang)
-- Upgrade DataFusion to 23.0.0 [#755](https://github.com/apache/datafusion-ballista/pull/755) (yahoNanJing)
-- Fix plan metrics length and stage metrics length not match [#764](https://github.com/apache/datafusion-ballista/pull/764) (yahoNanJing)
-- added match arms to create ClusterStorageConfig [#766](https://github.com/apache/datafusion-ballista/pull/766) (BokarevNik)
-- [Improve] refactor the offer_reservation avoid wait result [#760](https://github.com/apache/datafusion-ballista/pull/760) (Ted-Jiang)
-- [fea] Avoid multithreaded write lock conflicts in event queue. [#754](https://github.com/apache/datafusion-ballista/pull/754) (Ted-Jiang)
-- Upgrade DataFusion to 24.0.0, Object_Store to 0.5.6 [#769](https://github.com/apache/datafusion-ballista/pull/769) (r4ntix)
-- Refine create_datafusion_context() [#778](https://github.com/apache/datafusion-ballista/pull/778) (yahoNanJing)
-- Remove output_partitioning for task definition [#776](https://github.com/apache/datafusion-ballista/pull/776) (yahoNanJing)
-- Upgrade DataFusion to 25.0.0 [#779](https://github.com/apache/datafusion-ballista/pull/779) (r4ntix)
-- Disable the ansi feature of tracing-subscriber [#784](https://github.com/apache/datafusion-ballista/pull/784) (yahoNanJing)
-- Add config grpc_server_max_decoding_message_size to make the maximum size of a decoded message at the grpc server side configurable [#782](https://github.com/apache/datafusion-ballista/pull/782) (yahoNanJing)
-- Fix nodejs issues in Docker build [#731](https://github.com/apache/datafusion-ballista/pull/731) (jnaous)
-- Upgrade node version to fix build in `main` [#794](https://github.com/apache/datafusion-ballista/pull/794) (avantgardnerio)
-- Remove redundant mod session_registry [#792](https://github.com/apache/datafusion-ballista/pull/792) (yahoNanJing)
-- Make last_seen_ts_threshold for getting alive executor at the scheduler side larger than the heartbeat time interval [#786](https://github.com/apache/datafusion-ballista/pull/786) (yahoNanJing)
-- Remove the prometheus-metrics from the default feature [#788](https://github.com/apache/datafusion-ballista/pull/788) (yahoNanJing)
-- Refine the ExecuteQuery grpc interface [#790](https://github.com/apache/datafusion-ballista/pull/790) (yahoNanJing)
-- Add config to collect statistics, enable in TPC-H benchmark [#796](https://github.com/apache/datafusion-ballista/pull/796) (Dandandan)
-- Add support for GCS data sources [#805](https://github.com/apache/datafusion-ballista/pull/805) (haoxins)
-- Update DataFusion to 26 [#798](https://github.com/apache/datafusion-ballista/pull/798) (Dandandan)
-- Issue 162 build docker image in ci [#716](https://github.com/apache/datafusion-ballista/pull/716) (paolorechia)
-- Fix index out of bounds panic [#819](https://github.com/apache/datafusion-ballista/pull/819) (yahoNanJing)
-- Refactor the TaskDefinition by changing encoding execution plan to the decoded one [#817](https://github.com/apache/datafusion-ballista/pull/817) (yahoNanJing)
-- Fix ballista-cli docs [#800](https://github.com/apache/datafusion-ballista/pull/800) (jonahgao)
-- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins)
-- Implement the with_new_children for ShuffleReaderExec [#821](https://github.com/apache/datafusion-ballista/pull/821) (yahoNanJing)
-- Update to point to the correct documentation [#838](https://github.com/apache/datafusion-ballista/pull/838) (dadepo)
-- Remove ExecutorReservation and change the task assignment philosophy from executor first to task first [#823](https://github.com/apache/datafusion-ballista/pull/823) (yahoNanJing)
-- Upgrade DataFusion to 27.0.0 [#834](https://github.com/apache/datafusion-ballista/pull/834) (r4ntix)
-- Reduce the number of calls to `create_logical_plan` [#842](https://github.com/apache/datafusion-ballista/pull/842) (jonahgao)
-- Bump semver from 5.7.1 to 5.7.2 in /ballista/scheduler/ui [#843](https://github.com/apache/datafusion-ballista/pull/843) (dependabot[bot])
-- Bump actions/labeler from 4.1.0 to 4.3.0 [#841](https://github.com/apache/datafusion-ballista/pull/841) (dependabot[bot])
-- Bump tough-cookie from 4.1.2 to 4.1.3 in /ballista/scheduler/ui [#840](https://github.com/apache/datafusion-ballista/pull/840) (dependabot[bot])
-- Update flatbuffers requirement from 22.9.29 to 23.5.26 [#801](https://github.com/apache/datafusion-ballista/pull/801) (dependabot[bot])
-- Update dirs requirement from 4.0.0 to 5.0.1 [#767](https://github.com/apache/datafusion-ballista/pull/767) (dependabot[bot])
-- Update libloading requirement from 0.7.3 to 0.8.0 [#761](https://github.com/apache/datafusion-ballista/pull/761) (dependabot[bot])
-- Introduce a cache crate supporting concurrent cache value loading [#825](https://github.com/apache/datafusion-ballista/pull/825) (yahoNanJing)
-- Fix cargo clippy for latest rust version [#848](https://github.com/apache/datafusion-ballista/pull/848) (yahoNanJing)
-- Introduce CachedBasedObjectStoreRegistry to use data source cache transparently [#827](https://github.com/apache/datafusion-ballista/pull/827) (yahoNanJing)
-- Add ConsistentHash for node topology management [#830](https://github.com/apache/datafusion-ballista/pull/830) (yahoNanJing)
-- Implement 3-phase consistent hash based task assignment policy [#833](https://github.com/apache/datafusion-ballista/pull/833) (yahoNanJing)
-- Update tonic requirement from 0.8 to 0.9 [#733](https://github.com/apache/datafusion-ballista/pull/733) (dependabot[bot])
-- Update itertools requirement from 0.10 to 0.11 [#844](https://github.com/apache/datafusion-ballista/pull/844) (dependabot[bot])
-- Update etcd-client requirement from 0.10 to 0.11 [#845](https://github.com/apache/datafusion-ballista/pull/845) (dependabot[bot])
-- Update hashbrown requirement from 0.13 to 0.14 [#846](https://github.com/apache/datafusion-ballista/pull/846) (dependabot[bot])
-- Bump word-wrap from 1.2.3 to 1.2.4 in /ballista/scheduler/ui [#849](https://github.com/apache/datafusion-ballista/pull/849) (dependabot[bot])
-- Update hdfs requirement from 0.1.1 to 0.1.4 [#856](https://github.com/apache/datafusion-ballista/pull/856) (yahoNanJing)
-- Update to DataFusion 28 [#858](https://github.com/apache/datafusion-ballista/pull/858) (Dandandan)
-- Upgrade datafusion to 30.0.0 [#866](https://github.com/apache/datafusion-ballista/pull/866) (r4ntix)
-- refactor: port get_scan_files to Ballista [#877](https://github.com/apache/datafusion-ballista/pull/877) (alamb)
-- Upgrade datafusion to 31.0.0 [#878](https://github.com/apache/datafusion-ballista/pull/878) (r4ntix)
-- Upgrade datafusion to 32.0.0 [#899](https://github.com/apache/datafusion-ballista/pull/899) (r4ntix)
-- Update to DataFusion 33 [#900](https://github.com/apache/datafusion-ballista/pull/900) (Dandandan)
-- Refactor lru mod, remove linked_hash_map [#918](https://github.com/apache/datafusion-ballista/pull/918) (PsiACE)
-- Dynamically optimize aggregate (count) based on shuffle stats [#919](https://github.com/apache/datafusion-ballista/pull/919) (Dandandan)
-- Use lz4 compression for shuffle files & flight stream, refactoring / improvements [#920](https://github.com/apache/datafusion-ballista/pull/920) (Dandandan)
-- Make max encoding message size configurable [#928](https://github.com/apache/datafusion-ballista/pull/928) (andygrove)
-- Set max message size to 16MB in gRPC clients [#931](https://github.com/apache/datafusion-ballista/pull/931) (andygrove)
-- Upgrade to DataFusion 34.0.0-rc1 [#927](https://github.com/apache/datafusion-ballista/pull/927) (andygrove)
-- Use official DF 34 release [#939](https://github.com/apache/datafusion-ballista/pull/939) (andygrove)
-- Use StreamWriter instead of FileWriter [#943](https://github.com/apache/datafusion-ballista/pull/943) (avantgardnerio)
-- Remove some TODO comments related to context fetching schemas from scheduler [#946](https://github.com/apache/datafusion-ballista/pull/946) (andygrove)
-- Fix Docker build [#947](https://github.com/apache/datafusion-ballista/pull/947) (andygrove)
-- Fix regression in DataFrame.write_xxx [#945](https://github.com/apache/datafusion-ballista/pull/945) (andygrove)
-
-## [0.11.0](https://github.com/apache/datafusion-ballista/tree/0.11.0) (2023-02-19)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.10.0...0.11.0)
-
-**Implemented enhancements:**
-
-- Remove `python` since it has been moved to its own repo, `datafusion-ballista-python` [\#653](https://github.com/apache/datafusion-ballista/issues/653)
-- Add executor self-registration mechanism in the heartbeat service [\#648](https://github.com/apache/datafusion-ballista/issues/648)
-- Upgrade to DataFusion 17 [\#638](https://github.com/apache/datafusion-ballista/issues/638)
-- Move Python bindings to separate repo? [\#635](https://github.com/apache/datafusion-ballista/issues/635)
-- Implement new release process [\#622](https://github.com/apache/datafusion-ballista/issues/622)
-- Change default branch name from master to main [\#618](https://github.com/apache/datafusion-ballista/issues/618)
-- Update latest datafusion dependency [\#610](https://github.com/apache/datafusion-ballista/issues/610)
-- Implement optimizer rule to remove redundant repartitioning [\#608](https://github.com/apache/datafusion-ballista/issues/608)
-- ballista-cli as \(docker\) images [\#600](https://github.com/apache/datafusion-ballista/issues/600)
-- Update contributor guide [\#598](https://github.com/apache/datafusion-ballista/issues/598)
-- Fix cargo clippy [\#570](https://github.com/apache/datafusion-ballista/issues/570)
-- Support Alibaba Cloud OSS with ObjectStore [\#566](https://github.com/apache/datafusion-ballista/issues/566)
-- Refactor `StateBackendClient` to be a higher-level interface [\#554](https://github.com/apache/datafusion-ballista/issues/554)
-- Make it concurrently to launch tasks to executors [\#544](https://github.com/apache/datafusion-ballista/issues/544)
-- Simplify docs [\#531](https://github.com/apache/datafusion-ballista/issues/531)
-- Provide an in-memory StateBackend [\#505](https://github.com/apache/datafusion-ballista/issues/505)
-- Add support for Azure blob storage [\#294](https://github.com/apache/datafusion-ballista/issues/294)
-- Add a workflow to build the image and publish it to the package [\#71](https://github.com/apache/datafusion-ballista/issues/71)
-
-**Fixed bugs:**
-
-- Rust / Check Cargo.toml formatting \(amd64, stable\) \(pull_request\) Failing [\#662](https://github.com/apache/datafusion-ballista/issues/662)
-- Protobuf parsing error [\#646](https://github.com/apache/datafusion-ballista/issues/646)
-- jobs from python client not showing up in Scheduler UI [\#625](https://github.com/apache/datafusion-ballista/issues/625)
-- ballista ui fails to build [\#594](https://github.com/apache/datafusion-ballista/issues/594)
-- cargo build --release fails for ballista-scheduler [\#590](https://github.com/apache/datafusion-ballista/issues/590)
-- docker build fails [\#589](https://github.com/apache/datafusion-ballista/issues/589)
-- Multi-scheduler Job Starvation [\#585](https://github.com/apache/datafusion-ballista/issues/585)
-- Cannot query file from S3 [\#559](https://github.com/apache/datafusion-ballista/issues/559)
-- Benchmark q16 fails [\#373](https://github.com/apache/datafusion-ballista/issues/373)
-
-**Documentation updates:**
-
-- Check in benchmark image [\#647](https://github.com/apache/datafusion-ballista/pull/647) ([andygrove](https://github.com/andygrove))
-- MINOR: Fix benchmark image link [\#596](https://github.com/apache/datafusion-ballista/pull/596) ([andygrove](https://github.com/andygrove))
-
-**Merged pull requests:**
-
-- Upgrade to DataFusion 18 [\#668](https://github.com/apache/datafusion-ballista/pull/668) ([andygrove](https://github.com/andygrove))
-- Enable physical plan round-trip tests [\#666](https://github.com/apache/datafusion-ballista/pull/666) ([andygrove](https://github.com/andygrove))
-- Upgrade to DataFusion 18.0.0-rc1 [\#664](https://github.com/apache/datafusion-ballista/pull/664) ([andygrove](https://github.com/andygrove))
-- add test_util to make examples work well [\#661](https://github.com/apache/datafusion-ballista/pull/661) ([jiangzhx](https://github.com/jiangzhx))
-- Minor refactor to reduce duplicate code [\#659](https://github.com/apache/datafusion-ballista/pull/659) ([andygrove](https://github.com/andygrove))
-- Cluster state refactor Part 2 [\#658](https://github.com/apache/datafusion-ballista/pull/658) ([thinkharderdev](https://github.com/thinkharderdev))
-- Remove `python` dir & python-related workflows [\#654](https://github.com/apache/datafusion-ballista/pull/654) ([iajoiner](https://github.com/iajoiner))
-- Add executor self-registration mechanism in the heartbeat service [\#649](https://github.com/apache/datafusion-ballista/pull/649) ([yahoNanJing](https://github.com/yahoNanJing))
-- Upgrade to DataFusion 17 [\#639](https://github.com/apache/datafusion-ballista/pull/639) ([avantgardnerio](https://github.com/avantgardnerio))
-- Upgrade to DataFusion 16 \(again\) [\#636](https://github.com/apache/datafusion-ballista/pull/636) ([avantgardnerio](https://github.com/avantgardnerio))
-- Update release process documentation [\#632](https://github.com/apache/datafusion-ballista/pull/632) ([andygrove](https://github.com/andygrove))
-- Implement new release process [\#623](https://github.com/apache/datafusion-ballista/pull/623) ([andygrove](https://github.com/andygrove))
-- Update contributor guide [\#617](https://github.com/apache/datafusion-ballista/pull/617) ([andygrove](https://github.com/andygrove))
-- Fix Cargo.toml format issue [\#616](https://github.com/apache/datafusion-ballista/pull/616) ([andygrove](https://github.com/andygrove))
-- Refactor scheduler main [\#615](https://github.com/apache/datafusion-ballista/pull/615) ([andygrove](https://github.com/andygrove))
-- Refactor executor main [\#614](https://github.com/apache/datafusion-ballista/pull/614) ([andygrove](https://github.com/andygrove))
-- Update datafusion dependency to the latest version [\#612](https://github.com/apache/datafusion-ballista/pull/612) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add support for Azure Blob Storage [\#599](https://github.com/apache/datafusion-ballista/pull/599) ([aidankovacic-8451](https://github.com/aidankovacic-8451))
-- Python: add method to get explain output as a string [\#593](https://github.com/apache/datafusion-ballista/pull/593) ([andygrove](https://github.com/andygrove))
-- Handle job resubmission [\#586](https://github.com/apache/datafusion-ballista/pull/586) ([thinkharderdev](https://github.com/thinkharderdev))
-- updated readme to contain correct versions of dependencies. [\#580](https://github.com/apache/datafusion-ballista/pull/580) ([saikrishna1-bidgely](https://github.com/saikrishna1-bidgely))
-- Update graphviz-rust requirement from 0.4.0 to 0.5.0 [\#574](https://github.com/apache/datafusion-ballista/pull/574) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Super minor spelling error [\#573](https://github.com/apache/datafusion-ballista/pull/573) ([jdye64](https://github.com/jdye64))
-- Fix cargo clippy [\#571](https://github.com/apache/datafusion-ballista/pull/571) ([yahoNanJing](https://github.com/yahoNanJing))
-- Support Alibaba Cloud OSS with ObjectStore [\#567](https://github.com/apache/datafusion-ballista/pull/567) ([r4ntix](https://github.com/r4ntix))
-- fix\(ui\): fix last seen [\#562](https://github.com/apache/datafusion-ballista/pull/562) ([duyet](https://github.com/duyet))
-- Cluster state refactor part 1 [\#560](https://github.com/apache/datafusion-ballista/pull/560) ([thinkharderdev](https://github.com/thinkharderdev))
-- Make it concurrently to launch tasks to executors [\#557](https://github.com/apache/datafusion-ballista/pull/557) ([yahoNanJing](https://github.com/yahoNanJing))
-- Update datafusion requirement from 14.0.0 to 15.0.0 [\#552](https://github.com/apache/datafusion-ballista/pull/552) ([yahoNanJing](https://github.com/yahoNanJing))
-- docs: fix style in the Helm readme [\#551](https://github.com/apache/datafusion-ballista/pull/551) ([haoxins](https://github.com/haoxins))
-- Fix Helm chart's image format [\#550](https://github.com/apache/datafusion-ballista/pull/550) ([haoxins](https://github.com/haoxins))
-- Update env_logger requirement from 0.9 to 0.10 [\#539](https://github.com/apache/datafusion-ballista/pull/539) ([dependabot[bot]](https://github.com/apps/dependabot))
-- only build docker images on rc tags [\#535](https://github.com/apache/datafusion-ballista/pull/535) ([andygrove](https://github.com/andygrove))
-- Remove `--locked` when building Python wheels [\#533](https://github.com/apache/datafusion-ballista/pull/533) ([andygrove](https://github.com/andygrove))
-- Bump actions/labeler from 4.0.2 to 4.1.0 [\#525](https://github.com/apache/datafusion-ballista/pull/525) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Provide a memory StateBackendClient [\#523](https://github.com/apache/datafusion-ballista/pull/523) ([yahoNanJing](https://github.com/yahoNanJing))
-
-## [0.10.0](https://github.com/apache/datafusion-ballista/tree/0.10.0) (2022-11-18)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.9.0...0.10.0)
-
-**Implemented enhancements:**
-
-- Add user guide section on prometheus metrics [\#507](https://github.com/apache/datafusion-ballista/issues/507)
-- Don't throw error when job path not exist in remove_job_data [\#502](https://github.com/apache/datafusion-ballista/issues/502)
-- Fix clippy warning [\#494](https://github.com/apache/datafusion-ballista/issues/494)
-- Use job_data_clean_up_interval_seconds == 0 to indicate executor_cleanup_enable [\#488](https://github.com/apache/datafusion-ballista/issues/488)
-- Add a config for tracing log rolling policy for both scheduler and executor [\#486](https://github.com/apache/datafusion-ballista/issues/486)
-- Set up repo where we can push benchmark results [\#473](https://github.com/apache/datafusion-ballista/issues/473)
-- Make the delayed time interval for cleanup job data in both scheduler and executor configurable [\#469](https://github.com/apache/datafusion-ballista/issues/469)
-- Add some validation for the remove_job_data grpc service [\#467](https://github.com/apache/datafusion-ballista/issues/467)
-- Add ability to build docker images using `release-lto` profile [\#463](https://github.com/apache/datafusion-ballista/issues/463)
-- Suggest users download \(rather than build\) the FlightSQL JDBC Driver [\#460](https://github.com/apache/datafusion-ballista/issues/460)
-- Clean up legacy job shuffle data [\#459](https://github.com/apache/datafusion-ballista/issues/459)
-- Add grpc service for the scheduler to make it able to be triggered by client explicitly [\#458](https://github.com/apache/datafusion-ballista/issues/458)
-- Replace Mutex\<HashMap\> by using DashMap [\#448](https://github.com/apache/datafusion-ballista/issues/448)
-- Refine log level [\#446](https://github.com/apache/datafusion-ballista/issues/446)
-- Upgrade to DataFusion 14.0.0 [\#445](https://github.com/apache/datafusion-ballista/issues/445)
-- Add a feature for hdfs3 [\#419](https://github.com/apache/datafusion-ballista/issues/419)
-- Add optional flag which advertises host for Arrow Flight SQL [\#418](https://github.com/apache/datafusion-ballista/issues/418)
-- Partitioning reasoning in DataFusion and Ballista [\#284](https://github.com/apache/datafusion-ballista/issues/284)
-- Stop wasting time in CI on MIRI runs [\#283](https://github.com/apache/datafusion-ballista/issues/283)
-- Publish Docker images as part of each release [\#236](https://github.com/apache/datafusion-ballista/issues/236)
-- Cleanup job/stage status from TaskManager and clean up shuffle data after a period after JobFinished [\#185](https://github.com/apache/datafusion-ballista/issues/185)
-
-**Fixed bugs:**
-
-- build broken: configure_me_codegen retroactively reserved `bind_host` [\#519](https://github.com/apache/datafusion-ballista/issues/519)
-- Return empty results for SQLs with order by [\#451](https://github.com/apache/datafusion-ballista/issues/451)
-- ballista scheduler is not taken inline parameters into account [\#443](https://github.com/apache/datafusion-ballista/issues/443)
-- \[FlightSQL\] Cannot connect with Tableau Desktop [\#428](https://github.com/apache/datafusion-ballista/issues/428)
-- Benchmark q15 fails [\#372](https://github.com/apache/datafusion-ballista/issues/372)
-- Incorrect documentation for building Ballista on Linux when using docker-compose [\#362](https://github.com/apache/datafusion-ballista/issues/362)
-- Scheduler silently replaces `ParquetExec` with `EmptyExec` if data path is not correctly mounted in container [\#353](https://github.com/apache/datafusion-ballista/issues/353)
-- SQL with order by limit returns nothing [\#334](https://github.com/apache/datafusion-ballista/issues/334)
-
-**Documentation updates:**
-
-- README updates [\#433](https://github.com/apache/datafusion-ballista/pull/433) ([andygrove](https://github.com/andygrove))
-
-**Merged pull requests:**
-
-- configure_me_codegen retroactively reserved on our `bind_host` parame… [\#520](https://github.com/apache/datafusion-ballista/pull/520) ([avantgardnerio](https://github.com/avantgardnerio))
-- Bump actions/cache from 2 to 3 [\#517](https://github.com/apache/datafusion-ballista/pull/517) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Update graphviz-rust requirement from 0.3.0 to 0.4.0 [\#515](https://github.com/apache/datafusion-ballista/pull/515) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Add Prometheus metrics endpoint [\#511](https://github.com/apache/datafusion-ballista/pull/511) ([thinkharderdev](https://github.com/thinkharderdev))
-- Enable tests that work since upgrading to DataFusion 14 [\#510](https://github.com/apache/datafusion-ballista/pull/510) ([andygrove](https://github.com/andygrove))
-- Update hashbrown requirement from 0.12 to 0.13 [\#506](https://github.com/apache/datafusion-ballista/pull/506) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Don't throw error when job shuffle data path not exist in executor [\#503](https://github.com/apache/datafusion-ballista/pull/503) ([yahoNanJing](https://github.com/yahoNanJing))
-- Upgrade to DataFusion 14.0.0 and Arrow 26.0.0 [\#499](https://github.com/apache/datafusion-ballista/pull/499) ([andygrove](https://github.com/andygrove))
-- Fix clippy warning [\#495](https://github.com/apache/datafusion-ballista/pull/495) ([yahoNanJing](https://github.com/yahoNanJing))
-- Stop wasting time in CI on MIRI runs [\#491](https://github.com/apache/datafusion-ballista/pull/491) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Remove executor config executor_cleanup_enable and make the configuation name for executor cleanup more intuitive [\#489](https://github.com/apache/datafusion-ballista/pull/489) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add a config for tracing log rolling policy for both scheduler and executor [\#487](https://github.com/apache/datafusion-ballista/pull/487) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add grpc service of cleaning up job shuffle data for the scheduler to make it able to be triggered by client explicitly [\#485](https://github.com/apache/datafusion-ballista/pull/485) ([yahoNanJing](https://github.com/yahoNanJing))
-- \[Minor\] Bump DataFusion [\#480](https://github.com/apache/datafusion-ballista/pull/480) ([Dandandan](https://github.com/Dandandan))
-- Remove benchmark results from README [\#478](https://github.com/apache/datafusion-ballista/pull/478) ([andygrove](https://github.com/andygrove))
-- Update `flightsql.md` to provide correct instruction [\#476](https://github.com/apache/datafusion-ballista/pull/476) ([iajoiner](https://github.com/iajoiner))
-- Add support for Tableau [\#475](https://github.com/apache/datafusion-ballista/pull/475) ([avantgardnerio](https://github.com/avantgardnerio))
-- Add SchedulerConfig for the scheduler configurations, like event_loop_buffer_size, finished_job_data_clean_up_interval_seconds, finished_job_state_clean_up_interval_seconds [\#472](https://github.com/apache/datafusion-ballista/pull/472) ([yahoNanJing](https://github.com/yahoNanJing))
-- Bump DataFusion [\#471](https://github.com/apache/datafusion-ballista/pull/471) ([Dandandan](https://github.com/Dandandan))
-- Add some validation for remove_job_data in the executor server [\#468](https://github.com/apache/datafusion-ballista/pull/468) ([yahoNanJing](https://github.com/yahoNanJing))
-- Update documentation to reflect the release of the FlightSQL JDBC Driver [\#461](https://github.com/apache/datafusion-ballista/pull/461) ([avantgardnerio](https://github.com/avantgardnerio))
-- Bump DataFusion version [\#453](https://github.com/apache/datafusion-ballista/pull/453) ([andygrove](https://github.com/andygrove))
-- Add shuffle for SortPreservingMergeExec physical operator [\#452](https://github.com/apache/datafusion-ballista/pull/452) ([yahoNanJing](https://github.com/yahoNanJing))
-- Replace Mutex\<HashMap\> by using DashMap [\#449](https://github.com/apache/datafusion-ballista/pull/449) ([yahoNanJing](https://github.com/yahoNanJing))
-- Refine log level for trial info and periodically invoked places [\#447](https://github.com/apache/datafusion-ballista/pull/447) ([yahoNanJing](https://github.com/yahoNanJing))
-- MINOR: Add `set -e` to scripts, fix a typo [\#444](https://github.com/apache/datafusion-ballista/pull/444) ([andygrove](https://github.com/andygrove))
-- Add optional flag which advertises host for Arrow Flight SQL \#418 [\#442](https://github.com/apache/datafusion-ballista/pull/442) ([DaltonModlin](https://github.com/DaltonModlin))
-- Reorder joins after resolving stage inputs [\#441](https://github.com/apache/datafusion-ballista/pull/441) ([Dandandan](https://github.com/Dandandan))
-- Add a feature for hdfs3 [\#439](https://github.com/apache/datafusion-ballista/pull/439) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add Spark benchmarks [\#438](https://github.com/apache/datafusion-ballista/pull/438) ([andygrove](https://github.com/andygrove))
-- scheduler now verifies that `file://` ListingTable URLs are accessible [\#414](https://github.com/apache/datafusion-ballista/pull/414) ([andygrove](https://github.com/andygrove))
-
-## [0.9.0](https://github.com/apache/datafusion-ballista/tree/0.9.0) (2022-10-22)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.8.0...0.9.0)
-
-**Implemented enhancements:**
-
-- Support count distinct aggregation function [\#411](https://github.com/apache/datafusion-ballista/issues/411)
-- Use multi-task definition in pull-based execution loop [\#400](https://github.com/apache/datafusion-ballista/issues/400)
-- Make the scheduler event loop buffer size configurable [\#397](https://github.com/apache/datafusion-ballista/issues/397)
-- Remove active execution graph when the related job is successful or failed. [\#391](https://github.com/apache/datafusion-ballista/issues/391)
-- Improve launch task efficiency by calling LaunchMultiTask [\#389](https://github.com/apache/datafusion-ballista/issues/389)
-- Use `tokio::sync::Semaphore` to wait for available task slots [\#388](https://github.com/apache/datafusion-ballista/issues/388)
-- stdout and file log level settings are inconsistent [\#385](https://github.com/apache/datafusion-ballista/issues/385)
-- Use dedicated executor in pull based loop [\#383](https://github.com/apache/datafusion-ballista/issues/383)
-- Avoid calling scheduler when the executor cannot accept new tasks [\#377](https://github.com/apache/datafusion-ballista/issues/377)
-- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#371](https://github.com/apache/datafusion-ballista/issues/371)
-- Switch to mimalloc and enable by default [\#369](https://github.com/apache/datafusion-ballista/issues/369)
-- Integration test script should use docker-compose [\#364](https://github.com/apache/datafusion-ballista/issues/364)
-- Use local shuffle reader in containerized environments [\#356](https://github.com/apache/datafusion-ballista/issues/356)
-- Add `--ext` option to benchmark [\#352](https://github.com/apache/datafusion-ballista/issues/352)
-- Add job cancel in the UI [\#350](https://github.com/apache/datafusion-ballista/issues/350)
-- Using local shuffle reader avoid flight rpc call. [\#346](https://github.com/apache/datafusion-ballista/issues/346)
-- Add a Helm Chart [\#321](https://github.com/apache/datafusion-ballista/issues/321)
-- \[UI\] Show list of query stages with metrics [\#306](https://github.com/apache/datafusion-ballista/issues/306)
-- \[UI\] Add ability to specify job name and have it show in the job listing page in the UI [\#277](https://github.com/apache/datafusion-ballista/issues/277)
-- \[UI\] Add ability to download query plans in dot format [\#276](https://github.com/apache/datafusion-ballista/issues/276)
-- \[UI\] Add ability to render query plans [\#275](https://github.com/apache/datafusion-ballista/issues/275)
-- Add REST API documentation to User Guide [\#272](https://github.com/apache/datafusion-ballista/issues/272)
-- Graceful shutdown: Handle `SIGTERM` [\#266](https://github.com/apache/datafusion-ballista/issues/266)
-- \[EPIC\] Scheduler UI [\#265](https://github.com/apache/datafusion-ballista/issues/265)
-- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#259](https://github.com/apache/datafusion-ballista/issues/259)
-- Add a feature based object store provider [\#257](https://github.com/apache/datafusion-ballista/issues/257)
-- Add docker build files [\#248](https://github.com/apache/datafusion-ballista/issues/248)
-- Allow IDEs to recognize generated code [\#246](https://github.com/apache/datafusion-ballista/issues/246)
-- Add user guide section on Flight SQL support [\#230](https://github.com/apache/datafusion-ballista/issues/230)
-- `dev/release/README.md` is outdated [\#228](https://github.com/apache/datafusion-ballista/issues/228)
-- Make ShuffleReaderExec output less verbose [\#211](https://github.com/apache/datafusion-ballista/issues/211)
-- Add LaunchMultiTask rpc interface for executor [\#209](https://github.com/apache/datafusion-ballista/issues/209)
-- Make executor fetch shuffle partition data in parallel [\#208](https://github.com/apache/datafusion-ballista/issues/208)
-- Concurrency control and rate limit during shuffle reader [\#195](https://github.com/apache/datafusion-ballista/issues/195)
-- Update User Guide [\#160](https://github.com/apache/datafusion-ballista/issues/160)
-- Ballista 0.8.0 Release [\#159](https://github.com/apache/datafusion-ballista/issues/159)
-- Save encoded execution plan in the ExecutionStage to reduce cost of task serialization and deserialization [\#142](https://github.com/apache/datafusion-ballista/issues/142)
-- Failed task retry [\#140](https://github.com/apache/datafusion-ballista/issues/140)
-- Redefine the executor task slots [\#132](https://github.com/apache/datafusion-ballista/issues/132)
-- Use ArrowFlight bearer token auth to create a session key for FlightSql clients [\#112](https://github.com/apache/datafusion-ballista/issues/112)
-- Leverage Atomic for the in-memory states in Scheduler [\#101](https://github.com/apache/datafusion-ballista/issues/101)
-- Introduce the object stores in datafusion-contrib as optional features [\#87](https://github.com/apache/datafusion-ballista/issues/87)
-- Support multiple paths for ListingTableScanNode [\#75](https://github.com/apache/datafusion-ballista/issues/75)
-- Need clean up intermediate data in Ballista [\#9](https://github.com/apache/datafusion-ballista/issues/9)
-- Ballista does not support external file systems [\#10](https://github.com/apache/datafusion-ballista/issues/10)
-
-**Fixed bugs:**
-
-- Build errors in ./dev/build-ballista-rust.sh [\#407](https://github.com/apache/datafusion-ballista/issues/407)
-- The Ballista Scheduler Dockerfile copies a file that no longer exists [\#402](https://github.com/apache/datafusion-ballista/issues/402)
-- Benchmark q20 fails [\#374](https://github.com/apache/datafusion-ballista/issues/374)
-- Integration tests fail [\#360](https://github.com/apache/datafusion-ballista/issues/360)
-- Helm deploy fails [\#344](https://github.com/apache/datafusion-ballista/issues/344)
-- Executor get stopped unexpected [\#333](https://github.com/apache/datafusion-ballista/issues/333)
-- Executor poll work loop failure [\#311](https://github.com/apache/datafusion-ballista/issues/311)
-- Queries with `LIMIT` are failing with "PhysicalExtensionCodec is not provided" [\#300](https://github.com/apache/datafusion-ballista/issues/300)
-- Schema inference does not work in Ballista-cli with a remote context [\#287](https://github.com/apache/datafusion-ballista/issues/287)
-- There are bugs in the yarn build github misses but break our internal build [\#270](https://github.com/apache/datafusion-ballista/issues/270)
-- Race condition running docker-compose [\#267](https://github.com/apache/datafusion-ballista/issues/267)
-- Scheduler UI not working in Docker image [\#250](https://github.com/apache/datafusion-ballista/issues/250)
-- Use bind host rather than the external host for starting a local executor service [\#244](https://github.com/apache/datafusion-ballista/issues/244)
-- Initial query stages read parquet files and repartition them needlessly [\#243](https://github.com/apache/datafusion-ballista/issues/243)
-- Cannot build Docker images on macOS 12.5.1 with M1 chip [\#234](https://github.com/apache/datafusion-ballista/issues/234)
-- CLI uses DataFusion context if no host or port are provided [\#219](https://github.com/apache/datafusion-ballista/issues/219)
-- Unsupported binary operator `StringConcat` [\#201](https://github.com/apache/datafusion-ballista/issues/201)
-- Ballista assumes all aggregate expressions are not DISTINCT [\#5](https://github.com/apache/datafusion-ballista/issues/5)
-- Start ballista ui with docker, but it can not found ballista scheduler [\#11](https://github.com/apache/datafusion-ballista/issues/11)
-- Cannot build Ballista docker images on Apple silicon [\#17](https://github.com/apache/datafusion-ballista/issues/17)
-
-**Documentation updates:**
-
-- Fixup links in README.md [\#366](https://github.com/apache/datafusion-ballista/pull/366) ([romanz](https://github.com/romanz))
-- Update README in preparation for 0.9.0 release [\#318](https://github.com/apache/datafusion-ballista/pull/318) ([andygrove](https://github.com/andygrove))
-- User Guide improvements [\#274](https://github.com/apache/datafusion-ballista/pull/274) ([andygrove](https://github.com/andygrove))
-
-**Closed issues:**
-
-- Automatic version updates for github actions with dependabot [\#127](https://github.com/apache/datafusion-ballista/issues/127)
-
-**Merged pull requests:**
-
-- Return multiple tasks in poll_work based on free slots [\#429](https://github.com/apache/datafusion-ballista/pull/429) ([Dandandan](https://github.com/Dandandan))
-- Run integration tests as part of release verification script [\#426](https://github.com/apache/datafusion-ballista/pull/426) ([andygrove](https://github.com/andygrove))
-- Bump actions/setup-node from 2 to 3 [\#424](https://github.com/apache/datafusion-ballista/pull/424) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump actions/setup-python from 2 to 4 [\#423](https://github.com/apache/datafusion-ballista/pull/423) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump actions/checkout from 2 to 3 [\#422](https://github.com/apache/datafusion-ballista/pull/422) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump actions/download-artifact from 2 to 3 [\#421](https://github.com/apache/datafusion-ballista/pull/421) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump actions/upload-artifact from 2 to 3 [\#420](https://github.com/apache/datafusion-ballista/pull/420) ([dependabot[bot]](https://github.com/apps/dependabot))
-- MINOR: Fix yarn warnings [\#415](https://github.com/apache/datafusion-ballista/pull/415) ([andygrove](https://github.com/andygrove))
-- Fix q20 sql typo in benchmarks [\#409](https://github.com/apache/datafusion-ballista/pull/409) ([r4ntix](https://github.com/r4ntix))
-- MINOR: Add notes on Apache Reporter [\#401](https://github.com/apache/datafusion-ballista/pull/401) ([andygrove](https://github.com/andygrove))
-- Use local shuffle reader in containerized environments and some impro… [\#399](https://github.com/apache/datafusion-ballista/pull/399) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Make the scheduler event loop buffer size configurable [\#398](https://github.com/apache/datafusion-ballista/pull/398) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add RoundRobinLocal slots policy for caching executor data to avoid seld persistency [\#396](https://github.com/apache/datafusion-ballista/pull/396) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#395](https://github.com/apache/datafusion-ballista/pull/395) ([yahoNanJing](https://github.com/yahoNanJing))
-- Improve launch task efficiency by calling LaunchMultiTask [\#394](https://github.com/apache/datafusion-ballista/pull/394) ([yahoNanJing](https://github.com/yahoNanJing))
-- Cache encoded stage plan [\#393](https://github.com/apache/datafusion-ballista/pull/393) ([yahoNanJing](https://github.com/yahoNanJing))
-- Remove active execution graph when the related job is successful or failed [\#392](https://github.com/apache/datafusion-ballista/pull/392) ([yahoNanJing](https://github.com/yahoNanJing))
-- Update flatbuffers requirement from 2.1.2 to 22.9.29 [\#390](https://github.com/apache/datafusion-ballista/pull/390) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Unified the log level configuration behavior [\#386](https://github.com/apache/datafusion-ballista/pull/386) ([r4ntix](https://github.com/r4ntix))
-- Add DistinctCount support [\#384](https://github.com/apache/datafusion-ballista/pull/384) ([r4ntix](https://github.com/r4ntix))
-- Pull-based execution loop improvements [\#380](https://github.com/apache/datafusion-ballista/pull/380) ([Dandandan](https://github.com/Dandandan))
-- Fix latest commit [\#379](https://github.com/apache/datafusion-ballista/pull/379) ([Dandandan](https://github.com/Dandandan))
-- Avoid calling scheduler when the executor cannot accept new tasks [\#378](https://github.com/apache/datafusion-ballista/pull/378) ([Dandandan](https://github.com/Dandandan))
-- Switch to mimalloc and enable by default in executor [\#370](https://github.com/apache/datafusion-ballista/pull/370) ([Dandandan](https://github.com/Dandandan))
-- Benchmark looks for path with and without extension [\#354](https://github.com/apache/datafusion-ballista/pull/354) ([andygrove](https://github.com/andygrove))
-- Implement job cancellation in UI [\#349](https://github.com/apache/datafusion-ballista/pull/349) ([Dandandan](https://github.com/Dandandan))
-- Using local shuffle reader avoid flight rpc call. [\#347](https://github.com/apache/datafusion-ballista/pull/347) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Make helm deployable [\#345](https://github.com/apache/datafusion-ballista/pull/345) ([avantgardnerio](https://github.com/avantgardnerio))
-- Benchmark & UI improvements [\#343](https://github.com/apache/datafusion-ballista/pull/343) ([andygrove](https://github.com/andygrove))
-- Add `cancel_job` REST API [\#340](https://github.com/apache/datafusion-ballista/pull/340) ([tfeda](https://github.com/tfeda))
-- Fix labeler [\#337](https://github.com/apache/datafusion-ballista/pull/337) ([andygrove](https://github.com/andygrove))
-- Upgrade to DataFusion 13.0.0 [\#336](https://github.com/apache/datafusion-ballista/pull/336) ([andygrove](https://github.com/andygrove))
-- Check executor id consistency when receive stop executor request [\#335](https://github.com/apache/datafusion-ballista/pull/335) ([yahoNanJing](https://github.com/yahoNanJing))
-- Enable more benchmark serde tests [\#331](https://github.com/apache/datafusion-ballista/pull/331) ([andygrove](https://github.com/andygrove))
-- Downgrade `docker-compose.yaml` to version 3.3 so that we can support Ubuntu 20.04.4 LTS [\#329](https://github.com/apache/datafusion-ballista/pull/329) ([andygrove](https://github.com/andygrove))
-- update labeler [\#326](https://github.com/apache/datafusion-ballista/pull/326) ([andygrove](https://github.com/andygrove))
-- Upgrade to DataFusion 13.0.0-rc1 [\#325](https://github.com/apache/datafusion-ballista/pull/325) ([andygrove](https://github.com/andygrove))
-- Dependabot stop suggesting arrow and datafusion updates [\#324](https://github.com/apache/datafusion-ballista/pull/324) ([andygrove](https://github.com/andygrove))
-- Show job stages metrics [\#323](https://github.com/apache/datafusion-ballista/pull/323) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
-- Add helm chart [\#322](https://github.com/apache/datafusion-ballista/pull/322) ([avantgardnerio](https://github.com/avantgardnerio))
-- Atomic support for enhancement [\#319](https://github.com/apache/datafusion-ballista/pull/319) ([metesynnada](https://github.com/metesynnada))
-- Allow automatic schema inference when registering csv [\#313](https://github.com/apache/datafusion-ballista/pull/313) ([r4ntix](https://github.com/r4ntix))
-- Add ability to specify job name and have it show in the job listing page in the UI [\#312](https://github.com/apache/datafusion-ballista/pull/312) ([andygrove](https://github.com/andygrove))
-- Add REST API to generate DOT graph for individual query stage [\#310](https://github.com/apache/datafusion-ballista/pull/310) ([andygrove](https://github.com/andygrove))
-- \[UI\] Use tabbed pane with Queries and Executors tabs [\#309](https://github.com/apache/datafusion-ballista/pull/309) ([andygrove](https://github.com/andygrove))
-- REST API to get query stages [\#305](https://github.com/apache/datafusion-ballista/pull/305) ([andygrove](https://github.com/andygrove))
-- Add support for SortPreservingMergeExec; fix LIMIT bug [\#304](https://github.com/apache/datafusion-ballista/pull/304) ([andygrove](https://github.com/andygrove))
-- Add Python script to run benchmarks [\#302](https://github.com/apache/datafusion-ballista/pull/302) ([andygrove](https://github.com/andygrove))
-- \[UI\] Add ability to view query plans directly in the UI [\#301](https://github.com/apache/datafusion-ballista/pull/301) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
-- Update datafusion.proto [\#299](https://github.com/apache/datafusion-ballista/pull/299) ([andygrove](https://github.com/andygrove))
-- Replace function `from_proto_binary_op` from upstream [\#298](https://github.com/apache/datafusion-ballista/pull/298) ([askoa](https://github.com/askoa))
-- Fix dead link in contribution guideline readme file [\#297](https://github.com/apache/datafusion-ballista/pull/297) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
-- UI code cleanup [\#291](https://github.com/apache/datafusion-ballista/pull/291) ([KenSuenobu](https://github.com/KenSuenobu))
-- Add support for S3 data sources [\#290](https://github.com/apache/datafusion-ballista/pull/290) ([andygrove](https://github.com/andygrove))
-- Use latest datafusion [\#289](https://github.com/apache/datafusion-ballista/pull/289) ([andygrove](https://github.com/andygrove))
-- Fix documentation example [\#288](https://github.com/apache/datafusion-ballista/pull/288) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
-- Improve formatting of job status in UI [\#286](https://github.com/apache/datafusion-ballista/pull/286) ([andygrove](https://github.com/andygrove))
-- Enabled download of dot files from Download icon [\#279](https://github.com/apache/datafusion-ballista/pull/279) ([KenSuenobu](https://github.com/KenSuenobu))
-- Executor graceful shutdown: Handle SIGTERM [\#278](https://github.com/apache/datafusion-ballista/pull/278) ([mingmwang](https://github.com/mingmwang))
-- Also run yarn build to catch JavaScript errors in CI [\#271](https://github.com/apache/datafusion-ballista/pull/271) ([avantgardnerio](https://github.com/avantgardnerio))
-- Store sessions so users can register tables and query them through flight [\#269](https://github.com/apache/datafusion-ballista/pull/269) ([avantgardnerio](https://github.com/avantgardnerio))
-- Fix compose for Ian [\#268](https://github.com/apache/datafusion-ballista/pull/268) ([avantgardnerio](https://github.com/avantgardnerio))
-- Task level retry and Stage level retry [\#261](https://github.com/apache/datafusion-ballista/pull/261) ([mingmwang](https://github.com/mingmwang))
-- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#260](https://github.com/apache/datafusion-ballista/pull/260) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add a feature based object store provider [\#258](https://github.com/apache/datafusion-ballista/pull/258) ([yahoNanJing](https://github.com/yahoNanJing))
-- Make fetch shuffle partition data in parallel [\#256](https://github.com/apache/datafusion-ballista/pull/256) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add LaunchMultiTask rpc interface for executor [\#255](https://github.com/apache/datafusion-ballista/pull/255) ([yahoNanJing](https://github.com/yahoNanJing))
-- CLI uses ballista context instead of datafusion context in local mode [\#252](https://github.com/apache/datafusion-ballista/pull/252) ([r4ntix](https://github.com/r4ntix))
-- Fix Scheduler UI in Docker image [\#251](https://github.com/apache/datafusion-ballista/pull/251) ([andygrove](https://github.com/andygrove))
-- Generate into source folder to make IDEs happy [\#247](https://github.com/apache/datafusion-ballista/pull/247) ([avantgardnerio](https://github.com/avantgardnerio))
-- Use bind host rather than the external host for starting a local executor service [\#245](https://github.com/apache/datafusion-ballista/pull/245) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add REST endpoint to get DOT graph of a job [\#242](https://github.com/apache/datafusion-ballista/pull/242) ([andygrove](https://github.com/andygrove))
-- Add list of jobs to scheduler UI [\#241](https://github.com/apache/datafusion-ballista/pull/241) ([andygrove](https://github.com/andygrove))
-- Clean up job data on both Scheduler and Executor [\#188](https://github.com/apache/datafusion-ballista/pull/188) ([mingmwang](https://github.com/mingmwang))
-- Update etcd-client requirement from 0.9 to 0.10 [\#111](https://github.com/apache/datafusion-ballista/pull/111) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump terser from 4.8.0 to 4.8.1 in /ballista/ui/scheduler [\#91](https://github.com/apache/datafusion-ballista/pull/91) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump jsdom from 16.4.0 to 16.7.0 in /ballista/ui/scheduler [\#74](https://github.com/apache/datafusion-ballista/pull/74) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump numpy from 1.21.3 to 1.22.0 in /python [\#72](https://github.com/apache/datafusion-ballista/pull/72) ([dependabot[bot]](https://github.com/apps/dependabot))
-
-## [0.8.0](https://github.com/apache/datafusion-ballista/tree/0.8.0) (2022-09-16)
-
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.7.0...0.8.0)
-
-**Implemented enhancements:**
-
-- Executor should use all available cores by default [\#218](https://github.com/apache/datafusion-ballista/issues/218)
-- Update task status to the task job curator scheduler [\#179](https://github.com/apache/datafusion-ballista/issues/179)
-- update datafusion and arrow to 20.0.0 [\#176](https://github.com/apache/datafusion-ballista/issues/176)
-- No scheduler logs when deployed to k8s [\#165](https://github.com/apache/datafusion-ballista/issues/165)
-- Upgrade to DataFusion 11.0.0 [\#163](https://github.com/apache/datafusion-ballista/issues/163)
-- Better encapsulation for ExecutionGraph [\#149](https://github.com/apache/datafusion-ballista/issues/149)
-- A stage may act as the input of multiple stages [\#144](https://github.com/apache/datafusion-ballista/issues/144)
-- Executor Lost handling [\#143](https://github.com/apache/datafusion-ballista/issues/143)
-- Cancel a running query. [\#139](https://github.com/apache/datafusion-ballista/issues/139)
-- Ignore the previous job_id inside fill_reservations\(\) [\#138](https://github.com/apache/datafusion-ballista/issues/138)
-- Normalize the serialization and deserialization places of protobuf structs [\#137](https://github.com/apache/datafusion-ballista/issues/137)
-- Remove revive offer event loop [\#136](https://github.com/apache/datafusion-ballista/issues/136)
-- Remove Keyspace::QueuedJobs [\#133](https://github.com/apache/datafusion-ballista/issues/133)
-- Spawn a thread for execution plan generation [\#131](https://github.com/apache/datafusion-ballista/issues/131)
-- Introduce CuratorTaskManager for make an active job be curated by only one scheduler [\#130](https://github.com/apache/datafusion-ballista/issues/130)
-- Using tokio tracing for log file [\#122](https://github.com/apache/datafusion-ballista/issues/122)
-- Ballista Executor report plan/operators metrics to Ballista Scheduler when task finish [\#116](https://github.com/apache/datafusion-ballista/issues/116)
-- Add timeout settings for Grpc Client [\#114](https://github.com/apache/datafusion-ballista/issues/114)
-- Add log level config in ballista [\#102](https://github.com/apache/datafusion-ballista/issues/102)
-- Use another channel to update the status of a task set for executor [\#96](https://github.com/apache/datafusion-ballista/issues/96)
-- Add config for concurrent_task in executor [\#94](https://github.com/apache/datafusion-ballista/issues/94)
-- Ballista should support Arrow FlightSQL [\#92](https://github.com/apache/datafusion-ballista/issues/92)
-- Why not include the `ballista-cli` in the member of workspace [\#88](https://github.com/apache/datafusion-ballista/issues/88)
-- Upgrade dependency of arrow-datafusion to commit d0d5564b8f689a01e542b8c1df829d74d0fab2b0 [\#84](https://github.com/apache/datafusion-ballista/issues/84)
-- Support sled path in config file. [\#79](https://github.com/apache/datafusion-ballista/issues/79)
-- Support for multi-scheduler deployments [\#39](https://github.com/apache/datafusion-ballista/issues/39)
-- Ballista 0.7.0 Release [\#126](https://github.com/apache/datafusion-ballista/issues/126)
-- Improvements to Ballista extensibility [\#8](https://github.com/apache/datafusion-ballista/issues/8)
-- Implement Python bindings for BallistaContext [\#15](https://github.com/apache/datafusion-ballista/issues/15)
-
-**Fixed bugs:**
-
-- Run example fails via PushStaged mode [\#214](https://github.com/apache/datafusion-ballista/issues/214)
-- Config settings in BallistaContext do not get passed to DataFusion context [\#213](https://github.com/apache/datafusion-ballista/issues/213)
-- Start scheduler fails with arguments "-s PushStaged" [\#207](https://github.com/apache/datafusion-ballista/issues/207)
-- FlightSQL is broken and CI isn't catching it [\#190](https://github.com/apache/datafusion-ballista/issues/190)
-- Query fails with "NULL is invalid as a DataFusion scalar value" [\#180](https://github.com/apache/datafusion-ballista/issues/180)
-- Executor doesn't compile, missing `tokio::signal` [\#171](https://github.com/apache/datafusion-ballista/issues/171)
-- Unable to build master [\#76](https://github.com/apache/datafusion-ballista/issues/76)
-
-## [ballista-0.7.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.7.0) (2022-05-12)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.1.0-rc1...ballista-0.7.0)
-
-**Breaking changes:**
-
-- Make `ExecutionPlan::execute` Sync [\#2434](https://github.com/apache/arrow-datafusion/pull/2434) ([tustvold](https://github.com/tustvold))
-- Add `Expr::Exists` to represent EXISTS subquery expression [\#2339](https://github.com/apache/arrow-datafusion/pull/2339) ([andygrove](https://github.com/andygrove))
-- Remove dependency from `LogicalPlan::TableScan` to `ExecutionPlan` [\#2284](https://github.com/apache/arrow-datafusion/pull/2284) ([andygrove](https://github.com/andygrove))
-- Move logical expression type-coercion code from `physical-expr` crate to `expr` crate [\#2257](https://github.com/apache/arrow-datafusion/pull/2257) ([andygrove](https://github.com/andygrove))
-- feat: 2061 create external table ddl table partition cols [\#2099](https://github.com/apache/arrow-datafusion/pull/2099) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jychen7](https://github.com/jychen7))
-- Reorganize the project folders [\#2081](https://github.com/apache/arrow-datafusion/pull/2081) ([yahoNanJing](https://github.com/yahoNanJing))
-- Support more ScalarFunction in Ballista [\#2008](https://github.com/apache/arrow-datafusion/pull/2008) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Merge dataframe and dataframe imp [\#1998](https://github.com/apache/arrow-datafusion/pull/1998) ([vchag](https://github.com/vchag))
-- Rename `ExecutionContext` to `SessionContext`, `ExecutionContextState` to `SessionState`, add `TaskContext` to support multi-tenancy configurations - Part 1 [\#1987](https://github.com/apache/arrow-datafusion/pull/1987) ([mingmwang](https://github.com/mingmwang))
-- Add Coalesce function [\#1969](https://github.com/apache/arrow-datafusion/pull/1969) ([msathis](https://github.com/msathis))
-- Add Create Schema functionality in SQL [\#1959](https://github.com/apache/arrow-datafusion/pull/1959) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
-- remove sync constraint of SendableRecordBatchStream [\#1884](https://github.com/apache/arrow-datafusion/pull/1884) ([doki23](https://github.com/doki23))
-
-**Implemented enhancements:**
-
-- Add `CREATE VIEW` [\#2279](https://github.com/apache/arrow-datafusion/pull/2279) ([matthewmturner](https://github.com/matthewmturner))
-- \[Ballista\] Support Union in ballista. [\#2098](https://github.com/apache/arrow-datafusion/pull/2098) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Add missing aggr_expr to PhysicalExprNode for Ballista. [\#1989](https://github.com/apache/arrow-datafusion/pull/1989) ([Ted-Jiang](https://github.com/Ted-Jiang))
-
-**Fixed bugs:**
-
-- Ballista integration tests no longer work [\#2440](https://github.com/apache/arrow-datafusion/issues/2440)
-- Ballista crates cannot be released from DafaFusion 7.0.0 source release [\#1980](https://github.com/apache/arrow-datafusion/issues/1980)
-- protobuf OctetLength should be deserialized as octet_length, not length [\#1834](https://github.com/apache/arrow-datafusion/pull/1834) ([carols10cents](https://github.com/carols10cents))
-
-**Documentation updates:**
-
-- MINOR: Make crate READMEs consistent [\#2437](https://github.com/apache/arrow-datafusion/pull/2437) ([andygrove](https://github.com/andygrove))
-- docs: Update the Ballista dev env instructions [\#2419](https://github.com/apache/arrow-datafusion/pull/2419) ([haoxins](https://github.com/haoxins))
-- Revise document of installing ballista pinned to specified version [\#2034](https://github.com/apache/arrow-datafusion/pull/2034) ([WinkerDu](https://github.com/WinkerDu))
-- Fix typos \(Datafusion -\> DataFusion\) [\#1993](https://github.com/apache/arrow-datafusion/pull/1993) ([andygrove](https://github.com/andygrove))
-
-**Performance improvements:**
-
-- Introduce StageManager for managing tasks stage by stage [\#1983](https://github.com/apache/arrow-datafusion/pull/1983) ([yahoNanJing](https://github.com/yahoNanJing))
-
-**Closed issues:**
-
-- Make expected result string in unit tests more readable [\#2412](https://github.com/apache/arrow-datafusion/issues/2412)
-- remove duplicated `fn aggregate()` in aggregate expression tests [\#2399](https://github.com/apache/arrow-datafusion/issues/2399)
-- split `distinct_expression.rs` into `count_distinct.rs` and `array_agg_distinct.rs` [\#2385](https://github.com/apache/arrow-datafusion/issues/2385)
-- move sql tests in `context.rs` to corresponding test files in `datafustion/core/tests/sql` [\#2328](https://github.com/apache/arrow-datafusion/issues/2328)
-- Date32/Date64 as join keys for merge join [\#2314](https://github.com/apache/arrow-datafusion/issues/2314)
-- Error precision and scale for decimal coercion in logic comparison [\#2232](https://github.com/apache/arrow-datafusion/issues/2232)
-- Support Multiple row layout [\#2188](https://github.com/apache/arrow-datafusion/issues/2188)
-- Discussion: Is Ballista a standalone system or framework [\#1916](https://github.com/apache/arrow-datafusion/issues/1916)
-
-**Merged pull requests:**
-
-- MINOR: Enable multi-statement benchmark queries [\#2507](https://github.com/apache/arrow-datafusion/pull/2507) ([andygrove](https://github.com/andygrove))
-- Persist session configs in scheduler [\#2501](https://github.com/apache/arrow-datafusion/pull/2501) ([thinkharderdev](https://github.com/thinkharderdev))
-- Update to `sqlparser` `0.17.0` [\#2500](https://github.com/apache/arrow-datafusion/pull/2500) ([alamb](https://github.com/alamb))
-- Limit cpu cores used when generating changelog [\#2494](https://github.com/apache/arrow-datafusion/pull/2494) ([andygrove](https://github.com/andygrove))
-- MINOR: Parameterize changelog script [\#2484](https://github.com/apache/arrow-datafusion/pull/2484) ([jychen7](https://github.com/jychen7))
-- Fix stage key extraction [\#2472](https://github.com/apache/arrow-datafusion/pull/2472) ([thinkharderdev](https://github.com/thinkharderdev))
-- Add support for list_dir\(\) on local fs [\#2467](https://github.com/apache/arrow-datafusion/pull/2467) ([wjones127](https://github.com/wjones127))
-- minor: update versions and paths in changelog scripts [\#2429](https://github.com/apache/arrow-datafusion/pull/2429) ([andygrove](https://github.com/andygrove))
-- Fix Ballista executing during plan [\#2428](https://github.com/apache/arrow-datafusion/pull/2428) ([tustvold](https://github.com/tustvold))
-- Re-organize and rename aggregates physical plan [\#2388](https://github.com/apache/arrow-datafusion/pull/2388) ([yjshen](https://github.com/yjshen))
-- Upgrade to arrow 13 [\#2382](https://github.com/apache/arrow-datafusion/pull/2382) ([alamb](https://github.com/alamb))
-- Grouped Aggregate in row format [\#2375](https://github.com/apache/arrow-datafusion/pull/2375) ([yjshen](https://github.com/yjshen))
-- Stop optimizing queries twice [\#2369](https://github.com/apache/arrow-datafusion/pull/2369) ([andygrove](https://github.com/andygrove))
-- Bump follow-redirects from 1.13.2 to 1.14.9 in /ballista/ui/scheduler [\#2325](https://github.com/apache/arrow-datafusion/pull/2325) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Move FileType enum from sql module to logical_plan module [\#2290](https://github.com/apache/arrow-datafusion/pull/2290) ([andygrove](https://github.com/andygrove))
-- Add BatchPartitioner \(\#2285\) [\#2287](https://github.com/apache/arrow-datafusion/pull/2287) ([tustvold](https://github.com/tustvold))
-- Update uuid requirement from 0.8 to 1.0 [\#2280](https://github.com/apache/arrow-datafusion/pull/2280) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump async from 2.6.3 to 2.6.4 in /ballista/ui/scheduler [\#2277](https://github.com/apache/arrow-datafusion/pull/2277) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump minimist from 1.2.5 to 1.2.6 in /ballista/ui/scheduler [\#2276](https://github.com/apache/arrow-datafusion/pull/2276) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump url-parse from 1.5.1 to 1.5.10 in /ballista/ui/scheduler [\#2275](https://github.com/apache/arrow-datafusion/pull/2275) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump nanoid from 3.1.20 to 3.3.3 in /ballista/ui/scheduler [\#2274](https://github.com/apache/arrow-datafusion/pull/2274) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Update to Arrow 12.0.0, update tonic and prost [\#2253](https://github.com/apache/arrow-datafusion/pull/2253) ([alamb](https://github.com/alamb))
-- Add ExecutorMetricsCollector interface [\#2234](https://github.com/apache/arrow-datafusion/pull/2234) ([thinkharderdev](https://github.com/thinkharderdev))
-- minor: add editor config file [\#2224](https://github.com/apache/arrow-datafusion/pull/2224) ([jackwener](https://github.com/jackwener))
-- \[Ballista\] Enable ApproxPercentileWithWeight in Ballista and fill UT [\#2192](https://github.com/apache/arrow-datafusion/pull/2192) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- make nightly clippy happy [\#2186](https://github.com/apache/arrow-datafusion/pull/2186) ([xudong963](https://github.com/xudong963))
-- \[Ballista\]Make PhysicalAggregateExprNode has repeated PhysicalExprNode [\#2184](https://github.com/apache/arrow-datafusion/pull/2184) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Add LogicalPlan::SubqueryAlias [\#2172](https://github.com/apache/arrow-datafusion/pull/2172) ([andygrove](https://github.com/andygrove))
-- Implement fast path of with_new_children\(\) in ExecutionPlan [\#2168](https://github.com/apache/arrow-datafusion/pull/2168) ([mingmwang](https://github.com/mingmwang))
-- \[MINOR\] ignore suspicious slow test in Ballista [\#2167](https://github.com/apache/arrow-datafusion/pull/2167) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- enable explain for ballista [\#2163](https://github.com/apache/arrow-datafusion/pull/2163) ([doki23](https://github.com/doki23))
-- Add delimiter for create external table [\#2162](https://github.com/apache/arrow-datafusion/pull/2162) ([matthewmturner](https://github.com/matthewmturner))
-- Update sqlparser requirement from 0.15 to 0.16 [\#2152](https://github.com/apache/arrow-datafusion/pull/2152) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Add IF NOT EXISTS to `CREATE TABLE` and `CREATE EXTERNAL TABLE` [\#2143](https://github.com/apache/arrow-datafusion/pull/2143) ([matthewmturner](https://github.com/matthewmturner))
-- Update quarterly roadmap for Q2 [\#2133](https://github.com/apache/arrow-datafusion/pull/2133) ([matthewmturner](https://github.com/matthewmturner))
-- \[Ballista\] Add ballista plugin manager and UDF plugin [\#2131](https://github.com/apache/arrow-datafusion/pull/2131) ([gaojun2048](https://github.com/gaojun2048))
-- Serialize scalar UDFs in physical plan [\#2130](https://github.com/apache/arrow-datafusion/pull/2130) ([thinkharderdev](https://github.com/thinkharderdev))
-- doc: update release schedule [\#2110](https://github.com/apache/arrow-datafusion/pull/2110) ([jychen7](https://github.com/jychen7))
-- Reduce repetition in Decimal binary kernels, upgrade to arrow 11.1 [\#2107](https://github.com/apache/arrow-datafusion/pull/2107) ([alamb](https://github.com/alamb))
-- update zlib version to 1.2.12 [\#2106](https://github.com/apache/arrow-datafusion/pull/2106) ([waitingkuo](https://github.com/waitingkuo))
-- Add CREATE DATABASE command to SQL [\#2094](https://github.com/apache/arrow-datafusion/pull/2094) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
-- Refactor SessionContext, BallistaContext to support multi-tenancy configurations - Part 3 [\#2091](https://github.com/apache/arrow-datafusion/pull/2091) ([mingmwang](https://github.com/mingmwang))
-- Remove dependency of common for the storage crate [\#2076](https://github.com/apache/arrow-datafusion/pull/2076) ([yahoNanJing](https://github.com/yahoNanJing))
-- [MINOR] fix doc in `EXTRACT\(field FROM source\) [\#2074](https://github.com/apache/arrow-datafusion/pull/2074) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- \[Bug\]\[Datafusion\] fix TaskContext session_config bug [\#2070](https://github.com/apache/arrow-datafusion/pull/2070) ([gaojun2048](https://github.com/gaojun2048))
-- Short-circuit evaluation for `CaseWhen` [\#2068](https://github.com/apache/arrow-datafusion/pull/2068) ([yjshen](https://github.com/yjshen))
-- split datafusion-object-store module [\#2065](https://github.com/apache/arrow-datafusion/pull/2065) ([yahoNanJing](https://github.com/yahoNanJing))
-- Change log level for noisy logs [\#2060](https://github.com/apache/arrow-datafusion/pull/2060) ([thinkharderdev](https://github.com/thinkharderdev))
-- Update to arrow/parquet 11.0 [\#2048](https://github.com/apache/arrow-datafusion/pull/2048) ([alamb](https://github.com/alamb))
-- minor: format comments \(`//` to `//`\) [\#2047](https://github.com/apache/arrow-datafusion/pull/2047) ([jackwener](https://github.com/jackwener))
-- use cargo-tomlfmt to check Cargo.toml formatting in CI [\#2033](https://github.com/apache/arrow-datafusion/pull/2033) ([WinkerDu](https://github.com/WinkerDu))
-- Refactor SessionContext, SessionState and SessionConfig to support multi-tenancy configurations - Part 2 [\#2029](https://github.com/apache/arrow-datafusion/pull/2029) ([mingmwang](https://github.com/mingmwang))
-- Simplify prerequisites for running examples [\#2028](https://github.com/apache/arrow-datafusion/pull/2028) ([doki23](https://github.com/doki23))
-- Use SessionContext to parse Expr protobuf [\#2024](https://github.com/apache/arrow-datafusion/pull/2024) ([thinkharderdev](https://github.com/thinkharderdev))
-- Fix stuck issue for the load testing of Push-based task scheduling [\#2006](https://github.com/apache/arrow-datafusion/pull/2006) ([yahoNanJing](https://github.com/yahoNanJing))
-- Fixing a typo in documentation [\#1997](https://github.com/apache/arrow-datafusion/pull/1997) ([psvri](https://github.com/psvri))
-- Fix minor clippy issue [\#1995](https://github.com/apache/arrow-datafusion/pull/1995) ([alamb](https://github.com/alamb))
-- Make it possible to only scan part of a parquet file in a partition [\#1990](https://github.com/apache/arrow-datafusion/pull/1990) ([yjshen](https://github.com/yjshen))
-- Update Dockerfile to fix integration tests [\#1982](https://github.com/apache/arrow-datafusion/pull/1982) ([andygrove](https://github.com/andygrove))
-- Update sqlparser requirement from 0.14 to 0.15 [\#1966](https://github.com/apache/arrow-datafusion/pull/1966) ([dependabot[bot]](https://github.com/apps/dependabot))
-- fix logical conflict with protobuf [\#1958](https://github.com/apache/arrow-datafusion/pull/1958) ([alamb](https://github.com/alamb))
-- Update to arrow 10.0.0, pyo3 0.16 [\#1957](https://github.com/apache/arrow-datafusion/pull/1957) ([alamb](https://github.com/alamb))
-- update jit-related dependencies [\#1953](https://github.com/apache/arrow-datafusion/pull/1953) ([xudong963](https://github.com/xudong963))
-- Allow different types of query variables \(`@@var`\) rather than just string [\#1943](https://github.com/apache/arrow-datafusion/pull/1943) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([maxburke](https://github.com/maxburke))
-- Pruning serialization [\#1941](https://github.com/apache/arrow-datafusion/pull/1941) ([thinkharderdev](https://github.com/thinkharderdev))
-- Fix select from EmptyExec always return 0 row after optimizer passes [\#1938](https://github.com/apache/arrow-datafusion/pull/1938) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Introduce Ballista query stage scheduler [\#1935](https://github.com/apache/arrow-datafusion/pull/1935) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add db benchmark script [\#1928](https://github.com/apache/arrow-datafusion/pull/1928) ([matthewmturner](https://github.com/matthewmturner))
-- fix a typo [\#1919](https://github.com/apache/arrow-datafusion/pull/1919) ([vchag](https://github.com/vchag))
-- \[MINOR\] Update copyright year in Docs [\#1918](https://github.com/apache/arrow-datafusion/pull/1918) ([alamb](https://github.com/alamb))
-- add metadata to DFSchema, close \#1806. [\#1914](https://github.com/apache/arrow-datafusion/pull/1914) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jiacai2050](https://github.com/jiacai2050))
-- Refactor scheduler state mod [\#1913](https://github.com/apache/arrow-datafusion/pull/1913) ([yahoNanJing](https://github.com/yahoNanJing))
-- Refactor the event channel [\#1912](https://github.com/apache/arrow-datafusion/pull/1912) ([yahoNanJing](https://github.com/yahoNanJing))
-- Refactor scheduler server [\#1911](https://github.com/apache/arrow-datafusion/pull/1911) ([yahoNanJing](https://github.com/yahoNanJing))
-- Clippy fix on nightly [\#1907](https://github.com/apache/arrow-datafusion/pull/1907) ([yjshen](https://github.com/yjshen))
-- Updated Rust version to 1.59 in all the files [\#1903](https://github.com/apache/arrow-datafusion/pull/1903) ([NaincyKumariKnoldus](https://github.com/NaincyKumariKnoldus))
-- Remove uneeded Mutex in Ballista Client [\#1898](https://github.com/apache/arrow-datafusion/pull/1898) ([alamb](https://github.com/alamb))
-- Create a `datafusion-proto` crate for datafusion protobuf serialization [\#1887](https://github.com/apache/arrow-datafusion/pull/1887) ([carols10cents](https://github.com/carols10cents))
-- Fix clippy lints [\#1885](https://github.com/apache/arrow-datafusion/pull/1885) ([HaoYang670](https://github.com/HaoYang670))
-- Separate cpu-bound \(query-execution\) and IO-bound\(heartbeat\) to … [\#1883](https://github.com/apache/arrow-datafusion/pull/1883) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- \[Minor\] Clean up DecimalArray API Usage [\#1869](https://github.com/apache/arrow-datafusion/pull/1869) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- Changes after went through "Datafusion as a library section" [\#1868](https://github.com/apache/arrow-datafusion/pull/1868) ([nonontb](https://github.com/nonontb))
-- Remove allow unused imports from ballista-core, then fix all warnings [\#1853](https://github.com/apache/arrow-datafusion/pull/1853) ([carols10cents](https://github.com/carols10cents))
-- Update to arrow 9.1.0 [\#1851](https://github.com/apache/arrow-datafusion/pull/1851) ([alamb](https://github.com/alamb))
-- move some tests out of context and into sql [\#1846](https://github.com/apache/arrow-datafusion/pull/1846) ([alamb](https://github.com/alamb))
-- Fix compiling ballista in standalone mode, add build to CI [\#1839](https://github.com/apache/arrow-datafusion/pull/1839) ([alamb](https://github.com/alamb))
-- Update documentation example for change in API [\#1812](https://github.com/apache/arrow-datafusion/pull/1812) ([alamb](https://github.com/alamb))
-- Refactor scheduler state with different management policy for volatile and stable states [\#1810](https://github.com/apache/arrow-datafusion/pull/1810) ([yahoNanJing](https://github.com/yahoNanJing))
-- DataFusion + Conbench Integration [\#1791](https://github.com/apache/arrow-datafusion/pull/1791) ([dianaclarke](https://github.com/dianaclarke))
-- Enable periodic cleanup of work_dir directories in ballista executor [\#1783](https://github.com/apache/arrow-datafusion/pull/1783) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Use`eq_dyn`, `neq_dyn`, `lt_dyn`, `lt_eq_dyn`, `gt_dyn`, `gt_eq_dyn` kernels from arrow [\#1475](https://github.com/apache/arrow-datafusion/pull/1475) ([alamb](https://github.com/alamb))
-
-## [7.1.0-rc1](https://github.com/apache/arrow-datafusion/tree/7.1.0-rc1) (2022-04-10)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.0.0-rc2...7.1.0-rc1)
-
-**Implemented enhancements:**
-
-- Support substring with three arguments: \(str, from, for\) for DataFrame API and Ballista [\#2092](https://github.com/apache/arrow-datafusion/issues/2092)
-- UnionAll support for Ballista [\#2032](https://github.com/apache/arrow-datafusion/issues/2032)
-- Separate cpu-bound and IO-bound work in ballista-executor by using diff tokio runtime. [\#1770](https://github.com/apache/arrow-datafusion/issues/1770)
-- \[Ballista\] Introduce DAGScheduler for better managing the stage-based task scheduling [\#1704](https://github.com/apache/arrow-datafusion/issues/1704)
-- \[Ballista\] Support to better manage cluster state, like alive executors, executor available task slots, etc [\#1703](https://github.com/apache/arrow-datafusion/issues/1703)
-
-**Closed issues:**
-
-- Optimize memory usage pattern to avoid "double memory" behavior [\#2149](https://github.com/apache/arrow-datafusion/issues/2149)
-- Document approx_percentile_cont_with_weight in users guide [\#2078](https://github.com/apache/arrow-datafusion/issues/2078)
-- \[follow up\]cleaning up statements.remove\(0\) [\#1986](https://github.com/apache/arrow-datafusion/issues/1986)
-- Formatting error on documentation for Python [\#1873](https://github.com/apache/arrow-datafusion/issues/1873)
-- Remove duplicate tests from `test_const_evaluator_scalar_functions` [\#1727](https://github.com/apache/arrow-datafusion/issues/1727)
-- Question: Is the Ballista project providing value to the overall DataFusion project? [\#1273](https://github.com/apache/arrow-datafusion/issues/1273)
-
-## [7.0.0-rc2](https://github.com/apache/arrow-datafusion/tree/7.0.0-rc2) (2022-02-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.0.0...7.0.0-rc2)
-
-## [7.0.0](https://github.com/apache/arrow-datafusion/tree/7.0.0) (2022-02-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/6.0.0-rc0...7.0.0)
-
-**Breaking changes:**
-
-- Update `ExecutionPlan` to know about sortedness and repartitioning optimizer pass respect the invariants [\#1776](https://github.com/apache/arrow-datafusion/pull/1776) ([alamb](https://github.com/alamb))
-- Update to `arrow 8.0.0` [\#1673](https://github.com/apache/arrow-datafusion/pull/1673) ([alamb](https://github.com/alamb))
-
-**Implemented enhancements:**
-
-- Task assignment between Scheduler and Executors [\#1221](https://github.com/apache/arrow-datafusion/issues/1221)
-- Add `approx_median()` aggregate function [\#1729](https://github.com/apache/arrow-datafusion/pull/1729) ([realno](https://github.com/realno))
-- \[Ballista\] Add Decimal128, Date64, TimestampSecond, TimestampMillisecond, Interv… [\#1659](https://github.com/apache/arrow-datafusion/pull/1659) ([gaojun2048](https://github.com/gaojun2048))
-- Add `corr` aggregate function [\#1561](https://github.com/apache/arrow-datafusion/pull/1561) ([realno](https://github.com/realno))
-- Add `covar`, `covar_pop` and `covar_samp` aggregate functions [\#1551](https://github.com/apache/arrow-datafusion/pull/1551) ([realno](https://github.com/realno))
-- Add `approx_quantile()` aggregation function [\#1539](https://github.com/apache/arrow-datafusion/pull/1539) ([domodwyer](https://github.com/domodwyer))
-- Initial MemoryManager and DiskManager APIs for query execution + External Sort implementation [\#1526](https://github.com/apache/arrow-datafusion/pull/1526) ([yjshen](https://github.com/yjshen))
-- Add `stddev` and `variance` [\#1525](https://github.com/apache/arrow-datafusion/pull/1525) ([realno](https://github.com/realno))
-- Add `rem` operation for Expr [\#1467](https://github.com/apache/arrow-datafusion/pull/1467) ([liukun4515](https://github.com/liukun4515))
-- Implement `array_agg` aggregate function [\#1300](https://github.com/apache/arrow-datafusion/pull/1300) ([viirya](https://github.com/viirya))
-
-**Fixed bugs:**
-
-- Ballista context::tests::test_standalone_mode test fails [\#1020](https://github.com/apache/arrow-datafusion/issues/1020)
-- \[Ballista\] Fix scheduler state mod bug [\#1655](https://github.com/apache/arrow-datafusion/pull/1655) ([gaojun2048](https://github.com/gaojun2048))
-- Pass local address host so we do not get mismatch between IPv4 and IP… [\#1466](https://github.com/apache/arrow-datafusion/pull/1466) ([thinkharderdev](https://github.com/thinkharderdev))
-- Add Timezone to Scalar::Time\* types, and better timezone awareness to Datafusion's time types [\#1455](https://github.com/apache/arrow-datafusion/pull/1455) ([maxburke](https://github.com/maxburke))
-
-**Documentation updates:**
-
-- Add dependencies to ballista example documentation [\#1346](https://github.com/apache/arrow-datafusion/pull/1346) ([jgoday](https://github.com/jgoday))
-- \[MINOR\] Fix some typos. [\#1310](https://github.com/apache/arrow-datafusion/pull/1310) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- fix some clippy warnings from nightly channel [\#1277](https://github.com/apache/arrow-datafusion/pull/1277) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist))
-
-**Performance improvements:**
-
-- Introduce push-based task scheduling for Ballista [\#1560](https://github.com/apache/arrow-datafusion/pull/1560) ([yahoNanJing](https://github.com/yahoNanJing))
-
-**Closed issues:**
-
-- Track memory usage in Non Limited Operators [\#1569](https://github.com/apache/arrow-datafusion/issues/1569)
-- \[Question\] Why does ballista store tables in the client instead of in the SchedulerServer [\#1473](https://github.com/apache/arrow-datafusion/issues/1473)
-- Why use the expr types before coercion to get the result type? [\#1358](https://github.com/apache/arrow-datafusion/issues/1358)
-- A problem about the projection_push_down optimizer gathers valid columns [\#1312](https://github.com/apache/arrow-datafusion/issues/1312)
-- apply constant folding to `LogicalPlan::Values` [\#1170](https://github.com/apache/arrow-datafusion/issues/1170)
-- reduce usage of `IntoIterator<Item = Expr>` in logical plan builder window fn [\#372](https://github.com/apache/arrow-datafusion/issues/372)
-
-**Merged pull requests:**
-
-- Fix verification scripts for 7.0.0 release [\#1830](https://github.com/apache/arrow-datafusion/pull/1830) ([alamb](https://github.com/alamb))
-- update README for ballista [\#1817](https://github.com/apache/arrow-datafusion/pull/1817) ([liukun4515](https://github.com/liukun4515))
-- Fix logical conflict [\#1801](https://github.com/apache/arrow-datafusion/pull/1801) ([alamb](https://github.com/alamb))
-- Improve the error message and UX of tpch benchmark program [\#1800](https://github.com/apache/arrow-datafusion/pull/1800) ([alamb](https://github.com/alamb))
-- Update to sqlparser 0.14 [\#1796](https://github.com/apache/arrow-datafusion/pull/1796) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- Update datafusion versions [\#1793](https://github.com/apache/arrow-datafusion/pull/1793) ([matthewmturner](https://github.com/matthewmturner))
-- Update datafusion to use arrow 9.0.0 [\#1775](https://github.com/apache/arrow-datafusion/pull/1775) ([alamb](https://github.com/alamb))
-- Update parking_lot requirement from 0.11 to 0.12 [\#1735](https://github.com/apache/arrow-datafusion/pull/1735) ([dependabot[bot]](https://github.com/apps/dependabot))
-- substitute `parking_lot::Mutex` for `std::sync::Mutex` [\#1720](https://github.com/apache/arrow-datafusion/pull/1720) ([xudong963](https://github.com/xudong963))
-- Create ListingTableConfig which includes file format and schema inference [\#1715](https://github.com/apache/arrow-datafusion/pull/1715) ([matthewmturner](https://github.com/matthewmturner))
-- Support `create_physical_expr` and `ExecutionContextState` or `DefaultPhysicalPlanner` for faster speed [\#1700](https://github.com/apache/arrow-datafusion/pull/1700) ([alamb](https://github.com/alamb))
-- Use NamedTempFile rather than `String` in DiskManager [\#1680](https://github.com/apache/arrow-datafusion/pull/1680) ([alamb](https://github.com/alamb))
-- Abstract over logical and physical plan representations in Ballista [\#1677](https://github.com/apache/arrow-datafusion/pull/1677) ([thinkharderdev](https://github.com/thinkharderdev))
-- upgrade clap to version 3 [\#1672](https://github.com/apache/arrow-datafusion/pull/1672) ([Jimexist](https://github.com/Jimexist))
-- Improve configuration and resource use of `MemoryManager` and `DiskManager` [\#1668](https://github.com/apache/arrow-datafusion/pull/1668) ([alamb](https://github.com/alamb))
-- Make `MemoryManager` and `MemoryStream` public [\#1664](https://github.com/apache/arrow-datafusion/pull/1664) ([yjshen](https://github.com/yjshen))
-- Consolidate Schema and RecordBatch projection [\#1638](https://github.com/apache/arrow-datafusion/pull/1638) ([alamb](https://github.com/alamb))
-- Update hashbrown requirement from 0.11 to 0.12 [\#1631](https://github.com/apache/arrow-datafusion/pull/1631) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Update etcd-client requirement from 0.7 to 0.8 [\#1626](https://github.com/apache/arrow-datafusion/pull/1626) ([dependabot[bot]](https://github.com/apps/dependabot))
-- update nightly version [\#1597](https://github.com/apache/arrow-datafusion/pull/1597) ([Jimexist](https://github.com/Jimexist))
-- Add support show tables and show columns for ballista [\#1593](https://github.com/apache/arrow-datafusion/pull/1593) ([gaojun2048](https://github.com/gaojun2048))
-- minor: improve the benchmark readme [\#1567](https://github.com/apache/arrow-datafusion/pull/1567) ([xudong963](https://github.com/xudong963))
-- Consolidate `batch_size` configuration in `ExecutionConfig`, `RuntimeConfig` and `PhysicalPlanConfig` [\#1562](https://github.com/apache/arrow-datafusion/pull/1562) ([yjshen](https://github.com/yjshen))
-- Update to rust 1.58 [\#1557](https://github.com/apache/arrow-datafusion/pull/1557) ([xudong963](https://github.com/xudong963))
-- support mathematics operation for decimal data type [\#1554](https://github.com/apache/arrow-datafusion/pull/1554) ([liukun4515](https://github.com/liukun4515))
-- Make call SchedulerServer::new once in ballista-scheduler process [\#1537](https://github.com/apache/arrow-datafusion/pull/1537) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Add load test command in tpch.rs. [\#1530](https://github.com/apache/arrow-datafusion/pull/1530) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Remove one copy of ballista datatype serialization code [\#1524](https://github.com/apache/arrow-datafusion/pull/1524) ([alamb](https://github.com/alamb))
-- Update to arrow-7.0.0 [\#1523](https://github.com/apache/arrow-datafusion/pull/1523) ([alamb](https://github.com/alamb))
-- Workaround build failure: Pin quote to 1.0.10 [\#1499](https://github.com/apache/arrow-datafusion/pull/1499) ([alamb](https://github.com/alamb))
-- add rfcs for datafusion [\#1490](https://github.com/apache/arrow-datafusion/pull/1490) ([xudong963](https://github.com/xudong963))
-- support comparison for decimal data type and refactor the binary coercion rule [\#1483](https://github.com/apache/arrow-datafusion/pull/1483) ([liukun4515](https://github.com/liukun4515))
-- Update arrow-rs to 6.4.0 and replace boolean comparison in datafusion with arrow compute kernel [\#1446](https://github.com/apache/arrow-datafusion/pull/1446) ([xudong963](https://github.com/xudong963))
-- support cast/try_cast for decimal: signed numeric to decimal [\#1442](https://github.com/apache/arrow-datafusion/pull/1442) ([liukun4515](https://github.com/liukun4515))
-- use 0.13 sql parser [\#1435](https://github.com/apache/arrow-datafusion/pull/1435) ([Jimexist](https://github.com/Jimexist))
-- Clarify communication on bi-weekly sync [\#1427](https://github.com/apache/arrow-datafusion/pull/1427) ([alamb](https://github.com/alamb))
-- Minimize features [\#1399](https://github.com/apache/arrow-datafusion/pull/1399) ([carols10cents](https://github.com/carols10cents))
-- Update rust vesion to 1.57 [\#1395](https://github.com/apache/arrow-datafusion/pull/1395) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Add coercion rules for AggregateFunctions [\#1387](https://github.com/apache/arrow-datafusion/pull/1387) ([liukun4515](https://github.com/liukun4515))
-- upgrade the arrow-rs version [\#1385](https://github.com/apache/arrow-datafusion/pull/1385) ([liukun4515](https://github.com/liukun4515))
-- Extract logical plan: rename the plan name \(follow up\) [\#1354](https://github.com/apache/arrow-datafusion/pull/1354) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([liukun4515](https://github.com/liukun4515))
-- upgrade arrow-rs to 6.2.0 [\#1334](https://github.com/apache/arrow-datafusion/pull/1334) ([liukun4515](https://github.com/liukun4515))
-- Update release instructions [\#1331](https://github.com/apache/arrow-datafusion/pull/1331) ([alamb](https://github.com/alamb))
-- Extract Aggregate, Sort, and Join to struct from AggregatePlan [\#1326](https://github.com/apache/arrow-datafusion/pull/1326) ([matthewmturner](https://github.com/matthewmturner))
-- Extract `EmptyRelation`, `Limit`, `Values` from `LogicalPlan` [\#1325](https://github.com/apache/arrow-datafusion/pull/1325) ([liukun4515](https://github.com/liukun4515))
-- Extract CrossJoin, Repartition, Union in LogicalPlan [\#1322](https://github.com/apache/arrow-datafusion/pull/1322) ([liukun4515](https://github.com/liukun4515))
-- Extract Explain, Analyze, Extension in LogicalPlan as independent struct [\#1317](https://github.com/apache/arrow-datafusion/pull/1317) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Extract CreateMemoryTable, DropTable, CreateExternalTable in LogicalPlan as independent struct [\#1311](https://github.com/apache/arrow-datafusion/pull/1311) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([liukun4515](https://github.com/liukun4515))
-- Extract Projection, Filter, Window in LogicalPlan as independent struct [\#1309](https://github.com/apache/arrow-datafusion/pull/1309) ([ic4y](https://github.com/ic4y))
-- Add PSQL comparison tests for except, intersect [\#1292](https://github.com/apache/arrow-datafusion/pull/1292) ([mrob95](https://github.com/mrob95))
-- Extract logical plans in LogicalPlan as independent struct: TableScan [\#1290](https://github.com/apache/arrow-datafusion/pull/1290) ([xudong963](https://github.com/xudong963))
-
-## [6.0.0-rc0](https://github.com/apache/arrow-datafusion/tree/6.0.0-rc0) (2021-11-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/6.0.0...6.0.0-rc0)
-
-## [6.0.0](https://github.com/apache/arrow-datafusion/tree/6.0.0) (2021-11-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.6.0...6.0.0)
-
-## [ballista-0.6.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.6.0) (2021-11-13)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.5.0...ballista-0.6.0)
-
-**Breaking changes:**
-
-- File partitioning for ListingTable [\#1141](https://github.com/apache/arrow-datafusion/pull/1141) ([rdettai](https://github.com/rdettai))
-- Register tables in BallistaContext using TableProviders instead of Dataframe [\#1028](https://github.com/apache/arrow-datafusion/pull/1028) ([rdettai](https://github.com/rdettai))
-- Make TableProvider.scan\(\) and PhysicalPlanner::create_physical_plan\(\) async [\#1013](https://github.com/apache/arrow-datafusion/pull/1013) ([rdettai](https://github.com/rdettai))
-- Reorganize table providers by table format [\#1010](https://github.com/apache/arrow-datafusion/pull/1010) ([rdettai](https://github.com/rdettai))
-- Move CBOs and Statistics to physical plan [\#965](https://github.com/apache/arrow-datafusion/pull/965) ([rdettai](https://github.com/rdettai))
-- Update to sqlparser v 0.10.0 [\#934](https://github.com/apache/arrow-datafusion/pull/934) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- FilePartition and PartitionedFile for scanning flexibility [\#932](https://github.com/apache/arrow-datafusion/pull/932) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([yjshen](https://github.com/yjshen))
-- Improve SQLMetric APIs, port existing metrics [\#908](https://github.com/apache/arrow-datafusion/pull/908) ([alamb](https://github.com/alamb))
-- Add support for EXPLAIN ANALYZE [\#858](https://github.com/apache/arrow-datafusion/pull/858) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- Rename concurrency to target_partitions [\#706](https://github.com/apache/arrow-datafusion/pull/706) ([andygrove](https://github.com/andygrove))
-
-**Implemented enhancements:**
-
-- Update datafusion-cli to support Ballista, or implement new ballista-cli [\#886](https://github.com/apache/arrow-datafusion/issues/886)
-- Prepare Ballista crates for publishing [\#509](https://github.com/apache/arrow-datafusion/issues/509)
-- Add drop table support [\#1266](https://github.com/apache/arrow-datafusion/pull/1266) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([viirya](https://github.com/viirya))
-- use arrow 6.1.0 [\#1255](https://github.com/apache/arrow-datafusion/pull/1255) ([Jimexist](https://github.com/Jimexist))
-- Add support for `create table as` via MemTable [\#1243](https://github.com/apache/arrow-datafusion/pull/1243) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan))
-- add values list expression [\#1165](https://github.com/apache/arrow-datafusion/pull/1165) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist))
-- Multiple files per partitions for CSV Avro Json [\#1138](https://github.com/apache/arrow-datafusion/pull/1138) ([rdettai](https://github.com/rdettai))
-- Implement INTERSECT & INTERSECT DISTINCT [\#1135](https://github.com/apache/arrow-datafusion/pull/1135) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Simplify file struct abstractions [\#1120](https://github.com/apache/arrow-datafusion/pull/1120) ([rdettai](https://github.com/rdettai))
-- Implement `is [not] distinct from` [\#1117](https://github.com/apache/arrow-datafusion/pull/1117) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan))
-- add digest\(utf8, method\) function and refactor all current hash digest functions [\#1090](https://github.com/apache/arrow-datafusion/pull/1090) ([Jimexist](https://github.com/Jimexist))
-- \[crypto\] add `blake3` algorithm to `digest` function [\#1086](https://github.com/apache/arrow-datafusion/pull/1086) ([Jimexist](https://github.com/Jimexist))
-- \[crypto\] add blake2b and blake2s functions [\#1081](https://github.com/apache/arrow-datafusion/pull/1081) ([Jimexist](https://github.com/Jimexist))
-- Update sqlparser-rs to 0.11 [\#1052](https://github.com/apache/arrow-datafusion/pull/1052) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- remove hard coded partition count in ballista logicalplan deserialization [\#1044](https://github.com/apache/arrow-datafusion/pull/1044) ([xudong963](https://github.com/xudong963))
-- Indexed field access for List [\#1006](https://github.com/apache/arrow-datafusion/pull/1006) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki))
-- Update DataFusion to arrow 6.0 [\#984](https://github.com/apache/arrow-datafusion/pull/984) ([alamb](https://github.com/alamb))
-- Implement Display for Expr, improve operator display [\#971](https://github.com/apache/arrow-datafusion/pull/971) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
-- ObjectStore API to read from remote storage systems [\#950](https://github.com/apache/arrow-datafusion/pull/950) ([yjshen](https://github.com/yjshen))
-- fixes \#933 replace placeholder fmt_as fr ExecutionPlan impls [\#939](https://github.com/apache/arrow-datafusion/pull/939) ([tiphaineruy](https://github.com/tiphaineruy))
-- Support `NotLike` in Ballista [\#916](https://github.com/apache/arrow-datafusion/pull/916) ([Dandandan](https://github.com/Dandandan))
-- Avro Table Provider [\#910](https://github.com/apache/arrow-datafusion/pull/910) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki))
-- Add BaselineMetrics, Timestamp metrics, add for `CoalescePartitionsExec`, rename output_time -\> elapsed_compute [\#909](https://github.com/apache/arrow-datafusion/pull/909) ([alamb](https://github.com/alamb))
-- \[Ballista\] Add executor last seen info to the ui [\#895](https://github.com/apache/arrow-datafusion/pull/895) ([msathis](https://github.com/msathis))
-- add cross join support to ballista [\#891](https://github.com/apache/arrow-datafusion/pull/891) ([houqp](https://github.com/houqp))
-- Add Ballista support to DataFusion CLI [\#889](https://github.com/apache/arrow-datafusion/pull/889) ([andygrove](https://github.com/andygrove))
-- Add support for PostgreSQL regex match [\#870](https://github.com/apache/arrow-datafusion/pull/870) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([b41sh](https://github.com/b41sh))
-
-**Fixed bugs:**
-
-- Test execution_plans::shuffle_writer::tests::test Fail [\#1040](https://github.com/apache/arrow-datafusion/issues/1040)
-- Integration test fails to build docker images [\#918](https://github.com/apache/arrow-datafusion/issues/918)
-- Ballista: Remove hard-coded concurrency from logical plan serde code [\#708](https://github.com/apache/arrow-datafusion/issues/708)
-- How can I make ballista distributed compute work? [\#327](https://github.com/apache/arrow-datafusion/issues/327)
-- fix subquery alias [\#1067](https://github.com/apache/arrow-datafusion/pull/1067) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Fix compilation for ballista in stand-alone mode [\#1008](https://github.com/apache/arrow-datafusion/pull/1008) ([Igosuki](https://github.com/Igosuki))
-
-**Documentation updates:**
-
-- Add Ballista roadmap [\#1166](https://github.com/apache/arrow-datafusion/pull/1166) ([andygrove](https://github.com/andygrove))
-- Adds note on compatible rust version [\#1097](https://github.com/apache/arrow-datafusion/pull/1097) ([1nF0rmed](https://github.com/1nF0rmed))
-- implement `approx_distinct` function using HyperLogLog [\#1087](https://github.com/apache/arrow-datafusion/pull/1087) ([Jimexist](https://github.com/Jimexist))
-- Improve User Guide [\#954](https://github.com/apache/arrow-datafusion/pull/954) ([andygrove](https://github.com/andygrove))
-- Update plan_query_stages doc [\#951](https://github.com/apache/arrow-datafusion/pull/951) ([rdettai](https://github.com/rdettai))
-- \[DataFusion\] - Add show and show_limit function for DataFrame [\#923](https://github.com/apache/arrow-datafusion/pull/923) ([francis-du](https://github.com/francis-du))
-- update docs related to protoc and optional syntax [\#902](https://github.com/apache/arrow-datafusion/pull/902) ([Jimexist](https://github.com/Jimexist))
-- Improve Ballista crate README content [\#878](https://github.com/apache/arrow-datafusion/pull/878) ([andygrove](https://github.com/andygrove))
-
-**Performance improvements:**
-
-- optimize build profile for datafusion python binding, cli and ballista [\#1137](https://github.com/apache/arrow-datafusion/pull/1137) ([houqp](https://github.com/houqp))
-
-**Closed issues:**
-
-- InList expr with NULL literals do not work [\#1190](https://github.com/apache/arrow-datafusion/issues/1190)
-- update the homepage README to include values, `approx_distinct`, etc. [\#1171](https://github.com/apache/arrow-datafusion/issues/1171)
-- \[Python\]: Inconsistencies with Python package name [\#1011](https://github.com/apache/arrow-datafusion/issues/1011)
-- Wanting to contribute to project where to start? [\#983](https://github.com/apache/arrow-datafusion/issues/983)
-- delete redundant code [\#973](https://github.com/apache/arrow-datafusion/issues/973)
-- How to build DataFusion python wheel [\#853](https://github.com/apache/arrow-datafusion/issues/853)
-- Produce a design for a metrics framework [\#21](https://github.com/apache/arrow-datafusion/issues/21)
-
-**Merged pull requests:**
-
-- \[nit\] simplify ballista executor `CollectExec` impl codes [\#1140](https://github.com/apache/arrow-datafusion/pull/1140) ([panarch](https://github.com/panarch))
-
-For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)
-
-## [ballista-0.5.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.5.0) (2021-08-10)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/4.0.0...ballista-0.5.0)
-
-**Breaking changes:**
-
-- \[ballista\] support date_part and date_turnc ser/de, pass tpch 7 [\#840](https://github.com/apache/arrow-datafusion/pull/840) ([houqp](https://github.com/houqp))
-- Box ScalarValue:Lists, reduce size by half size [\#788](https://github.com/apache/arrow-datafusion/pull/788) ([alamb](https://github.com/alamb))
-- Support DataFrame.collect for Ballista DataFrames [\#785](https://github.com/apache/arrow-datafusion/pull/785) ([andygrove](https://github.com/andygrove))
-- JOIN conditions are order dependent [\#778](https://github.com/apache/arrow-datafusion/pull/778) ([seddonm1](https://github.com/seddonm1))
-- UnresolvedShuffleExec should represent a single shuffle [\#727](https://github.com/apache/arrow-datafusion/pull/727) ([andygrove](https://github.com/andygrove))
-- Ballista: Make shuffle partitions configurable in benchmarks [\#702](https://github.com/apache/arrow-datafusion/pull/702) ([andygrove](https://github.com/andygrove))
-- Rename MergeExec to CoalescePartitionsExec [\#635](https://github.com/apache/arrow-datafusion/pull/635) ([andygrove](https://github.com/andygrove))
-- Ballista: Rename QueryStageExec to ShuffleWriterExec [\#633](https://github.com/apache/arrow-datafusion/pull/633) ([andygrove](https://github.com/andygrove))
-- fix 593, reduce cloning by taking ownership in logical planner's `from` fn [\#610](https://github.com/apache/arrow-datafusion/pull/610) ([Jimexist](https://github.com/Jimexist))
-- fix join column handling logic for `On` and `Using` constraints [\#605](https://github.com/apache/arrow-datafusion/pull/605) ([houqp](https://github.com/houqp))
-- Move ballista standalone mode to client [\#589](https://github.com/apache/arrow-datafusion/pull/589) ([edrevo](https://github.com/edrevo))
-- Ballista: Implement map-side shuffle [\#543](https://github.com/apache/arrow-datafusion/pull/543) ([andygrove](https://github.com/andygrove))
-- ShuffleReaderExec now supports multiple locations per partition [\#541](https://github.com/apache/arrow-datafusion/pull/541) ([andygrove](https://github.com/andygrove))
-- Make external hostname in executor optional [\#232](https://github.com/apache/arrow-datafusion/pull/232) ([edrevo](https://github.com/edrevo))
-- Remove namespace from executors [\#75](https://github.com/apache/arrow-datafusion/pull/75) ([edrevo](https://github.com/edrevo))
-- Support qualified columns in queries [\#55](https://github.com/apache/arrow-datafusion/pull/55) ([houqp](https://github.com/houqp))
-- Read CSV format text from stdin or memory [\#54](https://github.com/apache/arrow-datafusion/pull/54) ([heymind](https://github.com/heymind))
-- Remove Ballista DataFrame [\#48](https://github.com/apache/arrow-datafusion/pull/48) ([andygrove](https://github.com/andygrove))
-- Use atomics for SQLMetric implementation, remove unused name field [\#25](https://github.com/apache/arrow-datafusion/pull/25) ([returnString](https://github.com/returnString))
-
-**Implemented enhancements:**
-
-- Add crate documentation for Ballista crates [\#830](https://github.com/apache/arrow-datafusion/issues/830)
-- Support DataFrame.collect for Ballista DataFrames [\#787](https://github.com/apache/arrow-datafusion/issues/787)
-- Ballista: Prep for supporting shuffle correctly, part one [\#736](https://github.com/apache/arrow-datafusion/issues/736)
-- Ballista: Implement physical plan serde for ShuffleWriterExec [\#710](https://github.com/apache/arrow-datafusion/issues/710)
-- Ballista: Finish implementing shuffle mechanism [\#707](https://github.com/apache/arrow-datafusion/issues/707)
-- Rename QueryStageExec to ShuffleWriterExec [\#542](https://github.com/apache/arrow-datafusion/issues/542)
-- Ballista ShuffleReaderExec should be able to read from multiple locations per partition [\#540](https://github.com/apache/arrow-datafusion/issues/540)
-- \[Ballista\] Use deployments in k8s user guide [\#473](https://github.com/apache/arrow-datafusion/issues/473)
-- Ballista refactor QueryStageExec in preparation for map-side shuffle [\#458](https://github.com/apache/arrow-datafusion/issues/458)
-- Ballista: Implement map-side of shuffle [\#456](https://github.com/apache/arrow-datafusion/issues/456)
-- Refactor Ballista to separate Flight logic from execution logic [\#449](https://github.com/apache/arrow-datafusion/issues/449)
-- Use published versions of arrow rather than github shas [\#393](https://github.com/apache/arrow-datafusion/issues/393)
-- BallistaContext::collect\(\) logging is too noisy [\#352](https://github.com/apache/arrow-datafusion/issues/352)
-- Update Ballista to use new physical plan formatter utility [\#343](https://github.com/apache/arrow-datafusion/issues/343)
-- Add Ballista Getting Started documentation [\#329](https://github.com/apache/arrow-datafusion/issues/329)
-- Remove references to ballistacompute Docker Hub repo [\#325](https://github.com/apache/arrow-datafusion/issues/325)
-- Implement scalable distributed joins [\#63](https://github.com/apache/arrow-datafusion/issues/63)
-- Remove hard-coded Ballista version from scripts [\#32](https://github.com/apache/arrow-datafusion/issues/32)
-- Implement streaming versions of Dataframe.collect methods [\#789](https://github.com/apache/arrow-datafusion/pull/789) ([andygrove](https://github.com/andygrove))
-- Ballista shuffle is finally working as intended, providing scalable distributed joins [\#750](https://github.com/apache/arrow-datafusion/pull/750) ([andygrove](https://github.com/andygrove))
-- Update to use arrow 5.0 [\#721](https://github.com/apache/arrow-datafusion/pull/721) ([alamb](https://github.com/alamb))
-- Implement serde for ShuffleWriterExec [\#712](https://github.com/apache/arrow-datafusion/pull/712) ([andygrove](https://github.com/andygrove))
-- dedup using join column in wildcard expansion [\#678](https://github.com/apache/arrow-datafusion/pull/678) ([houqp](https://github.com/houqp))
-- Implement metrics for shuffle read and write [\#676](https://github.com/apache/arrow-datafusion/pull/676) ([andygrove](https://github.com/andygrove))
-- Remove hard-coded PartitionMode from Ballista serde [\#637](https://github.com/apache/arrow-datafusion/pull/637) ([andygrove](https://github.com/andygrove))
-- Ballista: Implement scalable distributed joins [\#634](https://github.com/apache/arrow-datafusion/pull/634) ([andygrove](https://github.com/andygrove))
-- Add Keda autoscaling for ballista in k8s [\#586](https://github.com/apache/arrow-datafusion/pull/586) ([edrevo](https://github.com/edrevo))
-- Add some resiliency to lost executors [\#568](https://github.com/apache/arrow-datafusion/pull/568) ([edrevo](https://github.com/edrevo))
-- Add `partition by` constructs in window functions and modify logical planning [\#501](https://github.com/apache/arrow-datafusion/pull/501) ([Jimexist](https://github.com/Jimexist))
-- Support anti join [\#482](https://github.com/apache/arrow-datafusion/pull/482) ([Dandandan](https://github.com/Dandandan))
-- add `order by` construct in window function and logical plans [\#463](https://github.com/apache/arrow-datafusion/pull/463) ([Jimexist](https://github.com/Jimexist))
-- Refactor Ballista executor so that FlightService delegates to an Executor struct [\#450](https://github.com/apache/arrow-datafusion/pull/450) ([andygrove](https://github.com/andygrove))
-- implement lead and lag built-in window function [\#429](https://github.com/apache/arrow-datafusion/pull/429) ([Jimexist](https://github.com/Jimexist))
-- Implement fmt_as for ShuffleReaderExec [\#400](https://github.com/apache/arrow-datafusion/pull/400) ([andygrove](https://github.com/andygrove))
-- Add window expression part 1 - logical and physical planning, structure, to/from proto, and explain, for empty over clause only [\#334](https://github.com/apache/arrow-datafusion/pull/334) ([Jimexist](https://github.com/Jimexist))
-- \[breaking change\] fix 265, log should be log10, and add ln [\#271](https://github.com/apache/arrow-datafusion/pull/271) ([Jimexist](https://github.com/Jimexist))
-- Allow table providers to indicate their type for catalog metadata [\#205](https://github.com/apache/arrow-datafusion/pull/205) ([returnString](https://github.com/returnString))
-- Add query 19 to TPC-H regression tests [\#59](https://github.com/apache/arrow-datafusion/pull/59) ([Dandandan](https://github.com/Dandandan))
-- Use arrow eq kernels in CaseWhen expression evaluation [\#52](https://github.com/apache/arrow-datafusion/pull/52) ([Dandandan](https://github.com/Dandandan))
-- Add option param for standalone mode [\#42](https://github.com/apache/arrow-datafusion/pull/42) ([djKooks](https://github.com/djKooks))
-- \[DataFusion\] Optimize hash join inner workings, null handling fix [\#24](https://github.com/apache/arrow-datafusion/pull/24) ([Dandandan](https://github.com/Dandandan))
-- \[Ballista\] Docker files for ui [\#22](https://github.com/apache/arrow-datafusion/pull/22) ([msathis](https://github.com/msathis))
-
-**Fixed bugs:**
-
-- Ballista: TPC-H q3 @ SF=1000 never completes [\#835](https://github.com/apache/arrow-datafusion/issues/835)
-- Ballista does not support MIN/MAX aggregate functions [\#832](https://github.com/apache/arrow-datafusion/issues/832)
-- Ballista docker images fail to build [\#828](https://github.com/apache/arrow-datafusion/issues/828)
-- Ballista: UnresolvedShuffleExec should only have a single stage_id [\#726](https://github.com/apache/arrow-datafusion/issues/726)
-- Ballista integration tests are failing [\#623](https://github.com/apache/arrow-datafusion/issues/623)
-- Integration test build failure due to arrow-rs using unstable feature [\#596](https://github.com/apache/arrow-datafusion/issues/596)
-- `cargo build` cannot build the project [\#531](https://github.com/apache/arrow-datafusion/issues/531)
-- ShuffleReaderExec does not get formatted correctly in displayable physical plan [\#399](https://github.com/apache/arrow-datafusion/issues/399)
-- Implement serde for MIN and MAX [\#833](https://github.com/apache/arrow-datafusion/pull/833) ([andygrove](https://github.com/andygrove))
-- Ballista: Prep for fixing shuffle mechansim, part 1 [\#738](https://github.com/apache/arrow-datafusion/pull/738) ([andygrove](https://github.com/andygrove))
-- Ballista: Shuffle write bug fix [\#714](https://github.com/apache/arrow-datafusion/pull/714) ([andygrove](https://github.com/andygrove))
-- honor table name for csv/parquet scan in ballista plan serde [\#629](https://github.com/apache/arrow-datafusion/pull/629) ([houqp](https://github.com/houqp))
-- MINOR: Fix integration tests by adding datafusion-cli module to docker image [\#322](https://github.com/apache/arrow-datafusion/pull/322) ([andygrove](https://github.com/andygrove))
-
-**Documentation updates:**
-
-- Add minimal crate documentation for Ballista crates [\#831](https://github.com/apache/arrow-datafusion/pull/831) ([andygrove](https://github.com/andygrove))
-- Add Ballista examples [\#775](https://github.com/apache/arrow-datafusion/pull/775) ([andygrove](https://github.com/andygrove))
-- Update ballista.proto link in architecture doc [\#502](https://github.com/apache/arrow-datafusion/pull/502) ([terrycorley](https://github.com/terrycorley))
-- Update k8s user guide to use deployments [\#474](https://github.com/apache/arrow-datafusion/pull/474) ([edrevo](https://github.com/edrevo))
-- use prettier to format md files [\#367](https://github.com/apache/arrow-datafusion/pull/367) ([Jimexist](https://github.com/Jimexist))
-- Make it easier for developers to find Ballista documentation [\#330](https://github.com/apache/arrow-datafusion/pull/330) ([andygrove](https://github.com/andygrove))
-- Instructions for cross-compiling Ballista to the Raspberry Pi [\#263](https://github.com/apache/arrow-datafusion/pull/263) ([andygrove](https://github.com/andygrove))
-- Add install guide in README [\#236](https://github.com/apache/arrow-datafusion/pull/236) ([djKooks](https://github.com/djKooks))
-
-**Performance improvements:**
-
-- Ballista: Avoid sleeping between polling for tasks [\#698](https://github.com/apache/arrow-datafusion/pull/698) ([Dandandan](https://github.com/Dandandan))
-- Make BallistaContext::collect streaming [\#535](https://github.com/apache/arrow-datafusion/pull/535) ([edrevo](https://github.com/edrevo))
-
-**Closed issues:**
-
-- Confirm git tagging strategy for releases [\#770](https://github.com/apache/arrow-datafusion/issues/770)
-- arrow::util::pretty::pretty_format_batches missing [\#769](https://github.com/apache/arrow-datafusion/issues/769)
-- move the `assert_batches_eq!` macros to a non part of datafusion [\#745](https://github.com/apache/arrow-datafusion/issues/745)
-- fix an issue where aliases are not respected in generating downstream schemas in window expr [\#592](https://github.com/apache/arrow-datafusion/issues/592)
-- make the planner to print more succinct and useful information in window function explain clause [\#526](https://github.com/apache/arrow-datafusion/issues/526)
-- move window frame module to be in `logical_plan` [\#517](https://github.com/apache/arrow-datafusion/issues/517)
-- use a more rust idiomatic way of handling nth_value [\#448](https://github.com/apache/arrow-datafusion/issues/448)
-- Make Ballista not depend on arrow directly [\#446](https://github.com/apache/arrow-datafusion/issues/446)
-- create a test with more than one partition for window functions [\#435](https://github.com/apache/arrow-datafusion/issues/435)
-- Implement hash-partitioned hash aggregate [\#27](https://github.com/apache/arrow-datafusion/issues/27)
-- Consider using GitHub pages for DataFusion/Ballista documentation [\#18](https://github.com/apache/arrow-datafusion/issues/18)
-- Add Ballista to default cargo workspace [\#17](https://github.com/apache/arrow-datafusion/issues/17)
-- Update "repository" in Cargo.toml [\#16](https://github.com/apache/arrow-datafusion/issues/16)
-- Consolidate TPC-H benchmarks [\#6](https://github.com/apache/arrow-datafusion/issues/6)
-- \[Ballista\] Fix integration test script [\#4](https://github.com/apache/arrow-datafusion/issues/4)
-- Ballista should not have separate DataFrame implementation [\#2](https://github.com/apache/arrow-datafusion/issues/2)
-
-**Merged pull requests:**
-
-- Change datatype of tpch keys from Int32 to UInt64 to support sf=1000 [\#836](https://github.com/apache/arrow-datafusion/pull/836) ([andygrove](https://github.com/andygrove))
-- Add ballista-examples to docker build [\#829](https://github.com/apache/arrow-datafusion/pull/829) ([andygrove](https://github.com/andygrove))
-- Update dependencies: prost to 0.8 and tonic to 0.5 [\#818](https://github.com/apache/arrow-datafusion/pull/818) ([alamb](https://github.com/alamb))
-- Move `hash_array` into hash_utils.rs [\#807](https://github.com/apache/arrow-datafusion/pull/807) ([alamb](https://github.com/alamb))
-- Fix: Update clippy lints for Rust 1.54 [\#794](https://github.com/apache/arrow-datafusion/pull/794) ([alamb](https://github.com/alamb))
-- MINOR: Remove unused Ballista query execution code path [\#732](https://github.com/apache/arrow-datafusion/pull/732) ([andygrove](https://github.com/andygrove))
-- \[fix\] benchmark run with compose [\#666](https://github.com/apache/arrow-datafusion/pull/666) ([rdettai](https://github.com/rdettai))
-- bring back dev scripts for ballista [\#648](https://github.com/apache/arrow-datafusion/pull/648) ([Jimexist](https://github.com/Jimexist))
-- Remove unnecessary mutex [\#639](https://github.com/apache/arrow-datafusion/pull/639) ([edrevo](https://github.com/edrevo))
-- round trip TPCH queries in tests [\#630](https://github.com/apache/arrow-datafusion/pull/630) ([houqp](https://github.com/houqp))
-- Fix build [\#627](https://github.com/apache/arrow-datafusion/pull/627) ([andygrove](https://github.com/andygrove))
-- in ballista also check for UI prettier changes [\#578](https://github.com/apache/arrow-datafusion/pull/578) ([Jimexist](https://github.com/Jimexist))
-- turn on clippy rule for needless borrow [\#545](https://github.com/apache/arrow-datafusion/pull/545) ([Jimexist](https://github.com/Jimexist))
-- reuse datafusion physical planner in ballista building from protobuf [\#532](https://github.com/apache/arrow-datafusion/pull/532) ([Jimexist](https://github.com/Jimexist))
-- update cargo.toml in python crate and fix unit test due to hash joins [\#483](https://github.com/apache/arrow-datafusion/pull/483) ([Jimexist](https://github.com/Jimexist))
-- make `VOLUME` declaration in tpch datagen docker absolute [\#466](https://github.com/apache/arrow-datafusion/pull/466) ([crepererum](https://github.com/crepererum))
-- Refactor QueryStageExec in preparation for implementing map-side shuffle [\#459](https://github.com/apache/arrow-datafusion/pull/459) ([andygrove](https://github.com/andygrove))
-- Simplified usage of `use arrow` in ballista. [\#447](https://github.com/apache/arrow-datafusion/pull/447) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- Benchmark subcommand to distinguish between DataFusion and Ballista [\#402](https://github.com/apache/arrow-datafusion/pull/402) ([jgoday](https://github.com/jgoday))
-- \#352: BallistaContext::collect\(\) logging is too noisy [\#394](https://github.com/apache/arrow-datafusion/pull/394) ([jgoday](https://github.com/jgoday))
-- cleanup function return type fn [\#350](https://github.com/apache/arrow-datafusion/pull/350) ([Jimexist](https://github.com/Jimexist))
-- Update Ballista to use new physical plan formatter utility [\#344](https://github.com/apache/arrow-datafusion/pull/344) ([andygrove](https://github.com/andygrove))
-- Update arrow dependencies again [\#341](https://github.com/apache/arrow-datafusion/pull/341) ([alamb](https://github.com/alamb))
-- Remove references to Ballista Docker images published to ballistacompute Docker Hub repo [\#326](https://github.com/apache/arrow-datafusion/pull/326) ([andygrove](https://github.com/andygrove))
-- Update arrow-rs deps [\#317](https://github.com/apache/arrow-datafusion/pull/317) ([alamb](https://github.com/alamb))
-- Update arrow deps [\#269](https://github.com/apache/arrow-datafusion/pull/269) ([alamb](https://github.com/alamb))
-- Enable redundant_field_names clippy lint [\#261](https://github.com/apache/arrow-datafusion/pull/261) ([Dandandan](https://github.com/Dandandan))
-- Update arrow-rs deps \(to fix build due to flatbuffers update\) [\#224](https://github.com/apache/arrow-datafusion/pull/224) ([alamb](https://github.com/alamb))
-- update arrow-rs deps to latest master [\#216](https://github.com/apache/arrow-datafusion/pull/216) ([alamb](https://github.com/alamb))
-
-\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_
+Per-release changelogs are available under
+[`docs/source/changelog/`](docs/source/changelog/index.md).
diff --git a/Cargo.lock b/Cargo.lock
index dd2c433c0f..6922e2d9cb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -132,35 +132,6 @@ version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
-[[package]]
-name = "apache-avro"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
-dependencies = [
- "bigdecimal",
- "bon",
- "bzip2",
- "crc32fast",
- "digest 0.10.7",
- "liblzma",
- "log",
- "miniz_oxide 0.8.9",
- "num-bigint",
- "quad-rand",
- "rand 0.9.4",
- "regex-lite",
- "serde",
- "serde_bytes",
- "serde_json",
- "snap",
- "strum 0.27.2",
- "strum_macros 0.27.2",
- "thiserror 2.0.18",
- "uuid",
- "zstd",
-]
-
 [[package]]
 name = "approx"
 version = "0.5.1"
@@ -260,6 +231,30 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-avro"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
+ "bytes",
+ "bzip2",
+ "crc",
+ "flate2",
+ "indexmap 2.14.0",
+ "liblzma",
+ "rand 0.9.4",
+ "serde",
+ "serde_json",
+ "snap",
+ "strum_macros",
+ "uuid",
+ "zstd",
+]
+
 [[package]]
 name = "arrow-buffer"
 version = "58.3.0"
@@ -562,9 +557,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "aws-config"
-version = "1.8.18"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275"
+checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -583,7 +578,7 @@ dependencies = [
  "fastrand",
  "hex",
  "http 1.4.2",
- "sha1",
+ "sha1 0.10.6",
  "time",
  "tokio",
  "tracing",
@@ -593,9 +588,9 @@ dependencies = [
 
 [[package]]
 name = "aws-credential-types"
-version = "1.2.14"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7"
+checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api",
@@ -627,9 +622,9 @@ dependencies = [
 
 [[package]]
 name = "aws-runtime"
-version = "1.7.4"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ed8e8c52d2dc2390ad9f15647fe663f71e9780b4262c190fbb823a32721566"
+checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d"
 dependencies = [
  "aws-credential-types",
  "aws-sigv4",
@@ -652,9 +647,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sso"
-version = "1.101.0"
+version = "1.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b647baea49ff551960b904f905681e9b4765a6c4ea08631e89dc52d8bd3f5896"
+checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -665,6 +660,7 @@ dependencies = [
  "aws-smithy-observability",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
+ "aws-smithy-schema",
  "aws-smithy-types",
  "aws-types",
  "bytes",
@@ -677,9 +673,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.103.0"
+version = "1.105.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ae401c65ff288aa7873117fe535cd32b7b1bb0bc43751d28901a1d5f20636b9"
+checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -690,6 +686,7 @@ dependencies = [
  "aws-smithy-observability",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
+ "aws-smithy-schema",
  "aws-smithy-types",
  "aws-types",
  "bytes",
@@ -702,9 +699,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "1.106.0"
+version = "1.108.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c80de7bb7d03e9ca8c9fd7b489f20f3948d3f3be91a7953591347d238115408"
+checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -716,6 +713,7 @@ dependencies = [
  "aws-smithy-query",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
+ "aws-smithy-schema",
  "aws-smithy-types",
  "aws-smithy-xml",
  "aws-types",
@@ -728,9 +726,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sigv4"
-version = "1.4.5"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71"
+checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-http",
@@ -750,9 +748,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-async"
-version = "1.2.14"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc"
+checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316"
 dependencies = [
  "futures-util",
  "pin-project-lite",
@@ -761,9 +759,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-http"
-version = "0.63.6"
+version = "0.64.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231"
+checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d"
 dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -782,9 +780,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-http-client"
-version = "1.1.13"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3"
+checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api",
@@ -806,9 +804,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-json"
-version = "0.62.7"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
+checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef"
 dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-schema",
@@ -817,18 +815,18 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-observability"
-version = "0.2.6"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c"
+checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c"
 dependencies = [
  "aws-smithy-runtime-api",
 ]
 
 [[package]]
 name = "aws-smithy-query"
-version = "0.60.15"
+version = "0.61.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd"
+checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77"
 dependencies = [
  "aws-smithy-types",
  "urlencoding",
@@ -836,9 +834,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime"
-version = "1.11.3"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182"
+checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
@@ -862,9 +860,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime-api"
-version = "1.12.3"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
+checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api-macros",
@@ -880,9 +878,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime-api-macros"
-version = "1.0.0"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
+checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -891,9 +889,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-schema"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
+checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910"
 dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -902,9 +900,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-types"
-version = "1.5.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
+checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -925,18 +923,21 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-xml"
-version = "0.60.15"
+version = "0.61.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3"
+checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957"
 dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
  "xmlparser",
 ]
 
 [[package]]
 name = "aws-types"
-version = "1.3.16"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531"
+checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-async",
@@ -1001,7 +1002,7 @@ dependencies = [
 
 [[package]]
 name = "ballista"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "async-trait",
  "ballista-core",
@@ -1017,22 +1018,23 @@ dependencies = [
  "tokio",
  "tonic",
  "url",
+ "uuid",
 ]
 
 [[package]]
 name = "ballista-benchmarks"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "ballista",
  "ballista-core",
- "clap 4.6.1",
+ "clap 4.6.2",
  "criterion",
  "datafusion",
  "datafusion-proto",
  "env_logger",
  "futures",
  "mimalloc",
- "rand 0.10.1",
+ "rand 0.10.2",
  "serde",
  "serde_json",
  "structopt",
@@ -1042,11 +1044,11 @@ dependencies = [
 
 [[package]]
 name = "ballista-cli"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "ballista",
  "chrono",
- "clap 4.6.1",
+ "clap 4.6.2",
  "config",
  "console_error_panic_hook",
  "critical-section",
@@ -1072,7 +1074,6 @@ dependencies = [
  "tracing-appender",
  "tracing-subscriber",
  "tracing-web",
- "tui-big-text",
  "tui-shimmer",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -1081,14 +1082,14 @@ dependencies = [
 
 [[package]]
 name = "ballista-core"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow-flight",
  "async-trait",
  "aws-config",
  "aws-credential-types",
  "chrono",
- "clap 4.6.1",
+ "clap 4.6.2",
  "datafusion",
  "datafusion-proto",
  "datafusion-proto-common",
@@ -1101,7 +1102,7 @@ dependencies = [
  "parking_lot",
  "prost",
  "prost-types",
- "rand 0.10.1",
+ "rand 0.10.2",
  "rustc_version",
  "serde",
  "tempfile",
@@ -1117,7 +1118,7 @@ dependencies = [
 
 [[package]]
 name = "ballista-examples"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow-flight",
  "ballista",
@@ -1143,20 +1144,22 @@ dependencies = [
 
 [[package]]
 name = "ballista-executor"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow",
  "arrow-flight",
  "async-trait",
+ "axum",
  "ballista-core",
  "bytesize",
- "clap 4.6.1",
+ "clap 4.6.2",
  "dashmap",
  "datafusion",
  "datafusion-proto",
  "futures",
  "libc",
  "log",
+ "lru 0.18.1",
  "memory-stats",
  "mimalloc",
  "parking_lot",
@@ -1175,13 +1178,13 @@ dependencies = [
 
 [[package]]
 name = "ballista-scheduler"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow-flight",
  "async-trait",
  "axum",
  "ballista-core",
- "clap 4.6.1",
+ "clap 4.6.2",
  "dashmap",
  "datafusion",
  "datafusion-proto",
@@ -1198,7 +1201,8 @@ dependencies = [
  "prometheus",
  "prost",
  "prost-types",
- "rand 0.10.1",
+ "rand 0.10.2",
+ "regex",
  "rstest",
  "serde",
  "tokio",
@@ -1297,7 +1301,6 @@ dependencies = [
  "num-bigint",
  "num-integer",
  "num-traits",
- "serde",
 ]
 
 [[package]]
@@ -1457,31 +1460,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "bon"
-version = "3.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b"
-dependencies = [
- "darling 0.23.0",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "brotli"
 version = "8.0.3"
@@ -1554,9 +1532,9 @@ dependencies = [
 
 [[package]]
 name = "bytesize"
-version = "2.4.0"
+version = "2.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d"
+checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
 
 [[package]]
 name = "bzip2"
@@ -1679,9 +1657,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1689,9 +1667,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1805,9 +1783,9 @@ checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
 
 [[package]]
 name = "config"
-version = "0.15.24"
+version = "0.15.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b34d0237145f33580b89724f75d16950efd3e2c91b2d823917ecb69ec7f84f0"
+checksum = "b85f248a4de22d204ceabc6299d89d2c70fbd7f09fea53c06c852369652d8139"
 dependencies = [
  "pathdiff",
  "serde_core",
@@ -1921,6 +1899,21 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -1940,7 +1933,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.6.1",
+ "clap 4.6.2",
  "criterion-plot",
  "itertools 0.13.0",
  "num-traits",
@@ -2109,9 +2102,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "1.0.7"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01334b89b69ff726750c5ce5073fc8bd860e99aa9a8fc5ca11b04730e3aee97a"
+checksum = "a394189d59f9befacce833f337f7b1eca5e9a91221bcdd4d28e0114d96e597b3"
 dependencies = [
  "link-section",
  "linktime-proc-macro",
@@ -2126,38 +2119,14 @@ dependencies = [
  "cmov",
 ]
 
-[[package]]
-name = "darling"
-version = "0.20.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
-dependencies = [
- "darling_core 0.20.11",
- "darling_macro 0.20.11",
-]
-
 [[package]]
 name = "darling"
 version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
 dependencies = [
- "darling_core 0.23.0",
- "darling_macro 0.23.0",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.20.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
-dependencies = [
- "fnv",
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn 2.0.117",
+ "darling_core",
+ "darling_macro",
 ]
 
 [[package]]
@@ -2173,24 +2142,13 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "darling_macro"
-version = "0.20.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
-dependencies = [
- "darling_core 0.20.11",
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "darling_macro"
 version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
 dependencies = [
- "darling_core 0.23.0",
+ "darling_core",
  "quote",
  "syn 2.0.117",
 ]
@@ -2211,14 +2169,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "bzip2",
  "chrono",
  "datafusion-catalog",
@@ -2249,14 +2206,13 @@ dependencies = [
  "datafusion-sql",
  "flate2",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
  "liblzma",
  "log",
  "object_store",
  "parking_lot",
  "parquet",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -2267,9 +2223,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2292,9 +2248,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2315,16 +2271,16 @@ dependencies = [
 
 [[package]]
 name = "datafusion-cli"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a22c001ad1ac11cda09dab69b151eef5b1a992e23bc524ab0d1e63e5dea327"
+checksum = "2a9533845ab888508718c2ad540cff205d1111525a9952def8b0e584b6453784"
 dependencies = [
  "arrow",
  "async-trait",
  "aws-config",
  "aws-credential-types",
  "chrono",
- "clap 4.6.1",
+ "clap 4.6.2",
  "datafusion",
  "datafusion-common",
  "dirs",
@@ -2343,17 +2299,17 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
 dependencies = [
- "ahash",
- "apache-avro",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "hex",
  "indexmap 2.14.0",
  "itertools 0.14.0",
@@ -2361,18 +2317,18 @@ dependencies = [
  "log",
  "object_store",
  "parquet",
- "paste",
  "recursive",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
 dependencies = [
  "futures",
  "log",
@@ -2381,9 +2337,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
 dependencies = [
  "arrow",
  "async-compression",
@@ -2407,6 +2363,7 @@ dependencies = [
  "liblzma",
  "log",
  "object_store",
+ "parking_lot",
  "rand 0.9.4",
  "tokio",
  "tokio-util",
@@ -2416,9 +2373,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -2440,29 +2397,28 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-avro"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4"
+checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195"
 dependencies = [
- "apache-avro",
  "arrow",
+ "arrow-avro",
  "async-trait",
  "bytes",
  "datafusion-common",
  "datafusion-datasource",
- "datafusion-physical-expr-common",
+ "datafusion-physical-expr-adapter",
  "datafusion-physical-plan",
  "datafusion-session",
  "futures",
- "num-traits",
  "object_store",
 ]
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2483,9 +2439,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2500,16 +2456,15 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
+checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2519,6 +2474,7 @@ dependencies = [
  "datafusion-datasource",
  "datafusion-execution",
  "datafusion-expr",
+ "datafusion-functions",
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-adapter",
@@ -2537,20 +2493,19 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -2567,11 +2522,12 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -2582,7 +2538,6 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
  "recursive",
  "serde_json",
  "sqlparser",
@@ -2590,22 +2545,21 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2620,26 +2574,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
  "rand 0.9.4",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -2649,19 +2602,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -2670,9 +2622,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2686,34 +2638,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2724,14 +2676,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2739,9 +2690,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
 dependencies = [
  "datafusion-doc",
  "quote",
@@ -2750,9 +2701,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
 dependencies = [
  "arrow",
  "chrono",
@@ -2770,11 +2721,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2782,11 +2732,10 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "recursive",
  "tokio",
@@ -2794,9 +2743,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2809,26 +2758,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2845,12 +2794,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2865,7 +2815,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2877,9 +2827,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-proto"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
+checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae"
 dependencies = [
  "arrow",
  "chrono",
@@ -2900,14 +2850,13 @@ dependencies = [
  "datafusion-proto-common",
  "object_store",
  "prost",
- "rand 0.9.4",
 ]
 
 [[package]]
 name = "datafusion-proto-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
+checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2916,9 +2865,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2927,15 +2876,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2947,9 +2895,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-spark"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5"
+checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2962,21 +2910,24 @@ dependencies = [
  "datafusion-expr",
  "datafusion-functions",
  "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
  "datafusion-functions-nested",
  "log",
+ "num-traits",
  "percent-encoding",
  "rand 0.9.4",
  "serde_json",
- "sha1",
- "sha2 0.10.9",
+ "sha1 0.11.0",
+ "sha2 0.11.0",
+ "twox-hash",
  "url",
 ]
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2993,9 +2944,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -3027,37 +2978,6 @@ dependencies = [
  "serde_core",
 ]
 
-[[package]]
-name = "derive_builder"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
-dependencies = [
- "derive_builder_macro",
-]
-
-[[package]]
-name = "derive_builder_core"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
-dependencies = [
- "darling 0.20.11",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
-
-[[package]]
-name = "derive_builder_macro"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
-dependencies = [
- "derive_builder_core",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "derive_more"
 version = "2.1.1"
@@ -3216,15 +3136,15 @@ dependencies = [
 
 [[package]]
 name = "endian-type"
-version = "0.1.2"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2"
 
 [[package]]
 name = "env_filter"
-version = "1.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
 dependencies = [
  "log",
  "regex",
@@ -3232,9 +3152,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.10"
+version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
 dependencies = [
  "anstream",
  "anstyle",
@@ -3306,17 +3226,6 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
 
-[[package]]
-name = "fd-lock"
-version = "4.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"
-dependencies = [
- "cfg-if",
- "rustix 1.1.4",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "ferroid"
 version = "2.0.0"
@@ -3324,7 +3233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4"
 dependencies = [
  "portable-atomic",
- "rand 0.10.1",
+ "rand 0.10.2",
  "web-time",
 ]
 
@@ -3412,12 +3321,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
 
-[[package]]
-name = "font8x8"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -4129,7 +4032,7 @@ version = "0.3.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971"
 dependencies = [
- "darling 0.23.0",
+ "darling",
  "indoc",
  "proc-macro2",
  "quote",
@@ -4465,9 +4368,9 @@ dependencies = [
 
 [[package]]
 name = "link-section"
-version = "0.18.2"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2b1dd6fe32e55c0fc0ea9493aa57459ca3cf4ff3c857c7d0302290150da6e4f"
+checksum = "e333fe507b738576d6da5bb3f1a7d7a1c80307ed9ef31624c057d844c19c93e9"
 
 [[package]]
 name = "linktime-proc-macro"
@@ -4525,9 +4428,9 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.18.0"
+version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
+checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
 dependencies = [
  "hashbrown 0.17.1",
 ]
@@ -4705,9 +4608,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.30.1"
+version = "0.31.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
+checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
 dependencies = [
  "bitflags 2.13.0",
  "cfg-if",
@@ -4765,7 +4668,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
 dependencies = [
  "num-integer",
  "num-traits",
- "serde",
 ]
 
 [[package]]
@@ -4901,7 +4803,7 @@ dependencies = [
  "parking_lot",
  "percent-encoding",
  "quick-xml",
- "rand 0.10.1",
+ "rand 0.10.2",
  "reqwest 0.12.28",
  "ring",
  "rustls-pki-types",
@@ -5581,12 +5483,6 @@ dependencies = [
  "pulldown-cmark",
 ]
 
-[[package]]
-name = "quad-rand"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
-
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -5682,9 +5578,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
 [[package]]
 name = "radix_trie"
-version = "0.2.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a"
 dependencies = [
  "endian-type",
  "nibble_vec",
@@ -5711,9 +5607,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
+checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
 dependencies = [
  "chacha20",
  "getrandom 0.4.2",
@@ -5780,10 +5676,10 @@ dependencies = [
  "hashbrown 0.17.1",
  "itertools 0.14.0",
  "kasuari",
- "lru 0.18.0",
+ "lru 0.18.1",
  "palette",
  "serde",
- "strum 0.28.0",
+ "strum",
  "thiserror 2.0.18",
  "unicode-segmentation",
  "unicode-truncate",
@@ -5859,7 +5755,7 @@ dependencies = [
  "line-clipping",
  "ratatui-core",
  "serde",
- "strum 0.28.0",
+ "strum",
  "time",
  "unicode-segmentation",
  "unicode-width 0.2.2",
@@ -5963,9 +5859,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.12.4"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -5975,9 +5871,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.14"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -6186,9 +6082,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.40"
+version = "0.23.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
+checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -6269,24 +6165,23 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
 [[package]]
 name = "rustyline"
-version = "17.0.2"
+version = "18.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564"
+checksum = "53f6a737db68eb1a8ccff86b584b2fc13eca6a7bb6f78ebc7c529547e3ab9684"
 dependencies = [
  "bitflags 2.13.0",
  "cfg-if",
  "clipboard-win",
- "fd-lock",
  "home",
  "libc",
  "log",
  "memchr",
- "nix 0.30.1",
+ "nix 0.31.3",
  "radix_trie",
  "unicode-segmentation",
  "unicode-width 0.2.2",
  "utf8parse",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6416,16 +6311,6 @@ dependencies = [
  "serde_derive",
 ]
 
-[[package]]
-name = "serde_bytes"
-version = "0.11.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
-dependencies = [
- "serde",
- "serde_core",
-]
-
 [[package]]
 name = "serde_core"
 version = "1.0.228"
@@ -6463,6 +6348,7 @@ version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -6542,7 +6428,7 @@ version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
 dependencies = [
- "darling 0.23.0",
+ "darling",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -6572,6 +6458,17 @@ dependencies = [
  "digest 0.10.7",
 ]
 
+[[package]]
+name = "sha1"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "digest 0.11.3",
+]
+
 [[package]]
 name = "sha2"
 version = "0.10.9"
@@ -6713,9 +6610,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "recursive",
@@ -6811,31 +6708,13 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "strum"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
-
 [[package]]
 name = "strum"
 version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
 dependencies = [
- "strum_macros 0.28.0",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
+ "strum_macros",
 ]
 
 [[package]]
@@ -6852,11 +6731,12 @@ dependencies = [
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck 0.5.0",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -7259,9 +7139,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.52.3"
+version = "1.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
 dependencies = [
  "bytes",
  "libc",
@@ -7578,19 +7458,6 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "tui-big-text"
-version = "0.8.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e36b4d581e4e44186f87156c1c281f066ad792551580acc0e02aa7333f769ed"
-dependencies = [
- "derive_builder",
- "font8x8",
- "itertools 0.15.0",
- "ratatui-core",
- "ratatui-widgets",
-]
-
 [[package]]
 name = "tui-shimmer"
 version = "0.1.3"
@@ -7605,6 +7472,9 @@ name = "twox-hash"
 version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+dependencies = [
+ "rand 0.9.4",
+]
 
 [[package]]
 name = "typenum"
@@ -7793,9 +7663,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.3"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
  "atomic",
  "getrandom 0.4.2",
diff --git a/Cargo.toml b/Cargo.toml
index 57346eaf05..5719b9b16c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,20 +26,20 @@ edition = "2024"
 rust-version = "1.88.0"
 
 [workspace.dependencies]
-arrow = { version = "58", features = ["ipc_compression"] }
-arrow-flight = { version = "58", features = ["flight-sql-experimental"] }
+arrow = { version = "58.3", features = ["ipc_compression"] }
+arrow-flight = { version = "58.3", features = ["flight-sql-experimental"] }
 clap = { version = "4.5", features = ["derive", "cargo"] }
 
-datafusion = "53"
-datafusion-cli = "53"
-datafusion-proto = "53"
-datafusion-proto-common = "53"
-datafusion-spark = "53"
-datafusion-substrait = "53"
+datafusion = "54"
+datafusion-cli = "54"
+datafusion-proto = "54"
+datafusion-proto-common = "54"
+datafusion-spark = "54"
+datafusion-substrait = "54"
 
 insta = "1.47"
 itertools = "0.15"
-object_store = "0.13"
+object_store = "0.13.2"
 prost = "0.14"
 prost-types = "0.14"
 rstest = { version = "0.26" }
@@ -101,4 +101,18 @@ strip = "debuginfo"
 debug = false
 debug-assertions = false
 strip = "debuginfo"
+incremental = false
+
+# Build profile for the TPC-H integration CI job. That job only checks that the
+# distributed suite runs correctly (no timing gate), so it trades some runtime
+# speed for much faster compilation: opt-level 1 keeps queries fast enough while
+# compiling dependencies far quicker than the opt-level 3 release profiles. Do
+# not use this for performance numbers — use release-nonlto / release-lto.
+[profile.tpch-ci]
+inherits = "release"
+opt-level = 1
+codegen-units = 256
+lto = false
+debug = false
+debug-assertions = false
 incremental = false
\ No newline at end of file
diff --git a/README.md b/README.md
index b95dd5256e..3adccbb157 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,16 @@ async fn main() -> datafusion::error::Result<()> {
 
 For documentation or more examples, please refer to the [Ballista User Guide][user-guide].
 
+## Who is Ballista for
+
+Ballista serves several distinct audiences:
+
+- **DataFusion users going multi-node** — you already use [Apache DataFusion](https://github.com/apache/datafusion) on a single machine and have outgrown it. Ballista runs the same SQL and DataFrame workloads across a cluster with minimal code changes and the same results.
+- **Spark users wanting the same execution model** — you run Spark SQL or batch jobs and want a lighter, Rust-native alternative without relearning a new paradigm. Ballista keeps the familiar model: plans split into stages at shuffle boundaries, one task per partition, executors with vcores, and adaptive query execution (AQE).
+- **Library users building a specialized engine** — you are building a bespoke distributed query engine and want reusable scheduler, executor, and plan-serialization building blocks with extension points, instead of writing distributed execution from scratch.
+
+These audiences are documented in more detail, along with the guarantees each relies on, in the [User Personas](docs/source/contributors-guide/user-personas.md) guide.
+
 ## Architecture
 
 A Ballista cluster consists of one or more scheduler processes and one or more executor processes. These processes
@@ -103,30 +113,6 @@ between the executor(s) and the scheduler for fetching tasks and reporting task
 
 See the [architecture guide](docs/source/contributors-guide/architecture.md) for more details.
 
-## Performance
-
-We run some simple benchmarks comparing Ballista with Apache Spark to track progress with performance optimizations.
-These are benchmarks derived from TPC-H and not official TPC-H benchmarks. These results are from running individual
-queries at scale factor 100 (100 GB) on a single node with a single executor and 8 concurrent tasks.
-
-### Overall Speedup
-
-The overall speedup is 2.9x
-
-![benchmarks](docs/source/_static/images/tpch_allqueries.png)
-
-### Per Query Comparison
-
-![benchmarks](docs/source/_static/images/tpch_queries_compare.png)
-
-### Relative Speedup
-
-![benchmarks](docs/source/_static/images/tpch_queries_speedup_rel.png)
-
-### Absolute Speedup
-
-![benchmarks](docs/source/_static/images/tpch_queries_speedup_abs.png)
-
 ## Getting Started
 
 The easiest way to get started is to run one of the standalone or distributed [examples](./examples/README.md). After
@@ -202,6 +188,15 @@ but still there is a gap between DataFusion and Ballista which we want to bridge
 Refer to the [DataFusion SQL Reference](https://datafusion.apache.org/user-guide/sql/index.html) for more
 information on supported SQL.
 
+## Who uses Ballista
+
+The following organizations use Ballista. To add yours, open a pull request.
+
+| Organization                                                                                                                   |                                                               |
+| ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------- |
+| <a href="https://spice.ai"><img src="docs/source/_static/images/adopters/spiceai.png" height="36" alt="Spice AI"/></a>         | [Spice AI](https://spice.ai/blog/apache-ballista-at-spice-ai) |
+| <a href="https://coralogix.com"><img src="docs/source/_static/images/adopters/coralogix.png" height="36" alt="Coralogix"/></a> | [Coralogix](https://coralogix.com/)                           |
+
 ## Contribution Guide
 
 Please see the [Contribution Guide](CONTRIBUTING.md) for information about contributing to Ballista.
diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml
index 6796f49eeb..0344a3b5c9 100644
--- a/ballista-cli/Cargo.toml
+++ b/ballista-cli/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "ballista-cli"
 description = "Command Line Client for Ballista distributed query engine."
-version = "53.0.0"
+version = "54.0.0"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
 edition = { workspace = true }
 rust-version = { workspace = true }
@@ -33,10 +33,10 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 # CLI-only deps (DataFusion/Ballista) — gated by `cli` feature
-ballista = { path = "../ballista/client", version = "53.0.0", features = ["standalone"], optional = true }
+ballista = { path = "../ballista/client", version = "54.0.0", features = ["standalone"], optional = true }
 datafusion = { workspace = true, optional = true }
 datafusion-cli = { workspace = true, optional = true }
-rustyline = { version = "17.0.1", optional = true }
+rustyline = { version = "18.0.0", optional = true }
 mimalloc = { workspace = true, optional = true }
 
 # TUI/web shared deps
@@ -59,7 +59,6 @@ tui-shimmer = { version = "0.1", optional = true }
 # Native-only TUI deps
 crossterm = { version = "0.29.0", features = ["event-stream"], optional = true }
 tracing-appender = { version = "0.2", optional = true }
-tui-big-text = { version = "0.8.4", optional = true }
 
 # Web-only deps (WASM)
 critical-section = { version = "1.2.0", features = ["std"], optional = true }
@@ -89,7 +88,7 @@ cli = [
 tui = [
     "dep:chrono", "dep:config", "dep:crossterm", "dep:dotparser", "dep:futures",
     "dep:percent-encoding", "dep:prometheus-parse", "dep:ratatui", "dep:reqwest",
-    "dep:serde", "dep:serde_json", "dep:tracing-appender", "dep:tui-big-text",
+    "dep:serde", "dep:serde_json", "dep:tracing-appender",
     "dep:tui-shimmer", "ratatui/crossterm", "ratatui/all-widgets", "ratatui/macros",
     "ratatui/layout-cache", "ratatui/underline-color", "ratatui/serde"
 ]
diff --git a/ballista-cli/README.md b/ballista-cli/README.md
index 79b8d72642..9bb0c2053f 100644
--- a/ballista-cli/README.md
+++ b/ballista-cli/README.md
@@ -35,8 +35,8 @@ OPTIONS:
         --color
             Enables console syntax highlighting
 
-        --concurrent-tasks <CONCURRENT_TASKS>
-            The max concurrent tasks, only for Ballista local mode. Default: all available cores
+        --vcores <VCORES>
+            Virtual cores for the local Ballista executor. Default: all available physical cores.
 
     -f, --file <FILE>...
             Execute commands from file(s), then exit
diff --git a/ballista-cli/Trunk.toml b/ballista-cli/Trunk.toml
index 5f59741483..37f83dd7e9 100644
--- a/ballista-cli/Trunk.toml
+++ b/ballista-cli/Trunk.toml
@@ -28,15 +28,7 @@
 [build]
 target = "ballista-web-tui.html"
 dist = "../target/web-tui"
+public_url = "./"
 
 [watch]
 ignore = []
-
-[[hooks]]
-stage = "build"
-command = "sh"
-command_arguments = ["-c", "cp -f ./src/tui/ui/header/ballista-logo.svg \"$TRUNK_STAGING_DIR/ballista-logo.svg\""]
-
-[hooks.windows]
-command = "cmd"
-command_arguments = ["/c", "copy .\\src\\tui\\ui\\header\\ballista-logo.svg \"%TRUNK_STAGING_DIR%\\ballista-logo.svg\""]
\ No newline at end of file
diff --git a/ballista-cli/ballista-web-tui.html b/ballista-cli/ballista-web-tui.html
index e55be1653b..ea79918a22 100644
--- a/ballista-cli/ballista-web-tui.html
+++ b/ballista-cli/ballista-web-tui.html
@@ -27,23 +27,9 @@
             margin: 0;
             background: #000;
         }
-
-        #ballista-logo {
-            position: absolute;
-            top: 0;
-            left: 0;
-            width: 30%;
-            max-width: 500px;
-            max-height: 15vh;
-            object-fit: contain;
-            pointer-events: none;
-            z-index: 10;
-        }
     </style>
-    <link data-trunk rel="copy-file" href="./src/tui/ui/header/ballista-logo.svg"/>
 </head>
 <body>
-<img src="ballista-logo.svg" id="ballista-logo" alt="Ballista Logo">
 
 <script type="module">
     document.addEventListener('keydown', (event) => {
diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs
index da76724c7b..3b8cff7193 100644
--- a/ballista-cli/src/main.rs
+++ b/ballista-cli/src/main.rs
@@ -62,8 +62,17 @@ struct Args {
 
     #[clap(
         long,
-        help = "The max concurrent tasks, only for Ballista local mode. Default: all available cores",
-        value_parser(parse_valid_concurrent_tasks_size)
+        help = "Virtual cores for the local Ballista executor. Default: all available physical cores.",
+        value_parser(parse_valid_vcores)
+    )]
+    vcores: Option<usize>,
+
+    #[clap(
+        long = "concurrent-tasks",
+        hide = true,
+        conflicts_with = "vcores",
+        help = "[deprecated] renamed to --vcores",
+        value_parser(parse_valid_vcores)
     )]
     concurrent_tasks: Option<usize>,
 
@@ -153,9 +162,17 @@ pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
             SessionContext::remote_with_state(&address, state).await?
         }
         _ => {
-            if let Some(concurrent_tasks) = args.concurrent_tasks {
-                ballista_config =
-                    ballista_config.with_target_partitions(concurrent_tasks);
+            let vcores = match args.concurrent_tasks {
+                Some(value) => {
+                    eprintln!(
+                        "warning: --concurrent-tasks is deprecated; use --vcores instead"
+                    );
+                    Some(value)
+                }
+                None => args.vcores,
+            };
+            if let Some(vcores) = vcores {
+                ballista_config = ballista_config.with_target_partitions(vcores);
             };
             let state = SessionStateBuilder::new()
                 .with_config(ballista_config)
@@ -225,9 +242,9 @@ fn parse_batch_size(size: &str) -> std::result::Result<usize, String> {
     }
 }
 
-fn parse_valid_concurrent_tasks_size(size: &str) -> std::result::Result<usize, String> {
+fn parse_valid_vcores(size: &str) -> std::result::Result<usize, String> {
     match size.parse::<usize>() {
         Ok(size) if size > 0 => Ok(size),
-        _ => Err(format!("Invalid concurrent_tasks size '{size}'")),
+        _ => Err(format!("Invalid vcores value '{size}'")),
     }
 }
diff --git a/ballista-cli/src/tui/app.rs b/ballista-cli/src/tui/app.rs
index 4345f18db8..8dc7a5165e 100644
--- a/ballista-cli/src/tui/app.rs
+++ b/ballista-cli/src/tui/app.rs
@@ -477,7 +477,7 @@ impl App {
                 if self.is_jobs_view() {
                     self.sort_jobs_by(JobsSortColumn::StagesCompleted);
                 } else if self.is_executors_view() {
-                    self.sort_executors_by(ExecutorsSortColumn::TaskSlots);
+                    self.sort_executors_by(ExecutorsSortColumn::Vcores);
                 }
             }
             KeyCode::Char('5') => {
@@ -1209,7 +1209,7 @@ impl App {
                 if self.is_jobs_view() {
                     self.sort_jobs_by(JobsSortColumn::StagesCompleted);
                 } else if self.is_executors_view() {
-                    self.sort_executors_by(ExecutorsSortColumn::TaskSlots);
+                    self.sort_executors_by(ExecutorsSortColumn::Vcores);
                 }
                 None
             }
@@ -1614,7 +1614,7 @@ mod tests {
             port: 8080,
             id: id.to_string(),
             last_seen: None,
-            specification: Specification { task_slots: 4 },
+            specification: Specification { vcores: 4 },
             metrics: vec![],
             os_info: OsInfo {
                 kernel_ver: "5.15".to_string(),
diff --git a/ballista-cli/src/tui/domain/executors.rs b/ballista-cli/src/tui/domain/executors.rs
index d65fb7535c..0e8933480f 100644
--- a/ballista-cli/src/tui/domain/executors.rs
+++ b/ballista-cli/src/tui/domain/executors.rs
@@ -35,8 +35,8 @@ impl Executor {
         self.os_info.physical_cores
     }
 
-    pub fn task_slots(&self) -> u32 {
-        self.specification.task_slots
+    pub fn vcores(&self) -> u32 {
+        self.specification.vcores
     }
 
     pub fn proc_physical_memory_usage(&self) -> u64 {
@@ -65,7 +65,7 @@ pub struct Metric {
 
 #[derive(Deserialize, Clone, Debug)]
 pub struct Specification {
-    pub task_slots: u32,
+    pub vcores: u32,
 }
 
 #[derive(Deserialize, Clone, Debug)]
@@ -113,7 +113,7 @@ pub enum SortColumn {
     Host,
     Id,
     CpuCores,
-    TaskSlots,
+    Vcores,
     ProcPhysicalMemoryUsage,
     PeakPhysicalMemoryUsage,
     LastSeen,
@@ -171,8 +171,8 @@ impl ExecutorsData {
                     cmp
                 }
             }),
-            SortColumn::TaskSlots => self.executors.sort_by(|a, b| {
-                let cmp = a.task_slots().cmp(&b.task_slots());
+            SortColumn::Vcores => self.executors.sort_by(|a, b| {
+                let cmp = a.vcores().cmp(&b.vcores());
                 if self.sort_order == SortOrder::Descending {
                     cmp.reverse()
                 } else {
@@ -270,7 +270,7 @@ mod tests {
         host: &str,
         port: u16,
         id: &str,
-        task_slots: u32,
+        vcores: u32,
         proc_physical_memory_usage: u64,
         peak_physical_memory_usage: u64,
         last_seen: Option<u64>,
@@ -280,7 +280,7 @@ mod tests {
             port,
             id: id.to_string(),
             last_seen,
-            specification: Specification { task_slots },
+            specification: Specification { vcores },
             metrics: vec![
                 Metric {
                     typ: "proc_physical_memory".to_string(),
@@ -468,43 +468,43 @@ mod tests {
     }
 
     #[test]
-    fn sort_by_task_slots_ascending() {
+    fn sort_by_vcores_ascending() {
         let mut data = make_executors_data(
             vec![
                 make_executor("host", 8080, "id-a", 2, 0, 0, Some(100)),
                 make_executor("host", 8080, "id-b", 3, 0, 0, Some(200)),
                 make_executor("host", 8080, "id-c", 1, 0, 0, Some(300)),
             ],
-            SortColumn::TaskSlots,
+            SortColumn::Vcores,
             SortOrder::Ascending,
         );
         data.sort();
         assert_eq!(data.executors[0].id, "id-c");
-        assert_eq!(data.executors[0].task_slots(), 1);
+        assert_eq!(data.executors[0].vcores(), 1);
         assert_eq!(data.executors[1].id, "id-a");
-        assert_eq!(data.executors[1].task_slots(), 2);
+        assert_eq!(data.executors[1].vcores(), 2);
         assert_eq!(data.executors[2].id, "id-b");
-        assert_eq!(data.executors[2].task_slots(), 3);
+        assert_eq!(data.executors[2].vcores(), 3);
     }
 
     #[test]
-    fn sort_by_task_slots_descending() {
+    fn sort_by_vcores_descending() {
         let mut data = make_executors_data(
             vec![
                 make_executor("host", 8080, "id-a", 1, 0, 0, Some(100)),
                 make_executor("host", 8080, "id-c", 2, 0, 0, Some(300)),
                 make_executor("host", 8080, "id-b", 3, 0, 0, Some(200)),
             ],
-            SortColumn::TaskSlots,
+            SortColumn::Vcores,
             SortOrder::Descending,
         );
         data.sort();
         assert_eq!(data.executors[0].id, "id-b");
-        assert_eq!(data.executors[0].task_slots(), 3);
+        assert_eq!(data.executors[0].vcores(), 3);
         assert_eq!(data.executors[1].id, "id-c");
-        assert_eq!(data.executors[1].task_slots(), 2);
+        assert_eq!(data.executors[1].vcores(), 2);
         assert_eq!(data.executors[2].id, "id-a");
-        assert_eq!(data.executors[2].task_slots(), 1);
+        assert_eq!(data.executors[2].vcores(), 1);
     }
 
     #[test]
diff --git a/ballista-cli/src/tui/domain/jobs/stages.rs b/ballista-cli/src/tui/domain/jobs/stages.rs
index 3297aca433..5b81d04109 100644
--- a/ballista-cli/src/tui/domain/jobs/stages.rs
+++ b/ballista-cli/src/tui/domain/jobs/stages.rs
@@ -43,11 +43,19 @@ pub struct JobStageResponse {
     pub tasks: Vec<Option<StageTaskResponse>>,
 }
 
+// TaskStatus
+#[derive(Deserialize, Clone, Debug)]
+pub enum StageTaskStatus {
+    Running,
+    Successful,
+    Failed { reason: String },
+}
+
 // TaskSummary
 #[derive(Deserialize, Clone, Debug)]
 pub struct StageTaskResponse {
     pub id: usize,
-    pub status: String,
+    pub status: StageTaskStatus,
     pub partition_id: u32,
     pub input_rows: usize,
     pub output_rows: usize,
@@ -364,7 +372,7 @@ impl StagesGraph {
 mod tests {
     use super::{
         JobStageResponse, JobStagesPopup, JobStagesResponse, StageTaskResponse,
-        StagesGraph, TaskPercentiles,
+        StageTaskStatus, StagesGraph, TaskPercentiles,
     };
 
     fn make_percentiles() -> TaskPercentiles {
@@ -608,7 +616,7 @@ mod tests {
     fn make_task(id: usize) -> StageTaskResponse {
         StageTaskResponse {
             id,
-            status: "Completed".to_string(),
+            status: StageTaskStatus::Successful,
             partition_id: id as u32,
             input_rows: 0,
             output_rows: 0,
diff --git a/ballista-cli/src/tui/infrastructure/theme.rs b/ballista-cli/src/tui/infrastructure/theme.rs
index c23a093977..e6612866fc 100644
--- a/ballista-cli/src/tui/infrastructure/theme.rs
+++ b/ballista-cli/src/tui/infrastructure/theme.rs
@@ -66,7 +66,6 @@ pub(super) struct ThemeOverride {
     pub cancel_success: Option<Style>,
     pub cancel_not_done: Option<Style>,
     pub cancel_failure: Option<Style>,
-    pub banner: Option<Style>,
     pub feature_enabled: Option<Style>,
     pub feature_disabled: Option<Style>,
     pub text_error: Option<Style>,
@@ -142,7 +141,6 @@ pub struct Theme {
     pub footer: Style,
 
     // ── Misc ──────────────────────────────────────────────────────────────
-    pub banner: Style,
     pub text_error: Style,
     pub text_info: Style,
     pub app_background: Style,
@@ -197,8 +195,6 @@ impl Theme {
             cancel_not_done: Style::default().fg(Color::LightYellow),
             cancel_failure: Style::default().fg(Color::LightRed),
 
-            banner: Style::default().fg(Color::Yellow),
-
             feature_enabled: Style::default().fg(Color::LightGreen),
             feature_disabled: Style::default().fg(Color::LightRed),
 
@@ -266,8 +262,6 @@ impl Theme {
             cancel_not_done: Style::default().fg(Color::Indexed(130)),
             cancel_failure: Style::default().fg(Color::Indexed(124)),
 
-            banner: Style::default().fg(Color::Indexed(136)), // dark gold
-
             feature_enabled: Style::default().fg(Color::Indexed(28)),
             feature_disabled: Style::default().fg(Color::Indexed(124)),
 
@@ -331,7 +325,6 @@ impl Theme {
         patch!(cancel_success);
         patch!(cancel_not_done);
         patch!(cancel_failure);
-        patch!(banner);
         patch!(feature_enabled);
         patch!(feature_disabled);
         patch!(text_error);
@@ -364,11 +357,11 @@ mod tests {
         }
     }
 
-    fn settings_with_banner_override(style: Style) -> ThemeSettings {
+    fn settings_with_app_background_override(style: Style) -> ThemeSettings {
         ThemeSettings {
             name: ThemeName::Dark,
             overrides: ThemeOverride {
-                banner: Some(style),
+                app_background: Some(style),
                 ..Default::default()
             },
         }
@@ -419,11 +412,6 @@ mod tests {
         assert_eq!(Theme::dark().row_selected.bg, Some(Color::Indexed(29)));
     }
 
-    #[test]
-    fn dark_banner_is_yellow() {
-        assert_eq!(Theme::dark().banner.fg, Some(Color::Yellow));
-    }
-
     // ── Theme::light() ─────────────────────────────────────────────────────
 
     #[test]
@@ -474,10 +462,10 @@ mod tests {
 
     #[test]
     fn override_named_color_changes_fg() {
-        let theme = Theme::from_settings(&settings_with_banner_override(
+        let theme = Theme::from_settings(&settings_with_app_background_override(
             Style::default().fg(Color::Red),
         ));
-        assert_eq!(theme.banner.fg, Some(Color::Red));
+        assert_eq!(theme.app_background.fg, Some(Color::Red));
     }
 
     #[test]
@@ -495,10 +483,10 @@ mod tests {
 
     #[test]
     fn override_rgb_color_changes_fg() {
-        let theme = Theme::from_settings(&settings_with_banner_override(
+        let theme = Theme::from_settings(&settings_with_app_background_override(
             Style::default().fg(Color::Rgb(180, 100, 0)),
         ));
-        assert_eq!(theme.banner.fg, Some(Color::Rgb(180, 100, 0)));
+        assert_eq!(theme.app_background.fg, Some(Color::Rgb(180, 100, 0)));
     }
 
     #[test]
@@ -506,12 +494,12 @@ mod tests {
         let settings = ThemeSettings {
             name: ThemeName::Dark,
             overrides: ThemeOverride {
-                banner: Some(Style::default().fg(Color::Cyan)),
+                app_background: Some(Style::default().fg(Color::Cyan)),
                 ..Default::default()
             },
         };
         let theme = Theme::from_settings(&settings);
-        assert_eq!(theme.banner.fg, Some(Color::Cyan));
+        assert_eq!(theme.app_background.fg, Some(Color::Cyan));
         // Unspecified field keeps dark-theme default
         assert_eq!(theme.status_running.fg, Some(Color::LightBlue));
     }
@@ -537,11 +525,11 @@ mod tests {
     #[test]
     fn named_color_is_case_insensitive() {
         for name in &["LIGHTBLUE", "LightBlue", "lightblue"] {
-            let theme = Theme::from_settings(&settings_with_banner_override(
+            let theme = Theme::from_settings(&settings_with_app_background_override(
                 Style::default().fg(Color::from_str(name).unwrap()),
             ));
             assert_eq!(
-                theme.banner.fg,
+                theme.app_background.fg,
                 Some(Color::LightBlue),
                 "case variant '{}' should resolve",
                 name
@@ -551,18 +539,18 @@ mod tests {
 
     #[test]
     fn named_color_grey_aliases_gray() {
-        let theme = Theme::from_settings(&settings_with_banner_override(
+        let theme = Theme::from_settings(&settings_with_app_background_override(
             Style::default().fg(Color::from_str("grey").unwrap()),
         ));
-        assert_eq!(theme.banner.fg, Some(Color::Gray));
+        assert_eq!(theme.app_background.fg, Some(Color::Gray));
     }
 
     #[test]
     fn named_color_darkgrey_aliases_darkgray() {
-        let theme = Theme::from_settings(&settings_with_banner_override(
+        let theme = Theme::from_settings(&settings_with_app_background_override(
             Style::default().fg(Color::from_str("darkgrey").unwrap()),
         ));
-        assert_eq!(theme.banner.fg, Some(Color::DarkGray));
+        assert_eq!(theme.app_background.fg, Some(Color::DarkGray));
     }
 
     #[test]
@@ -588,11 +576,11 @@ mod tests {
             ("white", Color::White),
         ];
         for (name, expected) in cases {
-            let theme = Theme::from_settings(&settings_with_banner_override(
+            let theme = Theme::from_settings(&settings_with_app_background_override(
                 Style::default().fg(Color::from_str(name).unwrap()),
             ));
             assert_eq!(
-                theme.banner.fg,
+                theme.app_background.fg,
                 Some(*expected),
                 "named color '{name}' did not resolve to {expected:?}",
             );
@@ -606,16 +594,15 @@ mod tests {
         let o = ThemeOverride::default();
         assert!(o.status_running.is_none());
         assert!(o.table_header.is_none());
-        assert!(o.banner.is_none());
         assert!(o.text_error.is_none());
     }
 
     #[test]
     fn color_spec_named_deserializes() {
         let o: ThemeOverride =
-            serde_json::from_str(r#"{"banner": {"fg": "Cyan"}}"#).unwrap();
+            serde_json::from_str(r#"{"app_background": {"fg": "Cyan"}}"#).unwrap();
         assert_eq!(
-            o.banner,
+            o.app_background,
             Some(Style::default().fg(Color::Cyan)),
             "expected Named(\"Cyan\")"
         );
@@ -635,9 +622,9 @@ mod tests {
     #[test]
     fn color_spec_rgb_deserializes() {
         let o: ThemeOverride =
-            serde_json::from_str(r##"{"banner": {"fg": "#1005FF"}}"##).unwrap();
+            serde_json::from_str(r##"{"app_background": {"fg": "#1005FF"}}"##).unwrap();
         assert_eq!(
-            o.banner,
+            o.app_background,
             Some(Style::default().fg(Color::Rgb(16, 5, 255))),
             "expected Rgb(16, 5, 255)"
         );
@@ -658,8 +645,8 @@ mod tests {
     #[test]
     fn theme_override_unspecified_fields_remain_none() {
         let o: ThemeOverride =
-            serde_json::from_str(r#"{"banner": {"fg": "Red"}}"#).unwrap();
-        assert!(o.banner.is_some());
+            serde_json::from_str(r#"{"app_background": {"fg": "Red"}}"#).unwrap();
+        assert!(o.app_background.is_some());
         assert!(o.status_running.is_none());
         assert!(o.table_header.is_none());
         assert!(o.row_selected.is_none());
diff --git a/ballista-cli/src/tui/ui/header/ballista-logo.svg b/ballista-cli/src/tui/ui/header/ballista-logo.svg
deleted file mode 100644
index 3f2ee3a3b3..0000000000
--- a/ballista-cli/src/tui/ui/header/ballista-logo.svg
+++ /dev/null
@@ -1,29 +0,0 @@
-<svg width="528" height="102" viewBox="0 0 528 102" fill="none" xmlns="http://www.w3.org/2000/svg">
-    <path fill-rule="evenodd" clip-rule="evenodd"
-          d="M46.1418 102C53.6175 102 60.3725 96.0798 65.2024 86.5499C77.1095 87.7913 86.7259 84.9975 90.7636 78.0233C94.7201 71.1894 92.5126 61.7364 85.8066 52.2678C92.3397 42.9047 94.4553 33.585 90.5426 26.8267C86.676 20.1481 77.6936 17.303 66.4829 18.164C61.6066 7.05696 54.3037 0 46.1418 0C37.9408 0 30.6072 7.1246 25.7309 18.3236C14.7597 17.5876 5.99204 20.4469 2.18467 27.0233C-1.70042 33.7339 0.358023 42.9699 6.78315 52.2663C0.184612 61.6696 -1.96566 71.0398 1.96364 77.8267C5.94278 84.6998 15.3399 87.5128 27.0079 86.4049C31.8427 96.0197 38.6282 102 46.1418 102ZM31.9533 85.7237C36.0686 93.6286 41.5905 98.4653 47.6608 98.4653C53.5703 98.4653 59.1785 93.8816 63.4906 86.3457C58.0302 85.6137 52.1363 84.0636 46.1099 81.7087C41.246 83.5752 36.4742 84.9158 31.9533 85.7237ZM39.434 78.7904C36.1433 79.8541 32.9389 80.6549 29.8815 81.1886C28.7033 78.2467 27.6938 75.0075 26.8835 71.5337C28.8935 72.9026 30.9957 74.2225 33.1825 75.4815C35.2609 76.6781 37.3487 77.7813 39.434 78.7904ZM46.1305 76.3064C49.7306 74.8008 53.3825 73.0003 57.0128 70.9101C62.2748 67.8805 67.0309 64.5075 71.1553 60.9761C71.5193 57.7595 71.7123 54.422 71.7123 51C71.7123 46.6211 71.3963 42.3807 70.8104 38.3546C67.3401 35.5364 63.4705 32.8621 59.2596 30.4377C54.8785 27.9153 50.4474 25.8471 46.1023 24.23C41.9249 25.817 37.6732 27.8198 33.4677 30.2412C30.8223 31.7643 28.3115 33.386 25.9499 35.0801C25.0901 40.0565 24.622 45.4156 24.622 51C24.622 55.6455 24.946 60.1351 25.5498 64.3782C28.6674 66.7448 32.0696 69.008 35.7144 71.1066C39.1882 73.1066 42.6816 74.8414 46.1305 76.3064ZM52.8064 78.8279C55.0535 77.7559 57.3046 76.5747 59.5447 75.285C63.2848 73.1316 66.7774 70.8003 69.9846 68.3505C68.9627 73.2999 67.5239 77.8436 65.7671 81.8193C61.7085 81.3247 57.3359 80.3238 52.8064 78.8279ZM71.3392 67.2953C70.4002 72.6417 69.0118 77.5916 67.2602 81.9797C77.2675 82.9076 85.1652 80.6223 88.4538 74.9421C91.6202 69.4727 90.042 61.6699 84.8509 53.5953C81.3697 58.2866 76.7982 62.9589 71.3392 67.2953ZM82.7622 48.3353C88.1554 40.4276 89.899 32.7827 86.7134 27.2802C83.6301 21.9546 76.3762 19.4079 67.127 19.6894C69.2744 24.979 70.8902 31.1018 71.8208 37.7625C76.0362 41.1368 79.715 44.7105 82.7622 48.3353ZM65.5798 19.7617C67.7903 24.6511 69.5152 30.426 70.565 36.7755C67.2599 34.2247 63.646 31.7991 59.7658 29.565C55.7337 27.2435 51.6657 25.2737 47.6504 23.6597C53.9747 21.402 60.0926 20.1058 65.5798 19.7617ZM64.8961 18.3091C59.0529 18.9272 52.661 20.5152 46.1079 23.056C40.7888 21.0313 35.5829 19.6362 30.6982 18.8801C34.9089 9.44639 40.9498 3.53465 47.6608 3.53465C54.2355 3.53465 60.4372 9.20845 64.8961 18.3091ZM44.5588 23.6732C39.5359 21.9186 34.651 20.7727 30.1179 20.2305C28.4813 24.1872 27.1545 28.7045 26.2148 33.623C28.3598 32.1444 30.6118 30.7213 32.9615 29.3685C36.8203 27.1468 40.7119 25.2472 44.5588 23.6732ZM20.4377 37.9438C21.3535 31.29 22.9526 25.1687 25.0824 19.8731C16.0807 19.7008 9.03942 22.2509 6.01392 27.4768C2.85747 32.9288 4.54031 40.4843 9.81777 48.3146C12.7807 44.7647 16.3503 41.2614 20.4377 37.9438ZM7.74162 53.6095C11.1385 58.225 15.5931 62.8262 20.9123 67.1112C21.8409 72.4662 23.2198 77.4267 24.9628 81.8273C15.1948 82.633 7.50664 80.3301 4.27353 74.7456C1.1346 69.3238 2.65811 61.609 7.74162 53.6095ZM10.6457 49.5043C13.2739 46.1323 16.4955 42.7704 20.2291 39.5669C19.7903 43.2423 19.5586 47.0692 19.5586 51C19.5586 53.9815 19.6919 56.9033 19.9478 59.7445C16.2428 56.3978 13.1076 52.9337 10.6457 49.5043ZM81.934 49.5123C79.3917 53.017 76.1488 56.5549 72.3158 59.9636C72.5847 57.0535 72.725 54.0582 72.725 51C72.725 46.9972 72.4847 43.1021 72.0301 39.3649C75.8927 42.6281 79.223 46.0635 81.934 49.5123Z"
-          fill="url(#paint0_linear_805_84)"/>
-    <path d="M114 30.0994H139.765C145.173 30.0994 149.817 31.658 153.698 34.7753C157.642 37.829 159.614 42.4413 159.614 48.6123C159.614 51.8568 158.946 54.7514 157.61 57.2961C156.274 59.8409 154.747 61.6222 153.03 62.64C155.511 63.3398 157.801 65.0893 159.9 67.8885C162 70.6241 163.05 74.2504 163.05 78.7672C163.05 84.4292 161.046 89.0097 157.038 92.5087C153.093 96.0077 148.227 97.7572 142.437 97.7572H114V30.0994ZM126.406 42.1232V58.823H138.716C141.26 58.7593 143.233 57.9959 144.632 56.5327C146.032 55.0695 146.732 53.0973 146.732 50.6162C146.732 48.1351 145.936 46.0993 144.346 44.5089C142.755 42.9184 140.624 42.1232 137.952 42.1232H126.406ZM126.31 68.7474V85.7334H141.388C144.187 85.7334 146.35 84.97 147.877 83.4432C149.467 81.9163 150.262 79.8806 150.262 77.3358C150.262 74.7911 149.372 72.7235 147.59 71.1331C145.873 69.5426 143.678 68.7474 141.006 68.7474H126.31Z"
-          fill="#fff"/>
-    <path d="M214.114 82.8706H186.631L181.001 97.7572H168.405L194.361 30.0994H207.243L233.677 97.7572H219.935L214.114 82.8706ZM210.297 72.7553L200.277 47.0854L190.544 72.7553H210.297Z"
-          fill="#fff"/>
-    <path d="M253.808 85.3517H281.673V97.7572H240.926V30.0994H253.808V85.3517Z" fill="#fff"/>
-    <path d="M303.665 85.3517H331.53V97.7572H290.782V30.0994H303.665V85.3517Z" fill="#fff"/>
-    <path d="M353.522 97.7572H340.639V30.0994H353.522V97.7572Z" fill="#ffffff"/>
-    <path d="M389.092 28C395.899 28 401.784 29.495 406.746 32.4851L401.116 43.1729C396.535 41.2007 392.082 40.2147 387.756 40.2147C384.511 40.2147 381.935 40.8827 380.026 42.2186C378.181 43.491 377.259 45.3359 377.259 47.7534C377.259 49.9164 378.34 51.7614 380.503 53.2882C382.73 54.7514 386.07 56.3419 390.523 58.0595C394.977 59.7136 398.221 61.1132 400.257 62.2583C407.191 66.139 410.69 71.6738 410.754 78.8627C410.754 85.3517 408.368 90.473 403.597 94.2264C398.825 97.9163 392.909 99.7612 385.847 99.7612C381.903 99.7612 377.8 99.0296 373.537 97.5664C369.275 96.1032 365.967 94.4809 363.613 92.6996L370.484 82.3935C377.036 86.2105 383.08 87.8964 388.615 87.4511C391.223 87.2603 393.418 86.4014 395.199 84.8746C396.98 83.3477 397.744 81.3756 397.489 78.9581C397.299 76.9223 395.931 75.2364 393.386 73.9005C390.841 72.5645 387.438 71.1013 383.175 69.5108C378.977 67.8567 375.605 66.0436 373.06 64.0715C367.016 59.6182 363.994 54.3379 363.994 48.2305C363.994 44.6679 364.726 41.5506 366.189 38.8787C367.653 36.1431 369.593 34.0119 372.01 32.4851C374.491 30.9582 377.163 29.8449 380.026 29.1451C382.889 28.3817 385.911 28 389.092 28Z"
-          fill="#fff"/>
-    <path d="M467.472 42.5049H447.241V97.7572H434.359V42.5049H414.51V30.0994H467.472V42.5049Z" fill="#fff"/>
-    <path d="M508.224 82.8706H480.741L475.11 97.7572H462.514L488.47 30.0994H501.353L527.786 97.7572H514.045L508.224 82.8706ZM504.407 72.7553L494.387 47.0854L484.653 72.7553H504.407Z"
-          fill="#fff"/>
-    <path d="M123.499 13.14H117.419L115.919 17H114.079L119.579 2.82H121.419L126.979 17H124.999L123.499 13.14ZM122.879 11.6L120.439 5.3L117.999 11.6H122.879ZM140.732 2.82H145.792C147.072 2.82 148.152 3.2 149.032 3.96C149.912 4.70667 150.352 5.76 150.352 7.12C150.352 8.48 149.912 9.54 149.032 10.3C148.152 11.06 147.072 11.44 145.792 11.44L142.592 11.42V17H140.732V2.82ZM142.592 9.64H145.812C146.558 9.65333 147.178 9.43333 147.672 8.98C148.165 8.52667 148.412 7.90667 148.412 7.12C148.412 6.33333 148.158 5.72 147.652 5.28C147.158 4.82667 146.538 4.6 145.792 4.6H142.592V9.64ZM170.956 13.14H164.876L163.376 17H161.536L167.036 2.82H168.876L174.436 17H172.456L170.956 13.14ZM170.336 11.6L167.896 5.3L165.456 11.6H170.336ZM193.532 2.36C194.758 2.36 195.865 2.66 196.852 3.26C197.838 3.86 198.652 4.7 199.292 5.78L197.592 6.62C196.512 4.91333 195.158 4.06 193.532 4.06C192.132 4.06 190.965 4.62667 190.032 5.76C189.112 6.88 188.652 8.26 188.652 9.9C188.652 11.5667 189.112 12.96 190.032 14.08C190.952 15.1867 192.118 15.74 193.532 15.74C195.212 15.74 196.558 14.9267 197.572 13.3L199.252 14.12C198.625 15.1867 197.812 16.0067 196.812 16.58C195.825 17.1533 194.732 17.44 193.532 17.44C191.558 17.44 189.932 16.7333 188.652 15.32C187.372 13.8933 186.732 12.0867 186.732 9.9C186.732 7.74 187.378 5.94667 188.672 4.52C189.965 3.08 191.585 2.36 193.532 2.36ZM223.933 17H222.073V10.8H215.213V17H213.353V2.82H215.213V9.02H222.073V2.82H223.933V17ZM246.513 4.6H240.553V9.02H245.033V10.8H240.553V15.22H246.513V17H238.693V2.82H246.513V4.6ZM277.829 2.82H282.229C284.403 2.82 286.196 3.46667 287.609 4.76C289.036 6.05333 289.749 7.76667 289.749 9.9C289.749 12.0467 289.029 13.7667 287.589 15.06C286.163 16.3533 284.363 17 282.189 17H277.829V2.82ZM279.689 4.6V15.22H282.129C282.849 15.22 283.536 15.12 284.189 14.92C284.843 14.72 285.443 14.4133 285.989 14C286.549 13.5867 286.989 13.0267 287.309 12.32C287.643 11.6133 287.809 10.8 287.809 9.88C287.809 8.96 287.643 8.15333 287.309 7.46C286.989 6.75333 286.549 6.2 285.989 5.8C285.443 5.38667 284.843 5.08667 284.189 4.9C283.536 4.7 282.856 4.6 282.149 4.6H279.689ZM311.452 13.14H305.372L303.872 17H302.032L307.532 2.82H309.372L314.932 17H312.952L311.452 13.14ZM310.832 11.6L308.392 5.3L305.952 11.6H310.832ZM336.702 4.6H332.222V17H330.362V4.6H326.022V2.82H336.702V4.6ZM357.171 13.14H351.091L349.591 17H347.751L353.251 2.82H355.091L360.651 17H358.671L357.171 13.14ZM356.551 11.6L354.111 5.3L351.671 11.6H356.551ZM382.223 4.6H376.263V9.02H380.743V10.8H376.263V17.02H374.403V2.82H382.223V4.6ZM407.786 2.82V11.94C407.773 13.58 407.226 14.9067 406.146 15.92C405.066 16.9333 403.693 17.44 402.026 17.44C400.36 17.44 398.986 16.94 397.906 15.94C396.84 14.9267 396.3 13.5933 396.286 11.94V2.82H398.146V11.94C398.16 13.0467 398.526 13.9467 399.246 14.64C399.98 15.32 400.906 15.66 402.026 15.66C403.146 15.66 404.073 15.32 404.806 14.64C405.54 13.9467 405.913 13.0467 405.926 11.94V2.82H407.786ZM427.084 2.36C428.364 2.36 429.471 2.64 430.404 3.2L429.584 4.74C428.704 4.32667 427.811 4.12 426.904 4.12C426.051 4.12 425.377 4.32 424.884 4.72C424.404 5.10667 424.164 5.63333 424.164 6.3C424.164 6.7 424.311 7.07333 424.604 7.42C424.911 7.75333 425.231 8.01333 425.564 8.2C425.911 8.37333 426.417 8.61333 427.084 8.92C427.764 9.21333 428.257 9.44667 428.564 9.62C429.444 10.0867 430.104 10.6 430.544 11.16C430.997 11.72 431.231 12.4467 431.244 13.34C431.257 14.6333 430.791 15.64 429.844 16.36C428.897 17.08 427.744 17.44 426.384 17.44C425.611 17.44 424.817 17.3067 424.004 17.04C423.191 16.7733 422.571 16.48 422.144 16.16L423.124 14.68C424.364 15.4 425.577 15.7267 426.764 15.66C427.471 15.6333 428.084 15.4267 428.604 15.04C429.124 14.64 429.371 14.08 429.344 13.36C429.304 12.84 429.071 12.4133 428.644 12.08C428.217 11.7333 427.571 11.3667 426.704 10.98C425.837 10.58 425.184 10.2333 424.744 9.94C423.971 9.44667 423.357 8.92 422.904 8.36C422.464 7.8 422.244 7.13333 422.244 6.36C422.244 5.04 422.711 4.04667 423.644 3.38C424.591 2.7 425.737 2.36 427.084 2.36ZM447.459 17H445.599V2.82H447.459V17ZM469.248 2.36C469.888 2.36 470.541 2.44667 471.208 2.62C471.875 2.79333 472.541 3.07333 473.208 3.46C473.888 3.84667 474.488 4.32667 475.008 4.9C475.528 5.46 475.948 6.17333 476.268 7.04C476.601 7.89333 476.768 8.84 476.768 9.88C476.755 12.0667 476.041 13.8667 474.628 15.28C473.215 16.68 471.421 17.4 469.248 17.44C468.608 17.44 467.955 17.3533 467.288 17.18C466.621 17.0067 465.948 16.72 465.268 16.32C464.601 15.92 464.008 15.44 463.488 14.88C462.968 14.3067 462.541 13.5867 462.208 12.72C461.888 11.8533 461.728 10.9067 461.728 9.88C461.715 7.72 462.428 5.92667 463.868 4.5C465.308 3.07333 467.101 2.36 469.248 2.36ZM469.248 4.06C468.288 4.06 467.428 4.24667 466.668 4.62C465.921 4.99333 465.335 5.48 464.908 6.08C464.495 6.66667 464.181 7.28667 463.968 7.94C463.755 8.59333 463.648 9.24 463.648 9.88C463.648 11.56 464.168 12.96 465.208 14.08C466.261 15.1867 467.608 15.74 469.248 15.74C470.141 15.74 470.955 15.5733 471.688 15.24C472.421 14.8933 473.008 14.44 473.448 13.88C473.888 13.3067 474.228 12.68 474.468 12C474.708 11.3067 474.828 10.6 474.828 9.88C474.815 8.17333 474.295 6.78 473.268 5.7C472.241 4.62 470.901 4.07333 469.248 4.06ZM501.965 17H500.065L492.725 6.04V17H490.985V2.82H492.765L500.225 13.82V2.82H501.965V17Z"
-          fill="#fff"/>
-    <path d="M509.863 3.97065H508.28V8.29376H507.272V3.97065H505.719V3H509.863V3.97065Z" fill="#fff"/>
-    <path d="M515.719 8.29376H514.748V4.8965L513.359 7.51724H512.717L511.328 4.8965L511.321 8.29376H510.38V3H511.328L513.076 6.233L514.748 3H515.711L515.719 8.29376Z"
-          fill="#fff"/>
-    <defs>
-        <linearGradient id="paint0_linear_805_84" x1="0" y1="0" x2="111" y2="27.0674" gradientUnits="userSpaceOnUse">
-            <stop stop-color="#5800FF"/>
-            <stop offset="0.489583" stop-color="#E900FF"/>
-            <stop offset="1" stop-color="#FFC600"/>
-        </linearGradient>
-    </defs>
-</svg>
diff --git a/ballista-cli/src/tui/ui/header/mod.rs b/ballista-cli/src/tui/ui/header/mod.rs
index 0d6a81e812..27d563f426 100644
--- a/ballista-cli/src/tui/ui/header/mod.rs
+++ b/ballista-cli/src/tui/ui/header/mod.rs
@@ -17,7 +17,7 @@
 
 use crate::tui::app::App;
 use crate::tui::ui::header::scheduler_state::render_scheduler_state;
-use ratatui::style::{Style, Stylize};
+use ratatui::style::Stylize;
 use ratatui::widgets::BorderType;
 use ratatui::{
     Frame,
@@ -29,52 +29,14 @@ use ratatui::{
 pub mod scheduler_state;
 
 const MENU_ITEMS: [&str; 3] = ["Jobs", "Executors", "Metrics"];
-const PERCENTAGE: u16 = 100 / MENU_ITEMS.len() as u16;
-const MENU_CONSTRAINTS: [Constraint; MENU_ITEMS.len()] =
-    [Constraint::Percentage(PERCENTAGE); MENU_ITEMS.len()];
+const MENU_CONSTRAINTS: [Constraint; MENU_ITEMS.len()] = [
+    Constraint::Percentage(33),
+    Constraint::Percentage(34),
+    Constraint::Percentage(33),
+];
 
 pub(super) fn render_header(f: &mut Frame, area: Rect, app: &App) {
-    let banner_percentage = match area.width {
-        0..200 => 30,
-        220..230 => 40,
-        _ => 45,
-    };
-    let chunks = Layout::default()
-        .direction(Direction::Horizontal)
-        .constraints([
-            Constraint::Percentage(banner_percentage), // banner
-            Constraint::Percentage(100 - banner_percentage), // scheduler info and menu
-        ])
-        .split(area);
-
-    render_banner(f, chunks[0], app.theme.banner);
-    render_navbar(f, chunks[1], app);
-}
-
-#[cfg(not(feature = "web"))]
-fn render_banner(f: &mut Frame, area: Rect, banner_style: Style) {
-    use tui_big_text::{BigText, PixelSize};
-
-    let banner_size = match area.width {
-        0..70 => PixelSize::Octant,
-        70..80 => PixelSize::QuarterHeight,
-        _ => PixelSize::ThirdHeight,
-    };
-
-    let big_text = BigText::builder()
-        .pixel_size(banner_size)
-        .style(banner_style)
-        .lines(vec![" DataFusion".into(), " Ballista".into()])
-        .build();
-    f.render_widget(big_text, area);
-}
-
-// Web: logo is displayed as a DOM <img> element overlaid on the Ratzilla canvas.
-// Halfblock characters are not reliably present in Ratzilla's WebGL2 glyph atlas,
-// so we leave this area blank and rely on setup_logo_dom() in wasm.rs.
-#[cfg(feature = "web")]
-fn render_banner(f: &mut Frame, area: Rect, _banner_style: Style) {
-    f.render_widget(Block::default(), area);
+    render_navbar(f, area, app);
 }
 
 fn render_navbar(f: &mut Frame, area: Rect, app: &App) {
diff --git a/ballista-cli/src/tui/ui/header/scheduler_state.rs b/ballista-cli/src/tui/ui/header/scheduler_state.rs
index 9657dac9b9..f21a8bea6a 100644
--- a/ballista-cli/src/tui/ui/header/scheduler_state.rs
+++ b/ballista-cli/src/tui/ui/header/scheduler_state.rs
@@ -97,7 +97,7 @@ fn render_scheduler_state_up(
 
     let scheduler_url_block = Block::default()
         .borders(Borders::ALL)
-        .title(" Scheduler URL ");
+        .title(" Ballista Scheduler URL ");
     let scheduler_url_paragraph = Paragraph::new(scheduler_url)
         .block(scheduler_url_block)
         .left_aligned();
diff --git a/ballista-cli/src/tui/ui/main/executors/executors_table.rs b/ballista-cli/src/tui/ui/main/executors/executors_table.rs
index 09bd302597..7441c4fadb 100644
--- a/ballista-cli/src/tui/ui/main/executors/executors_table.rs
+++ b/ballista-cli/src/tui/ui/main/executors/executors_table.rs
@@ -62,7 +62,7 @@ fn render_executors_table(frame: &mut Frame, area: Rect, app: &App) {
     let host_suffix = sort_suffix!(SortColumn::Host, sort_column, sort_order);
     let id_suffix = sort_suffix!(SortColumn::Id, sort_column, sort_order);
     let cpu_cores_suffix = sort_suffix!(SortColumn::CpuCores, sort_column, sort_order);
-    let task_slots_suffix = sort_suffix!(SortColumn::TaskSlots, sort_column, sort_order);
+    let vcores_suffix = sort_suffix!(SortColumn::Vcores, sort_column, sort_order);
     let proc_physical_memory_suffix =
         sort_suffix!(SortColumn::ProcPhysicalMemoryUsage, sort_column, sort_order);
     let peak_physical_memory_suffix =
@@ -73,7 +73,7 @@ fn render_executors_table(frame: &mut Frame, area: Rect, app: &App) {
         Cell::from(Text::from(format!("Host{host_suffix}")).centered()),
         Cell::from(Text::from(format!("Id{id_suffix}")).centered()),
         Cell::from(Text::from(format!("CPU cores{cpu_cores_suffix}")).right_aligned()),
-        Cell::from(Text::from(format!("Task Slots{task_slots_suffix}")).right_aligned()),
+        Cell::from(Text::from(format!("vCores{vcores_suffix}")).right_aligned()),
         Cell::from(
             Text::from(format!("Physical Memory{proc_physical_memory_suffix}"))
                 .right_aligned(),
@@ -107,8 +107,8 @@ fn render_executors_table(frame: &mut Frame, area: Rect, app: &App) {
                 Text::from(app.format_count(executor.cpu_cores() as usize))
                     .right_aligned(),
             );
-            let task_slots_cell = Cell::from(
-                Text::from(app.format_count(executor.specification.task_slots as usize))
+            let vcores_cell = Cell::from(
+                Text::from(app.format_count(executor.specification.vcores as usize))
                     .right_aligned(),
             );
             let proc_physical_memory_cell = Cell::from(
@@ -129,7 +129,7 @@ fn render_executors_table(frame: &mut Frame, area: Rect, app: &App) {
                 host_cell,
                 id_cell,
                 cpu_cores_cell,
-                task_slots_cell,
+                vcores_cell,
                 proc_physical_memory_cell,
                 peak_physical_memory_cell,
                 last_seen_cell,
@@ -143,7 +143,7 @@ fn render_executors_table(frame: &mut Frame, area: Rect, app: &App) {
             Constraint::Percentage(10), // Host
             Constraint::Percentage(20), // Id
             Constraint::Percentage(8),  // CPU cores
-            Constraint::Percentage(8),  // Task slots
+            Constraint::Percentage(8),  // vCores
             Constraint::Percentage(18), // Proc physical memory
             Constraint::Percentage(18), // Peak physical memory
             Constraint::Percentage(18), // Last seen
diff --git a/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs b/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs
index dfa3401604..8c89a699bd 100644
--- a/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs
+++ b/ballista-cli/src/tui/ui/main/jobs/stage_tasks_popup.rs
@@ -16,7 +16,7 @@
 // under the License.
 
 use crate::tui::app::App;
-use crate::tui::domain::jobs::stages::StageTaskResponse;
+use crate::tui::domain::jobs::stages::{StageTaskResponse, StageTaskStatus};
 use crate::tui::ui::components::clear_area::clear_area;
 use crate::tui::ui::vertical_scrollbar;
 use ratatui::Frame;
@@ -77,7 +77,7 @@ pub(crate) fn render_stage_tasks_popup(f: &mut Frame, app: &App) {
         Block::default()
             .title(format!(
                 " Tasks for stage '{}' of job '{}' ",
-                stage.id, &popup.job_id
+                stage.id, popup.job_id
             ))
             .borders(Borders::ALL)
             .border_style(app.theme.popup_border_alt)
@@ -101,21 +101,19 @@ fn build_stage_task_row(i: usize, task: &StageTaskResponse, app: &App) -> Row<'s
         app.theme.row_odd
     };
 
-    let status_style = match task.status.as_str() {
-        "Running" => app.theme.status_running,
-        "Queued" => app.theme.status_queued,
-        "Successful" | "Completed" => app.theme.status_completed,
-        "Failed" => app.theme.status_failed,
-        _ => app.theme.status_unknown,
+    let status_text = match &task.status {
+        StageTaskStatus::Running => Text::from("Running").style(app.theme.status_running),
+        StageTaskStatus::Successful => {
+            Text::from("Successful").style(app.theme.status_completed)
+        }
+        StageTaskStatus::Failed { reason } => {
+            Text::from(reason.clone()).style(app.theme.status_failed)
+        }
     };
 
     Row::new(vec![
         Cell::from(Text::from(task.id.to_string()).right_aligned()),
-        Cell::from(
-            Text::from(task.status.clone())
-                .style(status_style)
-                .centered(),
-        ),
+        Cell::from(status_text.centered()),
         Cell::from(Text::from(app.format_count(task.input_rows)).right_aligned()),
         Cell::from(Text::from(app.format_count(task.output_rows)).right_aligned()),
         Cell::from(Text::from(task.partition_id.to_string()).right_aligned()),
diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml
index e8d4eb9900..062b3435d2 100644
--- a/ballista/client/Cargo.toml
+++ b/ballista/client/Cargo.toml
@@ -19,7 +19,7 @@
 name = "ballista"
 description = "Ballista Distributed Compute"
 license = "Apache-2.0"
-version = "53.0.0"
+version = "54.0.0"
 homepage = "https://datafusion.apache.org/ballista/"
 repository = "https://github.com/apache/datafusion-ballista"
 readme = "README.md"
@@ -29,9 +29,9 @@ rust-version = { workspace = true }
 
 [dependencies]
 async-trait = { workspace = true }
-ballista-core = { path = "../core", version = "53.0.0" }
-ballista-executor = { path = "../executor", version = "53.0.0", optional = true, default-features = false, features = ["arrow-ipc-optimizations"] }
-ballista-scheduler = { path = "../scheduler", version = "53.0.0", optional = true, default-features = false }
+ballista-core = { path = "../core", version = "54.0.0" }
+ballista-executor = { path = "../executor", version = "54.0.0", optional = true, default-features = false, features = ["arrow-ipc-optimizations"] }
+ballista-scheduler = { path = "../scheduler", version = "54.0.0", optional = true, default-features = false }
 datafusion = { workspace = true }
 log = { workspace = true }
 
@@ -39,13 +39,14 @@ tokio = { workspace = true }
 url = { workspace = true }
 
 [dev-dependencies]
-ballista-executor = { path = "../executor", version = "53.0.0" }
-ballista-scheduler = { path = "../scheduler", version = "53.0.0" }
+ballista-executor = { path = "../executor", version = "54.0.0" }
+ballista-scheduler = { path = "../scheduler", version = "54.0.0" }
 ctor = { workspace = true }
 datafusion-proto = { workspace = true }
 env_logger = { workspace = true }
 rstest = { workspace = true }
 tempfile = { workspace = true }
+uuid = { workspace = true }
 tonic = { workspace = true }
 
 [features]
diff --git a/ballista/client/README.md b/ballista/client/README.md
index 5e94087341..10222a5555 100644
--- a/ballista/client/README.md
+++ b/ballista/client/README.md
@@ -171,27 +171,3 @@ The output should look similar to the following table.
 ```
 
 More [examples](../../examples/examples/) can be found in the datafusion-ballista repository.
-
-## Performance
-
-We run some simple benchmarks comparing Ballista with Apache Spark to track progress with performance optimizations.
-
-These are benchmarks derived from TPC-H and not official TPC-H benchmarks. These results are from running individual queries at scale factor 100 (100 GB) on a single node with a single executor and 8 concurrent tasks.
-
-### Overall Speedup
-
-The overall speedup is 2.9x
-
-![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_allqueries.png?raw=true)
-
-### Per Query Comparison
-
-![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_queries_compare.png?raw=true)
-
-### Relative Speedup
-
-![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_queries_speedup_rel.png?raw=true)
-
-### Absolute Speedup
-
-![benchmarks](https://github.com/apache/datafusion-ballista/blob/main/docs/source/_static/images/tpch_queries_speedup_abs.png?raw=true)
diff --git a/ballista/client/src/extension.rs b/ballista/client/src/extension.rs
index 54b2123442..bf86265a81 100644
--- a/ballista/client/src/extension.rs
+++ b/ballista/client/src/extension.rs
@@ -169,7 +169,7 @@ impl Extension {
             "hostname should be provided".to_string(),
         ))?;
         let port = url.port().unwrap_or(DEFAULT_SCHEDULER_PORT);
-        let scheduler_url = format!("http://{}:{}", &host, port);
+        let scheduler_url = format!("http://{}:{}", host, port);
 
         Ok(scheduler_url)
     }
@@ -208,13 +208,13 @@ impl Extension {
             }
         };
 
-        let concurrent_tasks = config.ballista_standalone_parallelism();
+        let vcores = config.ballista_standalone_parallelism();
 
         match session_state {
             None => {
                 ballista_executor::new_standalone_executor(
                     scheduler,
-                    concurrent_tasks,
+                    vcores,
                     BallistaCodec::default(),
                 )
                 .await
@@ -223,7 +223,7 @@ impl Extension {
             Some(session_state) => {
                 ballista_executor::new_standalone_executor_from_state(
                     scheduler,
-                    concurrent_tasks,
+                    vcores,
                     session_state,
                 )
                 .await
diff --git a/ballista/client/tests/context_basic.rs b/ballista/client/tests/context_basic.rs
index 5e9c29bb38..395f570313 100644
--- a/ballista/client/tests/context_basic.rs
+++ b/ballista/client/tests/context_basic.rs
@@ -449,7 +449,7 @@ mod basic {
             "+-------------------------------------------------------------------+",
             "| approx_percentile_cont_with_weight(test.id,Int64(2),Float64(0.5)) |",
             "+-------------------------------------------------------------------+",
-            "| 3                                                                 |",
+            "| 3.5                                                               |",
             "+-------------------------------------------------------------------+",
         ];
         assert_result_eq(expected, &res);
diff --git a/ballista/client/tests/context_checks.rs b/ballista/client/tests/context_checks.rs
index 2638942886..dd67cc1032 100644
--- a/ballista/client/tests/context_checks.rs
+++ b/ballista/client/tests/context_checks.rs
@@ -520,17 +520,14 @@ mod supported {
         Ok(())
     }
 
-    // As mentioned in https://github.com/apache/datafusion-ballista/issues/1055
-    // "Left/full outer join incorrect for CollectLeft / broadcast"
-    //
-    // In order to make correct results (decreasing performance) CollectLeft
-    // has been disabled until fixed
-
+    // Ballista raises these DataFusion thresholds above zero so the adaptive
+    // join resolver can collect a small build side into a broadcast
+    // (CollectLeft) hash join instead of repartitioning it.
     #[rstest]
     #[case::standalone(standalone_context())]
     #[case::remote(remote_context())]
     #[tokio::test]
-    async fn should_disable_collect_left(
+    async fn should_set_collect_left_thresholds(
         #[future(awt)]
         #[case]
         ctx: SessionContext,
@@ -542,12 +539,12 @@ mod supported {
             .await?;
 
         let expected = [
-            "+----------------------------------------------------------------+-------+",
-            "| name                                                           | value |",
-            "+----------------------------------------------------------------+-------+",
-            "| datafusion.optimizer.hash_join_single_partition_threshold      | 0     |",
-            "| datafusion.optimizer.hash_join_single_partition_threshold_rows | 0     |",
-            "+----------------------------------------------------------------+-------+",
+            "+----------------------------------------------------------------+----------+",
+            "| name                                                           | value    |",
+            "+----------------------------------------------------------------+----------+",
+            "| datafusion.optimizer.hash_join_single_partition_threshold      | 10485760 |",
+            "| datafusion.optimizer.hash_join_single_partition_threshold_rows | 1000000  |",
+            "+----------------------------------------------------------------+----------+",
         ];
 
         assert_batches_eq!(expected, &result);
@@ -1115,7 +1112,7 @@ mod supported {
             "|                  | ShuffleWriterExec: partitioning: None                                                                                                                                            |",
             "|                  |   ProjectionExec: expr=[count(Int64(1))@1 as count(*), id@0 as id]                                                                                                               |",
             "|                  |     AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[count(Int64(1))]                                                                                               |",
-            "|                  |       UnresolvedShuffleExec: partitioning: Hash([id@0], 16)                                                                                                                      |",
+            "|                  |       UnresolvedShuffleExec: stage=1, partitioning: Hash([id@0], 16)                                                                                                             |",
             "|                  |                                                                                                                                                                                  |",
             "|                  |                                                                                                                                                                                  |",
             "+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+",
@@ -1198,7 +1195,7 @@ mod supported {
             "|                   | ShuffleWriterExec: partitioning: None, metrics=[output_rows=..., input_rows=..., repart_time=..., write_time=...]                                                                                                                                                                                                                                                                                 |",
             "|                   |   ProjectionExec: expr=[count(Int64(1))@1 as count(*), id@0 as id], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., expr_0_eval_time=..., expr_1_eval_time=...]                                                                                                                                                                                              |",
             "|                   |     AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[count(Int64(1))], metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., spill_count=..., spilled_bytes=..., spilled_rows=..., peak_mem_used=..., aggregate_arguments_time=..., aggregation_time=..., emitting_time=..., time_calculating_group_ids=...]                                          |",
-            "|                   |       ShuffleReaderExec: upstream_stage: 1, partitioning: Hash([id@0], 16), metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=...]                                                                                                                                                                                                                                  |",
+            "|                   |       ShuffleReaderExec: upstream_stage: 1, partitioning: Hash([id@0], 16), metrics=[output_rows=..., elapsed_compute=..., output_bytes=..., output_batches=..., decoded_bytes=..., fetch_requests=..., fetch_retries=..., local_partitions=..., remote_partitions=..., fetch_time=..., local_read_time=..., permit_wait_time=...]                                                                |",
             "+-------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+",
         ];
 
diff --git a/ballista/client/tests/physical_plan_submission.rs b/ballista/client/tests/physical_plan_submission.rs
new file mode 100644
index 0000000000..140ecdf222
--- /dev/null
+++ b/ballista/client/tests/physical_plan_submission.rs
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! End-to-end test for submitting an already-built physical plan directly to a
+//! Ballista scheduler, bypassing logical plan creation on the scheduler side.
+
+mod common;
+
+#[cfg(test)]
+#[cfg(feature = "standalone")]
+mod physical_plan_submission_tests {
+    use crate::common::setup_test_cluster;
+    use ballista_core::config::BallistaConfig;
+    use ballista_core::execution_plans::execute_physical_plan;
+    use ballista_core::extension::SessionConfigExt;
+    use datafusion::arrow::array::Int32Array;
+    use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::arrow::record_batch::RecordBatch;
+    use datafusion::common::test_util::batches_to_sort_string;
+    use datafusion::datasource::MemTable;
+    use datafusion::physical_expr::expressions::Column;
+    use datafusion::physical_plan::repartition::RepartitionExec;
+    use datafusion::physical_plan::{self, ExecutionPlan, Partitioning};
+    use datafusion::prelude::{SessionConfig, SessionContext};
+    use datafusion_proto::protobuf::PhysicalPlanNode;
+    use std::sync::Arc;
+
+    /// Builds a two-partition `MemTable`-backed scan wrapped in a hash
+    /// `RepartitionExec`, i.e. a physical plan with a shuffle boundary that
+    /// `plan_query_stages` will split into two stages.
+    async fn build_physical_plan() -> (Arc<dyn ExecutionPlan>, SessionContext) {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("value", DataType::Int32, false),
+        ]));
+
+        let batch1 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(Int32Array::from(vec![10, 20, 30])),
+            ],
+        )
+        .unwrap();
+        let batch2 = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![4, 5])),
+                Arc::new(Int32Array::from(vec![40, 50])),
+            ],
+        )
+        .unwrap();
+
+        let table = MemTable::try_new(schema, vec![vec![batch1], vec![batch2]]).unwrap();
+
+        let ctx = SessionContext::new();
+        ctx.register_table("t", Arc::new(table)).unwrap();
+
+        let scan = ctx
+            .table("t")
+            .await
+            .unwrap()
+            .create_physical_plan()
+            .await
+            .unwrap();
+
+        let repartitioned: Arc<dyn ExecutionPlan> = Arc::new(
+            RepartitionExec::try_new(
+                scan,
+                Partitioning::Hash(vec![Arc::new(Column::new("id", 0))], 4),
+            )
+            .unwrap(),
+        );
+
+        (repartitioned, ctx)
+    }
+
+    /// Submits a hand-built physical plan (scan -> hash repartition) directly to
+    /// a standalone Ballista cluster via the new physical-plan submission path,
+    /// and checks that the rows returned match what executing the same plan
+    /// locally produces. The plan's `RepartitionExec(Hash)` boundary means the
+    /// scheduler must have split it into two shuffle stages for this to work.
+    #[tokio::test]
+    async fn should_execute_submitted_physical_plan_across_shuffle_stages() {
+        let (host, port) = setup_test_cluster().await;
+        let scheduler_url = format!("http://{host}:{port}");
+
+        let (plan, ctx) = build_physical_plan().await;
+
+        let expected = physical_plan::collect(plan.clone(), ctx.task_ctx())
+            .await
+            .unwrap();
+
+        let session_config = SessionConfig::new_with_ballista();
+        let codec = session_config.ballista_physical_extension_codec();
+        let session_id = uuid::Uuid::new_v4().to_string();
+
+        let stream = execute_physical_plan::<PhysicalPlanNode>(
+            scheduler_url,
+            &BallistaConfig::default(),
+            plan,
+            codec.as_ref(),
+            session_id,
+            session_config,
+        )
+        .await
+        .expect("physical plan job should be submitted and executed");
+
+        let actual = physical_plan::common::collect(stream)
+            .await
+            .expect("collecting results from the submitted physical plan");
+
+        assert_eq!(
+            batches_to_sort_string(&expected),
+            batches_to_sort_string(&actual)
+        );
+    }
+}
diff --git a/ballista/client/tests/sort_shuffle.rs b/ballista/client/tests/sort_shuffle.rs
index 48a27715fb..0a9cc4bf72 100644
--- a/ballista/client/tests/sort_shuffle.rs
+++ b/ballista/client/tests/sort_shuffle.rs
@@ -30,7 +30,12 @@ mod common;
 mod sort_shuffle_tests {
     use ballista::prelude::{SessionConfigExt, SessionContextExt};
     use ballista_core::config::{
-        BALLISTA_ADAPTIVE_PLANNER_ENABLED, BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ,
+        BALLISTA_ADAPTIVE_PLANNER_ENABLED,
+        BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE,
+        BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE,
+        BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ,
+        BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS,
+        BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT,
         BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT,
         BALLISTA_SHUFFLE_SORT_BASED_ENABLED,
     };
@@ -90,6 +95,28 @@ mod sort_shuffle_tests {
         SessionContext::standalone_with_state(state).await.unwrap()
     }
 
+    /// Remote-flight sort-shuffle context with a deliberately tiny governor
+    /// budget (64 KiB in-flight, 2 blocks/address) but a large connection window
+    /// (8 MiB). This forces many partition fetches to serialize through the
+    /// governor while multiplexing over the pooled connection — the scenario
+    /// that deadlocked before the governor existed. The query must still
+    /// complete and return correct results.
+    async fn create_tiny_budget_remote_context() -> SessionContext {
+        let config = SessionConfig::new_with_ballista()
+            .set_str(BALLISTA_SHUFFLE_SORT_BASED_ENABLED, "true")
+            .set_str(BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ, "true")
+            .set_str(BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT, "true")
+            .set_str(BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT, "65536")
+            .set_str(BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS, "2")
+            .set_str(BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE, "8388608")
+            .set_str(BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE, "8388608");
+        let state = SessionStateBuilder::new()
+            .with_config(config)
+            .with_default_features()
+            .build();
+        SessionContext::standalone_with_state(state).await.unwrap()
+    }
+
     /// Creates a standalone session context with hash-based shuffle.
     async fn create_hash_shuffle_context() -> SessionContext {
         let config = SessionConfig::new_with_ballista()
@@ -568,4 +595,30 @@ mod sort_shuffle_tests {
         assert_result_eq(expected, &results);
         Ok(())
     }
+
+    // ==================== Governor Budget ====================
+
+    #[tokio::test]
+    async fn test_shuffle_completes_under_tiny_governor_budget() -> Result<()> {
+        let ctx = create_tiny_budget_remote_context().await;
+        register_test_data(&ctx).await;
+
+        // GROUP BY forces a shuffle; the reducer fetches every upstream partition
+        // remotely under the tiny in-flight budget.
+        let df = ctx
+            .sql("SELECT bool_col, COUNT(*) as cnt FROM test GROUP BY bool_col ORDER BY bool_col")
+            .await?;
+        let results = df.collect().await?;
+
+        let expected = vec![
+            "+----------+-----+",
+            "| bool_col | cnt |",
+            "+----------+-----+",
+            "| false    | 4   |",
+            "| true     | 4   |",
+            "+----------+-----+",
+        ];
+        assert_result_eq(expected, &results);
+        Ok(())
+    }
 }
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 914c1eed3e..7359183906 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -19,7 +19,7 @@
 name = "ballista-core"
 description = "Ballista Distributed Compute"
 license = "Apache-2.0"
-version = "53.0.0"
+version = "54.0.0"
 homepage = "https://datafusion.apache.org/ballista/"
 repository = "https://github.com/apache/datafusion-ballista"
 readme = "README.md"
diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto
index 2154b1a17d..7bd4191e5a 100644
--- a/ballista/core/proto/ballista.proto
+++ b/ballista/core/proto/ballista.proto
@@ -132,7 +132,6 @@ message ExecutionGraph {
   uint64 output_partitions = 5;
   repeated PartitionLocation output_locations = 6;
   string scheduler_id = 7;
-  uint32 task_id_gen = 8;
   repeated StageAttempts failed_attempts = 9;
   string job_name = 10;
   uint64 start_time = 11;
@@ -289,10 +288,30 @@ message PartitionId {
   uint32 partition_id = 4;
 }
 
+// Within-stage task identity (paired with job_id + stage_id on the parent
+// MultiTaskDefinition). One task processes a slice of partitions;
+// `task_id` names the task within the stage (it's the task's append-order
+// slot in `RunningStage.task_infos`, so (job_id, stage_id, task_id) is
+// globally unique). `global_output_partition_ids` gives the concrete
+// global partition ids the task's restricted plan is covering. Writers
+// use these to name shuffle files with global identity (via
+// `create_shuffle_path`) so downstream reads have a stable, canonical
+// address regardless of how the scheduler split the work.
 message TaskId {
   uint32 task_id = 1;
   uint32 task_attempt_num = 2;
-  uint32 partition_id = 3;
+  // Global partition ids covered by this task, in slice order. Position i in
+  // the restricted plan corresponds to `global_output_partition_ids[i]` globally.
+  // `[packed=false]` keeps the single-partition case wire-compatible with the
+  // legacy `uint32 partition_id = 3` field: one element writes as a single
+  // varint at tag 3, exactly matching the old scalar encoding.
+  repeated uint32 global_output_partition_ids = 3 [packed=false];
+  // Vcores this task consumed from the executor's budget at bind time.
+  // The executor uses this to size the task's memory pool proportionally
+  // (`pool = total_memory * vcores_consumed / total_vcores`), so a
+  // multi-partition task claiming N vcores gets N/total of the executor's
+  // memory budget rather than a single per-vcore share.
+  uint32 vcores_consumed = 5;
 }
 
 message PartitionStats {
@@ -359,6 +378,10 @@ message OperatorMetric {
     NamedRatio ratio = 14;
     uint64 output_batches = 15;
   }
+  // Local partition index within the reporting task's plan (0..slice_len-1).
+  // The scheduler maps this to a global partition id via the task's
+  // `global_input_partition_ids` when folding metrics into stage metrics.
+  optional uint32 partition = 16;
 }
 
 // Used by scheduler
@@ -381,6 +404,11 @@ message ExecutorRegistration {
   uint32 grpc_port = 4;
   ExecutorSpecification specification = 5;
   ExecutorOperatingSystemSpecification os_info = 6;
+  // Wire-protocol version this executor was built against. The scheduler
+  // rejects heartbeats/registrations whose version does not match its own
+  // compiled-in BALLISTA_PROTOCOL_VERSION. Old executors that predate this
+  // field default to 0, which is never a valid scheduler version.
+  uint32 ballista_protocol_version = 7;
 }
 
 message ExecutorHeartbeat {
@@ -419,9 +447,14 @@ message ExecutorSpecification {
   repeated ExecutorResource resources = 1;
 }
 
+// Virtual cores this executor has been assigned at runtime — analogous to
+// YARN vcores (yarn.nodemanager.resource.cpu-vcores) or Spark's
+// spark.executor.cores. Not physical `nproc`: may over- or under-subscribe.
+// One task per executor drives this many DataFusion partitions in parallel
+// through one plan-Arc.
 message ExecutorResource {
   oneof resource {
-    uint32 task_slots = 1;
+    uint32 vcores = 1;
   }
 }
 
@@ -437,13 +470,16 @@ message ExecutorOperatingSystemSpecification {
   uint64 open_files_limit = 9;
 }
 
-message AvailableTaskSlots {
+// One instance per executor. Tracks free vcores for scheduler-side task
+// binding. See ExecutorResource for the vcore definition (YARN vcores or
+// Spark's spark.executor.cores).
+message AvailableVcores {
   string executor_id = 1;
-  uint32 slots = 2;
-    }
+  uint32 vcores = 2;
+}
 
-message ExecutorTaskSlots {
-  repeated AvailableTaskSlots task_slots = 1;
+message ExecutorVcores {
+  repeated AvailableVcores vcores = 1;
 }
 
 message ExecutorData {
@@ -518,11 +554,14 @@ message ShuffleWritePartition {
 }
 
 message TaskStatus {
+  // Task's append-order slot in `RunningStage.task_infos`; (job_id,
+  // stage_id, task_id) is globally unique. Under the multi-partition-task
+  // model one task processes a partition slice, so this names the task,
+  // not a single partition.
   uint32 task_id = 1;
   string job_id = 2;
   uint32 stage_id = 3;
   uint32 stage_attempt_num = 4;
-  uint32 partition_id = 5;
   uint64 launch_time = 6;
   uint64 start_exec_time = 7;
   uint64 end_exec_time = 8;
@@ -536,22 +575,35 @@ message TaskStatus {
 
 message PollWorkParams {
   ExecutorRegistration metadata = 1;
-  uint32 num_free_slots = 2;
+  uint32 num_free_vcores = 2;
   // All tasks must be reported until they reach the failed or completed state
   repeated TaskStatus task_status = 3;
 }
 
+// Pull-based single-task dispatch. One task processes a slice of
+// partitions; `task_id` names the task within the stage (its append-order
+// slot in `RunningStage.task_infos`), `global_output_partition_ids` gives
+// the concrete global partition ids it covers. The task's plan is
+// scheduler-side shrink-restricted so its leaves report `slice.len()`
+// partitions; the writer uses `global_output_partition_ids` to attach
+// global identity to shuffle files (except in cases where the plan itself
+// resets partitioning: the writer walks its child plan to detect
+// SortPreservingMergeExec or RepartitionExec::Hash and picks the right
+// global mapping).
 message TaskDefinition {
   uint32 task_id = 1;
   uint32 task_attempt_num = 2;
   string job_id = 3;
   uint32 stage_id = 4;
   uint32 stage_attempt_num = 5;
-  uint32 partition_id = 6;
   bytes plan = 7;
   string session_id = 9;
   uint64 launch_time = 10;
   repeated KeyValuePair props = 11;
+  // Global partition ids covered by this task, in slice order.
+  repeated uint32 global_output_partition_ids = 12;
+  // Vcores this task consumed at bind time — see TaskId.vcores_consumed.
+  uint32 vcores_consumed = 13;
 }
 
 // A set of tasks in the same stage
@@ -630,6 +682,11 @@ message ExecuteQueryParams {
   oneof query {
     bytes logical_plan = 1;
     bytes substrait_plan = 6;
+    // An already-built physical plan, serialized with the scheduler's
+    // physical extension codec. Lets clients bypass logical-plan creation
+    // on the scheduler, e.g. for plans containing custom operators that
+    // have no logical-plan representation.
+    bytes physical_plan = 7;
   }
   // reserved after removing deprecated `sql` query type.
   reserved 2;
@@ -778,6 +835,9 @@ message CancelJobResult {
 
 message CleanJobDataParams {
   string job_id = 1;
+  // Specific stage ids to remove within the job dir.
+  // Empty means remove the whole job dir (legacy behavior).
+  repeated uint32 remove_stage_ids = 2;
 }
 
 message CleanJobDataResult {
@@ -815,16 +875,20 @@ message CancelTasksResult {
 
 message RemoveJobDataParams {
   string job_id = 1;
+  // Specific stage ids to remove within the job dir.
+  // Empty means remove the whole job dir (legacy behavior).
+  repeated uint32 remove_stage_ids = 2;
 }
 
 message RemoveJobDataResult {
 }
 
 message RunningTaskInfo {
+  // Per-stage append-order slot; (job_id, stage_id, task_id) is globally
+  // unique (see TaskStatus).
   uint32 task_id = 1;
   string job_id = 2;
   uint32 stage_id = 3;
-  uint32 partition_id = 4;;
 }
 
 service SchedulerGrpc {
diff --git a/ballista/core/proto/datafusion.proto b/ballista/core/proto/datafusion.proto
index fe81be0d4a..69c003acb8 100644
--- a/ballista/core/proto/datafusion.proto
+++ b/ballista/core/proto/datafusion.proto
@@ -62,6 +62,7 @@ message LogicalPlanNode {
     RecursiveQueryNode recursive_query = 31;
     CteWorkTableScanNode cte_work_table_scan = 32;
     DmlNode dml = 33;
+    EmptyTableScanNode empty_table_scan = 34;
   }
 }
 
@@ -228,6 +229,7 @@ message AnalyzeNode {
 message ExplainNode {
   LogicalPlanNode input = 1;
   bool verbose = 2;
+  datafusion_common.ExplainFormat format = 3;
 }
 
 message AggregateNode {
@@ -270,6 +272,25 @@ message CopyToNode {
   repeated string partition_by = 7;
 }
 
+// Identifies a built-in file format supported by DataFusion.
+// Used by DefaultLogicalExtensionCodec to serialize/deserialize
+// FileFormatFactory instances (e.g. in CopyTo plans).
+enum FileFormatKind {
+  FILE_FORMAT_KIND_UNSPECIFIED = 0;
+  FILE_FORMAT_KIND_CSV = 1;
+  FILE_FORMAT_KIND_JSON = 2;
+  FILE_FORMAT_KIND_PARQUET = 3;
+  FILE_FORMAT_KIND_ARROW = 4;
+  FILE_FORMAT_KIND_AVRO = 5;
+}
+
+// Wraps a serialized FileFormatFactory with its format kind tag,
+// so the decoder can dispatch to the correct format-specific codec.
+message FileFormatProto {
+  FileFormatKind kind = 1;
+  bytes encoded_file_format = 2;
+}
+
 message DmlNode{
    enum Type {
     UPDATE = 0;
@@ -407,6 +428,8 @@ message LogicalExprNode {
 
     Unnest unnest = 35;
 
+    // Subquery expressions
+    ScalarSubqueryExprNode scalar_subquery_expr = 36;
   }
 }
 
@@ -414,6 +437,15 @@ message Wildcard {
   TableReference qualifier = 1;
 }
 
+message SubqueryNode {
+  LogicalPlanNode subquery = 1;
+  repeated LogicalExprNode outer_ref_columns = 2;
+}
+
+message ScalarSubqueryExprNode {
+  SubqueryNode subquery = 1;
+}
+
 message PlaceholderNode {
   string id = 1;
   // We serialize the data type, metadata, and nullability separately to maintain
@@ -594,11 +626,15 @@ message WhenThen {
 message CastNode {
   LogicalExprNode expr = 1;
   datafusion_common.ArrowType arrow_type = 2;
+  map<string, string> metadata = 3;
+  optional bool nullable = 4;
 }
 
 message TryCastNode {
   LogicalExprNode expr = 1;
   datafusion_common.ArrowType arrow_type = 2;
+  map<string, string> metadata = 3;
+  optional bool nullable = 4;
 }
 
 message SortExprNode {
@@ -752,6 +788,7 @@ message PhysicalPlanNode {
     AsyncFuncExecNode async_func = 36;
     BufferExecNode buffer = 37;
     ArrowScanExecNode arrow_scan = 38;
+    ScalarSubqueryExecNode scalar_subquery = 39;
   }
 }
 
@@ -853,11 +890,9 @@ message PhysicalExprNode {
   reserved 17;
 
   // Unique identifier for this expression to do deduplication during deserialization.
-  // When serializing, this is set to a unique identifier for each combination of
-  // expression, process and serialization run.
-  // When deserializing, if this ID has been seen before, the cached Arc is returned
-  // instead of creating a new one, enabling reconstruction of referential integrity
-  // across serde roundtrips.
+  // When serializing, this is set via `PhysicalExpr::expression_id`. When deserializing,
+  // this id is used by the `DeduplicatingProtoConverter` to preserve referential
+  // integrity across serde roundtrips for different expressions with the same id.
   optional uint64 expr_id = 30;
 
   oneof ExprType {
@@ -897,9 +932,21 @@ message PhysicalExprNode {
     UnknownColumn unknown_column = 20;
 
     PhysicalHashExprNode hash_expr = 21;
+
+    PhysicalScalarSubqueryExprNode scalar_subquery = 22;
+
+    PhysicalDynamicFilterNode dynamic_filter = 23;
   }
 }
 
+message PhysicalDynamicFilterNode {
+  repeated PhysicalExprNode children = 1;
+  repeated PhysicalExprNode remapped_children = 2;
+  uint64 generation = 3;
+  PhysicalExprNode inner_expr = 4;
+  bool is_complete = 5;
+}
+
 message PhysicalScalarUdfNode {
   string name = 1;
   repeated PhysicalExprNode args = 2;
@@ -958,6 +1005,9 @@ message PhysicalBinaryExprNode {
   PhysicalExprNode l = 1;
   PhysicalExprNode r = 2;
   string op = 3;
+  // Linearized operands for chains of the same operator (e.g. a AND b AND c).
+  // When present, `l` and `r` are ignored and `operands` holds the flattened list.
+  repeated PhysicalExprNode operands = 4;
 }
 
 message PhysicalDateTimeIntervalExprNode {
@@ -1018,9 +1068,6 @@ message PhysicalExtensionExprNode {
 message PhysicalHashExprNode {
   repeated PhysicalExprNode on_columns = 1;
   uint64 seed0 = 2;
-  uint64 seed1 = 3;
-  uint64 seed2 = 4;
-  uint64 seed3 = 5;
   string description = 6;
 }
 
@@ -1141,6 +1188,8 @@ message HashJoinExecNode {
   JoinFilter filter = 8;
   repeated uint32 projection = 9;
   bool null_aware = 10;
+  // Optional dynamic filter expression for pushing down to the probe side.
+  PhysicalExprNode dynamic_filter = 11;
 }
 
 enum StreamPartitionMode {
@@ -1179,6 +1228,10 @@ message AnalyzeExecNode {
   bool show_statistics = 2;
   PhysicalPlanNode input = 3;
   datafusion_common.Schema schema = 4;
+  // Optional metric category filter.
+  // Empty means "plan only". Absent (has_metric_categories=false) means "all".
+  bool has_metric_categories = 5;
+  repeated string metric_categories = 6;
 }
 
 message CrossJoinExecNode {
@@ -1268,6 +1321,8 @@ message AggregateExecNode {
   repeated MaybeFilter filter_expr = 10;
   AggLimit limit = 11;
   bool has_grouping_set = 12;
+  // Optional dynamic filter expression for pushing down to the child.
+  PhysicalExprNode dynamic_filter = 13;
 }
 
 message GlobalLimitExecNode {
@@ -1289,6 +1344,8 @@ message SortExecNode {
   // Maximum number of highest/lowest rows to fetch; negative means no limit
   int64 fetch = 3;
   bool preserve_partitioning = 4;
+  // Optional dynamic filter expression for TopK pushdown.
+  PhysicalExprNode dynamic_filter = 5;
 }
 
 message SortPreservingMergeExecNode {
@@ -1330,6 +1387,7 @@ message RepartitionExecNode{
   //   uint64 unknown = 4;
   // }
   Partitioning partitioning = 5;
+  bool preserve_order = 6;
 }
 
 message Partitioning {
@@ -1384,6 +1442,13 @@ message CteWorkTableScanNode {
     datafusion_common.Schema schema = 2;
 }
 
+message EmptyTableScanNode {
+    TableReference table_name = 1;
+    datafusion_common.Schema schema = 2;
+    ProjectionColumns projection = 3;
+    repeated LogicalExprNode filters = 4;
+}
+
 enum GenerateSeriesName {
   GS_GENERATE_SERIES = 0;
   GS_RANGE = 1;
@@ -1449,4 +1514,15 @@ message AsyncFuncExecNode {
 message BufferExecNode {
   PhysicalPlanNode input = 1;
   uint64 capacity = 2;
-}
\ No newline at end of file
+}
+
+message ScalarSubqueryExecNode {
+  PhysicalPlanNode input = 1;
+  repeated PhysicalPlanNode subqueries = 2;
+}
+
+message PhysicalScalarSubqueryExprNode {
+  datafusion_common.ArrowType data_type = 1;
+  bool nullable = 2;
+  uint32 index = 3;
+}
diff --git a/ballista/core/proto/datafusion_common.proto b/ballista/core/proto/datafusion_common.proto
index 5848455cba..dd3588d134 100644
--- a/ballista/core/proto/datafusion_common.proto
+++ b/ballista/core/proto/datafusion_common.proto
@@ -199,7 +199,7 @@ message Union{
   repeated int32 type_ids = 3;
 }
 
-// Used for List/FixedSizeList/LargeList/Struct/Map
+// Used for List/FixedSizeList/LargeList/ListView/LargeListView/Struct/Map
 message ScalarNestedValue {
   message Dictionary {
     bytes ipc_message = 1;
@@ -306,6 +306,8 @@ message ScalarValue{
     ScalarNestedValue large_list_value = 16;
     ScalarNestedValue list_value = 17;
     ScalarNestedValue fixed_size_list_value = 18;
+    ScalarNestedValue list_view_value = 46;
+    ScalarNestedValue large_list_view_value = 47;
     ScalarNestedValue struct_value = 32;
     ScalarNestedValue map_value = 41;
 
@@ -398,6 +400,8 @@ message ArrowType{
     List LIST = 25;
     List LARGE_LIST = 26;
     FixedSizeList FIXED_SIZE_LIST = 27;
+    List LIST_VIEW = 43;
+    List LARGE_LIST_VIEW = 44;
     Struct STRUCT = 28;
     Union UNION = 29;
     Dictionary DICTIONARY = 30;
@@ -430,6 +434,13 @@ message JsonWriterOptions {
 }
 
 
+enum CsvQuoteStyle {
+  NECESSARY = 0;
+  ALWAYS = 1;
+  NON_NUMERIC = 2;
+  NEVER = 3;
+}
+
 message CsvWriterOptions {
   // Compression type
   CompressionTypeVariant compression = 1;
@@ -453,6 +464,12 @@ message CsvWriterOptions {
   string escape = 10;
   // Optional flag whether to double quotes, instead of escaping. Defaults to `true`
   bool double_quote = 11;
+  // Quote style for CSV writing
+  CsvQuoteStyle quote_style = 12;
+  // Whether to ignore leading whitespace in string values
+  bool ignore_leading_whitespace = 13;
+  // Whether to ignore trailing whitespace in string values
+  bool ignore_trailing_whitespace = 14;
 }
 
 // Options controlling CSV format
@@ -476,6 +493,12 @@ message CsvOptions {
   bytes terminator = 17; // Optional terminator character as a byte
   bytes truncated_rows = 18; // Indicates if truncated rows are allowed
   optional uint32 compression_level = 19; // Optional compression level
+  // Quote style for CSV writing
+  CsvQuoteStyle quote_style = 20;
+  // Whether to ignore leading whitespace in string values
+  bytes ignore_leading_whitespace = 21;
+  // Whether to ignore trailing whitespace in string values
+  bytes ignore_trailing_whitespace = 22;
 }
 
 // Options controlling CSV format
@@ -603,6 +626,24 @@ message ParquetOptions {
   oneof max_predicate_cache_size_opt {
     uint64 max_predicate_cache_size = 33;
   }
+
+  ParquetCdcOptions content_defined_chunking = 35;
+
+  // Optional timezone applied to INT96-coerced timestamps when `coerce_int96`
+  // is set. When `Some`, INT96 columns coerce to
+  // `Timestamp(<coerce_int96>, Some(<tz>))` instead of the default
+  // `Timestamp(<coerce_int96>, None)`. No effect when `coerce_int96` is unset.
+  oneof coerce_int96_tz_opt {
+    string coerce_int96_tz = 36;
+  }
+}
+
+// Content-defined chunking (CDC) options for writing parquet files.
+message ParquetCdcOptions {
+  bool enabled = 1;
+  uint64 min_chunk_size = 2;
+  uint64 max_chunk_size = 3;
+  int32 norm_level = 4;
 }
 
 enum JoinSide {
@@ -637,4 +678,11 @@ message ColumnStats {
   Precision null_count = 3;
   Precision distinct_count = 4;
   Precision byte_size = 6;
+}
+
+enum ExplainFormat {
+  EXPLAIN_FORMAT_INDENT = 0;
+  EXPLAIN_FORMAT_TREE = 1;
+  EXPLAIN_FORMAT_PGJSON = 2;
+  EXPLAIN_FORMAT_GRAPHVIZ = 3;
 }
\ No newline at end of file
diff --git a/ballista/core/src/client.rs b/ballista/core/src/client.rs
index e3c26b158b..267ef83008 100644
--- a/ballista/core/src/client.rs
+++ b/ballista/core/src/client.rs
@@ -62,6 +62,7 @@ pub struct BallistaClient {
 impl BallistaClient {
     /// Create a new BallistaClient to connect to the executor listening on the specified
     /// host and port
+    #[allow(clippy::too_many_arguments)]
     pub async fn try_new(
         host: &str,
         port: u16,
@@ -70,6 +71,8 @@ impl BallistaClient {
         customize_endpoint: Option<Arc<BallistaConfigGrpcEndpoint>>,
         io_retries_times: u8,
         io_retry_wait_time_ms: u64,
+        initial_connection_window_size: u32,
+        initial_stream_window_size: u32,
     ) -> BResult<Self> {
         let scheme = if use_tls { "https" } else { "http" };
 
@@ -83,6 +86,15 @@ impl BallistaClient {
                 ))
             })?;
 
+        if initial_connection_window_size > 0 {
+            endpoint = endpoint
+                .initial_connection_window_size(Some(initial_connection_window_size));
+        }
+        if initial_stream_window_size > 0 {
+            endpoint =
+                endpoint.initial_stream_window_size(Some(initial_stream_window_size));
+        }
+
         if let Some(customize) = customize_endpoint {
             endpoint = customize
                 .configure_endpoint(endpoint)
diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs
index fdad8d7f8b..bef5556fa6 100644
--- a/ballista/core/src/config.rs
+++ b/ballista/core/src/config.rs
@@ -20,6 +20,8 @@
 
 use crate::error::{BallistaError, Result};
 use crate::execution_plans::DEFAULT_SHUFFLE_CHANNEL_CAPACITY;
+use datafusion::arrow::ipc::CompressionType;
+use datafusion::error::DataFusionError;
 use datafusion::{
     arrow::datatypes::DataType, common::config_err, config::ConfigExtension,
 };
@@ -41,6 +43,21 @@ pub const BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ: &str =
 /// Configuration key to prefer Flight protocol for remote shuffle reads.
 pub const BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT: &str =
     "ballista.shuffle.remote_read_prefer_flight";
+/// Configuration key for the reduce-side in-flight-bytes governor budget.
+pub const BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT: &str =
+    "ballista.shuffle.reader.max_bytes_in_flight";
+/// Configuration key for the per-address in-flight block cap.
+pub const BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS: &str =
+    "ballista.shuffle.reader.max_blocks_in_flight_per_address";
+/// Configuration key for the assumed block size when partition stats lack a byte count.
+pub const BALLISTA_SHUFFLE_READER_DEFAULT_BLOCK_SIZE: &str =
+    "ballista.shuffle.reader.default_block_size_bytes";
+/// Configuration key for the gRPC client HTTP/2 initial connection-level flow-control window.
+pub const BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE: &str =
+    "ballista.client.initial_connection_window_size";
+/// Configuration key for the gRPC client HTTP/2 initial stream-level flow-control window.
+pub const BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE: &str =
+    "ballista.client.initial_stream_window_size";
 /// max message size for gRPC clients
 pub const BALLISTA_CLIENT_GRPC_MAX_MESSAGE_SIZE: &str =
     "ballista.client.grpc_max_message_size";
@@ -86,6 +103,13 @@ pub const BALLISTA_SHUFFLE_SORT_BASED_MEMORY_LIMIT_PER_TASK_BYTES: &str =
 pub const BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES: &str =
     "ballista.optimizer.broadcast_join_threshold_bytes";
 
+/// Configuration key for the row-count threshold below which a hash join's
+/// smaller side is promoted to `CollectLeft` and lowered via the broadcast
+/// pattern. Used as a fallback when byte-size statistics are unavailable.
+/// Set to `0` to disable promotion via the row-count path.
+pub const BALLISTA_BROADCAST_JOIN_THRESHOLD_ROWS: &str =
+    "ballista.optimizer.broadcast_join_threshold_rows";
+
 /// Configuration key to enable AQE coalesce-shuffle-partitions rule.
 /// Disabled by default — opt in when the workload benefits from larger
 /// downstream tasks more than from preserved parallelism.
@@ -121,6 +145,13 @@ pub const BALLISTA_CHAOS_EXECUTION_FAULT_TYPE: &str =
 /// Configuration key for the optional RNG seed used by chaos-monkey execution.
 /// An empty string (default) means non-deterministic; set to a `u64` value for reproducibility.
 pub const BALLISTA_CHAOS_EXECUTION_SEED: &str = "ballista.testing.chaos_execution.seed";
+/// Configuration key for the compression codec used in the shuffle write process
+/// Valid values are: none, lz4, zstd
+pub const BALLISTA_SHUFFLE_COMPRESSION_CODEC: &str = "ballista.shuffle.compression.codec";
+
+/// Configuration key for the scheduler's per-task partition-slice cap.
+pub const BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK: &str =
+    "ballista.scheduler.max_partitions_per_task";
 
 /// Result type for configuration parsing operations.
 pub type ParseResult<T> = result::Result<T, String>;
@@ -150,6 +181,26 @@ static CONFIG_ENTRIES: LazyLock<HashMap<String, ConfigEntry>> = LazyLock::new(||
                          "Forces the shuffle reader to use flight reader instead of block reader for remote read. Block reader usually has better performance and resource utilization".to_string(),
                          DataType::Boolean,
                          Some((false).to_string())),
+        ConfigEntry::new(BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT.to_string(),
+                         "Reduce-side shuffle governor: maximum total in-flight bytes across concurrent remote partition fetches. Mirrors Spark's spark.reducer.maxSizeInFlight. Values above 4 GiB are clamped to 4 GiB (u32 semaphore limit).".to_string(),
+                         DataType::UInt64,
+                         Some((50331648).to_string())),
+        ConfigEntry::new(BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS.to_string(),
+                         "Reduce-side shuffle governor: maximum concurrent in-flight partition fetches to a single executor address.".to_string(),
+                         DataType::UInt64,
+                         Some((128).to_string())),
+        ConfigEntry::new(BALLISTA_SHUFFLE_READER_DEFAULT_BLOCK_SIZE.to_string(),
+                         "Assumed per-partition byte size charged to the shuffle governor when partition stats carry no byte count.".to_string(),
+                         DataType::UInt64,
+                         Some((1048576).to_string())),
+        ConfigEntry::new(BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE.to_string(),
+                         "HTTP/2 initial connection-level flow-control window for gRPC data-plane clients, in bytes. Should be >= the shuffle governor byte budget so the governor, not the transport window, is the binding backpressure. 0 leaves the tonic default.".to_string(),
+                         DataType::UInt64,
+                         Some((67108864).to_string())),
+        ConfigEntry::new(BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE.to_string(),
+                         "HTTP/2 initial stream-level flow-control window for gRPC data-plane clients, in bytes. 0 leaves the tonic default.".to_string(),
+                         DataType::UInt64,
+                         Some((16777216).to_string())),
         ConfigEntry::new(BALLISTA_CLIENT_GRPC_MAX_MESSAGE_SIZE.to_string(),
                          "Configuration for max message size in gRPC clients".to_string(),
                          DataType::UInt64,
@@ -190,15 +241,25 @@ static CONFIG_ENTRIES: LazyLock<HashMap<String, ConfigEntry>> = LazyLock::new(||
                          "Per-task buffered-bytes budget at which the sort shuffle writer spills its \
                          in-memory batches to disk. Counted independently of the runtime memory pool, so \
                          spilling kicks in even when the pool is unbounded. Total worst-case sort shuffle \
-                         memory per executor is approximately concurrent_tasks * this value.".to_string(),
+                         memory per executor is approximately vcores * this value.".to_string(),
                          DataType::UInt64,
                          Some((256 * 1024 * 1024).to_string())),
         ConfigEntry::new(BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES.to_string(),
                          "Byte-size threshold below which a hash join's smaller side is \
                           promoted to CollectLeft and lowered via the broadcast pattern. \
-                          Set to 0 to disable promotion.".to_string(),
+                          Governs broadcast selection under both the static distributed \
+                          planner and adaptive query planning (AQE). Set to 0 to disable \
+                          promotion.".to_string(),
                          DataType::UInt64,
                          Some((10 * 1024 * 1024).to_string())),
+        ConfigEntry::new(BALLISTA_BROADCAST_JOIN_THRESHOLD_ROWS.to_string(),
+                         "Row-count threshold below which a hash join's smaller side is \
+                          promoted to CollectLeft and lowered via the broadcast pattern, \
+                          used as a fallback when byte-size statistics are unavailable. \
+                          Applies to adaptive query planning (AQE). Set to 0 to disable \
+                          promotion via the row-count path.".to_string(),
+                         DataType::UInt64,
+                         Some((1_000_000).to_string())),
         ConfigEntry::new(BALLISTA_CLIENT_PULL.to_string(),
                          "Should client employ pull or push job tracking. In pull mode client will make a request to server in the loop, until job finishes. Pull mode is kept for legacy clients.".to_string(),
                          DataType::Boolean,
@@ -287,6 +348,27 @@ static CONFIG_ENTRIES: LazyLock<HashMap<String, ConfigEntry>> = LazyLock::new(||
             DataType::Utf8,
             Some("".to_string()),
         ),
+        ConfigEntry::new(
+            BALLISTA_SHUFFLE_COMPRESSION_CODEC.to_string(),
+            "Compression codec specification \
+            used in the shuffle process. Possible values: \
+            none, lz4, zstd. Defaults to lz4 to preserve current behaviour".to_string(),
+            DataType::Utf8,
+            Some("lz4".to_string()),
+        ),
+        ConfigEntry::new(
+            BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK.to_string(),
+            "Upper bound on the number of input partitions packed into a single \
+             task's `partition_slice`. `1` (default) means one task per input \
+             partition. Raise to enable multi-partition tasks (fewer tasks, \
+             parallel-sort / parallel-join wins); `0` means unbounded — the \
+             scheduler fills each task up to the executor's free vcore count. \
+             Does not apply to collapse stages, which must pack their full \
+             pending queue into a single task for correctness."
+                .to_string(),
+            DataType::UInt64,
+            Some(1.to_string()),
+        ),
     ];
     entries
         .into_iter()
@@ -413,6 +495,31 @@ impl BallistaConfig {
         self.get_usize_setting(BALLISTA_SHUFFLE_READER_MAX_REQUESTS)
     }
 
+    /// Reduce-side shuffle governor byte budget (`max_bytes_in_flight`).
+    pub fn shuffle_reader_max_bytes_in_flight(&self) -> u64 {
+        self.get_usize_setting(BALLISTA_SHUFFLE_READER_MAX_BYTES_IN_FLIGHT) as u64
+    }
+
+    /// Reduce-side shuffle governor per-address in-flight block cap.
+    pub fn shuffle_reader_max_blocks_in_flight_per_address(&self) -> usize {
+        self.get_usize_setting(BALLISTA_SHUFFLE_READER_MAX_BLOCKS_PER_ADDRESS)
+    }
+
+    /// Assumed block size charged to the governor when stats lack a byte count.
+    pub fn shuffle_reader_default_block_size_bytes(&self) -> u64 {
+        self.get_usize_setting(BALLISTA_SHUFFLE_READER_DEFAULT_BLOCK_SIZE) as u64
+    }
+
+    /// HTTP/2 initial connection-level flow-control window (bytes) for data-plane clients.
+    pub fn grpc_client_initial_connection_window_size(&self) -> u32 {
+        self.get_usize_setting(BALLISTA_CLIENT_INITIAL_CONNECTION_WINDOW_SIZE) as u32
+    }
+
+    /// HTTP/2 initial stream-level flow-control window (bytes) for data-plane clients.
+    pub fn grpc_client_initial_stream_window_size(&self) -> u32 {
+        self.get_usize_setting(BALLISTA_CLIENT_INITIAL_STREAM_WINDOW_SIZE) as u32
+    }
+
     /// Returns the gRPC client connection timeout in seconds.
     pub fn grpc_client_connect_timeout_seconds(&self) -> usize {
         self.get_usize_setting(BALLISTA_CLIENT_GRPC_CONNECT_TIMEOUT_SECONDS)
@@ -496,6 +603,14 @@ impl BallistaConfig {
         self.get_usize_setting(BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES)
     }
 
+    /// Returns the row-count threshold below which a hash join's smaller side
+    /// is promoted to `CollectLeft` and lowered via the broadcast pattern.
+    /// Used as a fallback when byte-size statistics are unavailable. `0`
+    /// disables promotion via the row-count path.
+    pub fn broadcast_join_threshold_rows(&self) -> usize {
+        self.get_usize_setting(BALLISTA_BROADCAST_JOIN_THRESHOLD_ROWS)
+    }
+
     /// Returns whether the AQE coalesce-shuffle-partitions rule is enabled.
     pub fn coalesce_enabled(&self) -> bool {
         self.get_bool_setting(BALLISTA_COALESCE_ENABLED)
@@ -506,6 +621,25 @@ impl BallistaConfig {
         self.get_bool_setting(BALLISTA_PROPAGATE_EMPTY_ENABLED)
     }
 
+    /// Returns compression codec that will be used during write stage of shuffle
+    pub fn shuffle_compression_codec(
+        &self,
+    ) -> std::result::Result<Option<CompressionType>, DataFusionError> {
+        match self
+            .get_string_setting(BALLISTA_SHUFFLE_COMPRESSION_CODEC)
+            .to_lowercase()
+            .trim()
+        {
+            "lz4" => Ok(Some(CompressionType::LZ4_FRAME)),
+            "zstd" => Ok(Some(CompressionType::ZSTD)),
+            "none" => Ok(None),
+            other => Err(DataFusionError::Configuration(format!(
+                "Got an invalid compression codec {other}: \
+                Valid options are: lz4, zstd and none"
+            ))),
+        }
+    }
+
     /// Returns the target post-coalesce partition byte size in bytes
     /// (Spark's `advisoryPartitionSizeInBytes`).
     pub fn coalesce_target_partition_bytes(&self) -> u64 {
@@ -527,6 +661,11 @@ impl BallistaConfig {
         self.get_bool_setting(BALLISTA_ADAPTIVE_JOIN_ENABLED)
     }
 
+    /// Returns the scheduler's per-task partition-slice cap. `0` means unbounded.
+    pub fn max_partitions_per_task(&self) -> usize {
+        self.get_usize_setting(BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK)
+    }
+
     /// Returns whether chaos-monkey execution injection is enabled.
     pub fn chaos_execution_enabled(&self) -> bool {
         self.get_bool_setting(BALLISTA_CHAOS_EXECUTION_ENABLED)
diff --git a/ballista/core/src/diagram.rs b/ballista/core/src/diagram.rs
index 5498c1161c..79604e2a13 100644
--- a/ballista/core/src/diagram.rs
+++ b/ballista/core/src/diagram.rs
@@ -84,43 +84,27 @@ fn build_exec_plan_diagram(
     id: &mut AtomicUsize,
     draw_entity: bool,
 ) -> Result<usize> {
-    let operator_str = if plan.as_any().downcast_ref::<AggregateExec>().is_some() {
+    let operator_str = if plan.downcast_ref::<AggregateExec>().is_some() {
         "AggregateExec"
-    } else if plan.as_any().downcast_ref::<SortExec>().is_some() {
+    } else if plan.downcast_ref::<SortExec>().is_some() {
         "SortExec"
-    } else if plan.as_any().downcast_ref::<ProjectionExec>().is_some() {
+    } else if plan.downcast_ref::<ProjectionExec>().is_some() {
         "ProjectionExec"
-    } else if plan.as_any().downcast_ref::<HashJoinExec>().is_some() {
+    } else if plan.downcast_ref::<HashJoinExec>().is_some() {
         "HashJoinExec"
-    } else if plan.as_any().downcast_ref::<DataSourceExec>().is_some() {
+    } else if plan.downcast_ref::<DataSourceExec>().is_some() {
         "DataSourceExec"
-    } else if plan.as_any().downcast_ref::<FilterExec>().is_some() {
+    } else if plan.downcast_ref::<FilterExec>().is_some() {
         "FilterExec"
-    } else if plan.as_any().downcast_ref::<ShuffleWriterExec>().is_some() {
+    } else if plan.downcast_ref::<ShuffleWriterExec>().is_some() {
         "ShuffleWriterExec"
-    } else if plan
-        .as_any()
-        .downcast_ref::<SortShuffleWriterExec>()
-        .is_some()
-    {
+    } else if plan.downcast_ref::<SortShuffleWriterExec>().is_some() {
         "SortShuffleWriterExec"
-    } else if plan
-        .as_any()
-        .downcast_ref::<UnresolvedShuffleExec>()
-        .is_some()
-    {
+    } else if plan.downcast_ref::<UnresolvedShuffleExec>().is_some() {
         "UnresolvedShuffleExec"
-    } else if plan
-        .as_any()
-        .downcast_ref::<CoalesceBatchesExec>()
-        .is_some()
-    {
+    } else if plan.downcast_ref::<CoalesceBatchesExec>().is_some() {
         "CoalesceBatchesExec"
-    } else if plan
-        .as_any()
-        .downcast_ref::<CoalescePartitionsExec>()
-        .is_some()
-    {
+    } else if plan.downcast_ref::<CoalescePartitionsExec>().is_some() {
         "CoalescePartitionsExec"
     } else {
         warn!("Unknown: {plan:?}");
@@ -137,7 +121,7 @@ fn build_exec_plan_diagram(
         )?;
     }
     for child in plan.children() {
-        if let Some(shuffle) = child.as_any().downcast_ref::<UnresolvedShuffleExec>() {
+        if let Some(shuffle) = child.downcast_ref::<UnresolvedShuffleExec>() {
             if !draw_entity {
                 writeln!(
                     w,
diff --git a/ballista/core/src/execution_plans/chaos_exec.rs b/ballista/core/src/execution_plans/chaos_exec.rs
index fda436ec58..569f4da724 100644
--- a/ballista/core/src/execution_plans/chaos_exec.rs
+++ b/ballista/core/src/execution_plans/chaos_exec.rs
@@ -42,7 +42,6 @@ use datafusion::physical_plan::{
 use futures::StreamExt;
 use rand::rngs::StdRng;
 use rand::{RngExt, SeedableRng};
-use std::any::Any;
 use std::sync::Arc;
 
 /// Physical execution plan node that randomly injects failures for chaos/robustness testing.
@@ -124,10 +123,6 @@ impl ExecutionPlan for ChaosExec {
         "ChaosExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.input.properties()
     }
@@ -218,7 +213,7 @@ impl ExecutionPlan for ChaosExec {
         self.input.metrics()
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
diff --git a/ballista/core/src/execution_plans/distributed_explain_analyze.rs b/ballista/core/src/execution_plans/distributed_explain_analyze.rs
index af02b0a22c..c562df8853 100644
--- a/ballista/core/src/execution_plans/distributed_explain_analyze.rs
+++ b/ballista/core/src/execution_plans/distributed_explain_analyze.rs
@@ -37,7 +37,6 @@ use datafusion::physical_plan::{
 };
 use datafusion_proto::logical_plan::AsLogicalPlan;
 use futures::StreamExt;
-use std::any::Any;
 use std::convert::TryInto;
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -108,10 +107,6 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedExplainAnalyzeExec
         "DistributedExplainAnalyzeExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -133,7 +128,6 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedExplainAnalyzeExec
 
         let query_exec = children.pop().unwrap();
         if query_exec
-            .as_any()
             .downcast_ref::<DistributedQueryExec<T>>()
             .is_some()
         {
@@ -173,7 +167,6 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedExplainAnalyzeExec
             }
 
             let job_id = query_exec
-                .as_any()
                 .downcast_ref::<DistributedQueryExec<T>>()
                 .ok_or_else(|| {
                     DataFusionError::Internal(
@@ -333,11 +326,13 @@ mod tests {
                         metrics: vec![
                             OperatorMetric {
                                 metric: Some(operator_metric::Metric::OutputRows(4)),
+                                partition: None,
                             },
                             OperatorMetric {
                                 metric: Some(operator_metric::Metric::ElapseTime(
                                     15_000_000,
                                 )),
+                                partition: None,
                             },
                         ],
                     }],
@@ -351,6 +346,7 @@ mod tests {
                         operator_desc: "FilterExec: a@0 > 1".to_string(),
                         metrics: vec![OperatorMetric {
                             metric: Some(operator_metric::Metric::OutputRows(2)),
+                            partition: None,
                         }],
                     }],
                 },
diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs
index 0181678104..20f3234cfe 100644
--- a/ballista/core/src/execution_plans/distributed_query.rs
+++ b/ballista/core/src/execution_plans/distributed_query.rs
@@ -46,10 +46,10 @@ use datafusion::prelude::SessionConfig;
 use datafusion_proto::logical_plan::{
     AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec,
 };
+use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec};
 use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt};
 use log::{debug, error, info};
 use parking_lot::Mutex;
-use std::any::Any;
 use std::fmt::Debug;
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -177,10 +177,6 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedQueryExec<T> {
         "DistributedQueryExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.plan.schema().as_arrow().clone().into()
     }
@@ -326,6 +322,91 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedQueryExec<T> {
     }
 }
 
+/// Submits an already-built physical plan directly to a Ballista scheduler for
+/// distributed execution, bypassing logical plan creation on the scheduler side.
+///
+/// This is a lower-level entry point than [DistributedQueryExec]: instead of a
+/// [LogicalPlan] that the scheduler turns into a physical plan itself, the caller
+/// supplies the physical plan up front - e.g. for plans containing custom
+/// operators that have no logical-plan representation.
+pub async fn execute_physical_plan<U: 'static + AsExecutionPlan>(
+    scheduler_url: String,
+    config: &BallistaConfig,
+    physical_plan: Arc<dyn ExecutionPlan>,
+    codec: &dyn PhysicalExtensionCodec,
+    session_id: String,
+    session_config: SessionConfig,
+) -> Result<SendableRecordBatchStream> {
+    let plan_message =
+        U::try_from_physical_plan(physical_plan.clone(), codec).map_err(|e| {
+            DataFusionError::Internal(format!("failed to serialize physical plan: {e:?}"))
+        })?;
+    let mut buf: Vec<u8> = vec![];
+    plan_message.try_encode(&mut buf).map_err(|e| {
+        DataFusionError::Execution(format!("failed to encode physical plan: {e:?}"))
+    })?;
+
+    let settings = session_config
+        .options()
+        .entries()
+        .iter()
+        .map(
+            |datafusion::config::ConfigEntry { key, value, .. }| KeyValuePair {
+                key: key.to_owned(),
+                value: value.clone(),
+            },
+        )
+        .collect();
+    let operation_id = uuid::Uuid::now_v7().to_string();
+    let query = ExecuteQueryParams {
+        query: Some(Query::PhysicalPlan(buf)),
+        settings,
+        session_id: session_id.clone(),
+        operation_id,
+    };
+
+    let max_message_size = config.grpc_client_max_message_size();
+    let grpc_config = GrpcClientConfig::from(config);
+    let metrics = Arc::new(ExecutionPlanMetricsSet::new());
+    let job_id_handle = Arc::new(Mutex::new(None));
+    let partition = 0;
+
+    let stream: std::pin::Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>> =
+        if session_config.ballista_config().client_pull() {
+            Box::pin(
+                execute_query_pull(
+                    scheduler_url,
+                    session_id,
+                    query,
+                    max_message_size,
+                    grpc_config,
+                    metrics,
+                    job_id_handle,
+                    partition,
+                    session_config,
+                )
+                .await?,
+            )
+        } else {
+            Box::pin(
+                execute_query_push(
+                    scheduler_url,
+                    query,
+                    max_message_size,
+                    grpc_config,
+                    metrics,
+                    job_id_handle,
+                    partition,
+                    session_config,
+                )
+                .await?,
+            )
+        };
+
+    let schema = physical_plan.schema();
+    Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
+}
+
 /// Client will periodically invoke scheduler to check
 /// job status. There is preconfigured wait period between
 /// pulls, which increases query latency.
@@ -744,6 +825,8 @@ async fn fetch_partition(
         customize_endpoint,
         io_retries_times,
         io_retry_wait_time_ms,
+        0,
+        0,
     )
     .await
     .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
@@ -842,7 +925,6 @@ mod test {
 
         let new_exec = exec.clone().with_new_children(vec![]).unwrap();
         let new_exec = new_exec
-            .as_any()
             .downcast_ref::<DistributedQueryExec<LogicalPlanNode>>()
             .unwrap();
 
diff --git a/ballista/core/src/execution_plans/mod.rs b/ballista/core/src/execution_plans/mod.rs
index f623685096..f863fd4776 100644
--- a/ballista/core/src/execution_plans/mod.rs
+++ b/ballista/core/src/execution_plans/mod.rs
@@ -32,11 +32,12 @@ use std::path::{Path, PathBuf};
 pub use chaos_exec::ChaosExec;
 use datafusion::common::exec_err;
 pub use distributed_explain_analyze::DistributedExplainAnalyzeExec;
-pub use distributed_query::DistributedQueryExec;
+pub use distributed_query::{DistributedQueryExec, execute_physical_plan};
 pub use shuffle_reader::{CoalescePlan, PartitionGroup, ShuffleReaderExec};
 pub use shuffle_reader::{stats_for_partition, stats_for_partitions};
 pub use shuffle_writer::DEFAULT_SHUFFLE_CHANNEL_CAPACITY;
 pub use shuffle_writer::ShuffleWriterExec;
+pub use shuffle_writer::compute_global_output_partition_ids;
 pub use shuffle_writer_trait::ShuffleWriter;
 pub use sort_shuffle::SortShuffleWriterExec;
 pub use unresolved_shuffle::UnresolvedShuffleExec;
diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs
index 556ccc8090..c4d7b511b1 100644
--- a/ballista/core/src/execution_plans/shuffle_reader.rs
+++ b/ballista/core/src/execution_plans/shuffle_reader.rs
@@ -34,7 +34,7 @@ use datafusion::error::{DataFusionError, Result};
 use datafusion::execution::context::TaskContext;
 use datafusion::physical_plan::coalesce::{LimitedBatchCoalescer, PushBatchStatus};
 use datafusion::physical_plan::metrics::{
-    BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet,
+    self, BaselineMetrics, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet,
 };
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
@@ -47,17 +47,18 @@ use itertools::Itertools;
 use log::{debug, error, trace};
 use rand::prelude::SliceRandom;
 use rand::rng;
-use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fs::File;
+use std::future::Future;
 use std::io::BufReader;
 use std::path::Path;
 use std::pin::Pin;
 use std::result;
 use std::sync::Arc;
 use std::task::{Context, Poll};
-use tokio::sync::{Semaphore, mpsc};
+use std::time::Duration;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
 use tokio_stream::wrappers::ReceiverStream;
 
 /// Coalesce plan attached to a `ShuffleReaderExec` or `UnresolvedShuffleExec`.
@@ -311,10 +312,6 @@ impl ExecutionPlan for ShuffleReaderExec {
         "ShuffleReaderExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -401,11 +398,14 @@ impl ExecutionPlan for ShuffleReaderExec {
                 "ShuffleReader work dir should have been set by executor".to_owned(),
             ))?;
 
+        let read_metrics = ShuffleReadMetrics::new(partition, &self.metrics);
+
         let response_receiver = send_fetch_partitions(
             work_dir,
             partition_locations,
             config,
             self.client_pool.clone(),
+            read_metrics,
         );
 
         let input_stream = Box::pin(RecordBatchStreamAdapter::new(
@@ -426,7 +426,7 @@ impl ExecutionPlan for ShuffleReaderExec {
         Some(self.metrics.clone_inner())
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         if self.broadcast {
             if let Some(idx) = partition
                 && idx != 0
@@ -445,7 +445,7 @@ impl ExecutionPlan for ShuffleReaderExec {
                 "broadcast shuffle reader at stage {} returned aggregated statistics: {:?}",
                 self.stage_id, stats
             );
-            return Ok(stats);
+            return Ok(Arc::new(stats));
         }
         if let Some(idx) = partition {
             let partition_count = self.properties().partitioning.partition_count();
@@ -474,7 +474,7 @@ impl ExecutionPlan for ShuffleReaderExec {
                 "shuffle reader at stage: {} and partition {} returned statistics: {:?}",
                 self.stage_id, idx, stat_for_partition
             );
-            stat_for_partition
+            stat_for_partition.map(Arc::new)
         } else {
             let stats_for_partitions = stats_for_partitions(
                 self.schema.fields().len(),
@@ -487,7 +487,7 @@ impl ExecutionPlan for ShuffleReaderExec {
                 "shuffle reader at stage: {} returned statistics for all partitions: {:?}",
                 self.stage_id, stats_for_partitions
             );
-            Ok(stats_for_partitions)
+            Ok(Arc::new(stats_for_partitions))
         }
     }
 }
@@ -617,6 +617,77 @@ impl Stream for AbortableReceiverStream {
             .map_err(|e| ArrowError::ExternalError(Box::new(e)))
     }
 }
+
+/// In-flight bytes charged to the governor for a block. Uses the partition's
+/// recorded byte size, falling back to `default` when stats carry none. Never 0,
+/// so every block occupies at least one permit.
+fn block_size(location: &PartitionLocation, default: u64) -> u64 {
+    location
+        .partition_stats
+        .num_bytes()
+        .unwrap_or(default)
+        .max(1)
+}
+
+/// Size of the byte semaphore. tokio permits are `usize`; clamp so it is at least
+/// 1 and never exceeds `u32::MAX` (the `acquire_many` argument type).
+fn byte_permits_cap(max_bytes: u64) -> usize {
+    max_bytes.clamp(1, u32::MAX as u64) as usize
+}
+
+/// Byte permits to acquire for a block: `min(size, max_bytes)`, clamped to
+/// `[1, u32::MAX]`. Capping at `max_bytes` means an oversized block requests the
+/// entire budget and can only proceed once all other fetches drain — the
+/// application-layer analog of Spark's `bytesInFlight == 0` progress clause.
+fn byte_permits_for(size: u64, max_bytes: u64) -> u32 {
+    let cap = max_bytes.clamp(1, u32::MAX as u64);
+    size.clamp(1, cap) as u32
+}
+
+/// Wraps a fetched partition stream and holds the governor permits for its
+/// lifetime. Dropping the stream — on normal end, consumer cancellation, or a
+/// mid-body error — releases all three permits, freeing budget for the next
+/// fetch. This is the release-on-body-completion behavior the governor needs.
+struct GovernedStream {
+    inner: SendableRecordBatchStream,
+    _byte_permit: OwnedSemaphorePermit,
+    _req_permit: OwnedSemaphorePermit,
+    _addr_permit: OwnedSemaphorePermit,
+}
+
+impl GovernedStream {
+    fn new(
+        inner: SendableRecordBatchStream,
+        byte_permit: OwnedSemaphorePermit,
+        req_permit: OwnedSemaphorePermit,
+        addr_permit: OwnedSemaphorePermit,
+    ) -> Self {
+        Self {
+            inner,
+            _byte_permit: byte_permit,
+            _req_permit: req_permit,
+            _addr_permit: addr_permit,
+        }
+    }
+}
+
+impl Stream for GovernedStream {
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        self.inner.poll_next_unpin(cx)
+    }
+}
+
+impl RecordBatchStream for GovernedStream {
+    fn schema(&self) -> SchemaRef {
+        self.inner.schema()
+    }
+}
+
 /// Splits the provided partition locations into local and remote partitions.
 /// Local partitions are read directly from local Arrow IPC files,
 /// while remote partitions are fetched using the Arrow Flight client.
@@ -635,17 +706,90 @@ fn local_remote_read_split(
     }
 }
 
+/// Fetch-side metrics for `ShuffleReaderExec`, recorded per output partition.
+///
+/// NOTE: the reader's `BaselineMetrics::elapsed_compute` measures poll time of
+/// the consuming stream, which overlaps with background fetching. `fetch_time`
+/// here is *additive* wall-time spent inside the fetch tasks — do not sum the
+/// two. `decoded_bytes` is the in-memory Arrow footprint of fetched batches,
+/// not compressed wire bytes. `fetch_time` and `permit_wait_time` are each
+/// summed across every concurrent remote fetch task, so their totals can
+/// exceed the operator's wall-clock elapsed time — read them as aggregate
+/// cost, not wall-clock.
+#[derive(Debug, Clone)]
+struct ShuffleReadMetrics {
+    /// Wall-time fetching remote partitions (Arrow-Flight fetch + buffering).
+    fetch_time: metrics::Time,
+    /// Wall-time opening node-local shuffle files.
+    local_read_time: metrics::Time,
+    /// Wall-time blocked acquiring the reduce-side in-flight governor permits
+    /// (request + per-address + byte semaphores, #1951).
+    permit_wait_time: metrics::Time,
+    /// Decoded (in-memory Arrow) bytes of fetched remote partitions.
+    decoded_bytes: metrics::Count,
+    /// Number of remote fetch attempts issued, including retries.
+    fetch_requests: metrics::Count,
+    /// Extra fetch attempts taken by the reduce-side retry loop.
+    fetch_retries: metrics::Count,
+    /// Partitions served from node-local shuffle files.
+    local_partitions: metrics::Count,
+    /// Partitions fetched from a remote executor.
+    remote_partitions: metrics::Count,
+}
+
+impl ShuffleReadMetrics {
+    fn new(partition: usize, metrics: &ExecutionPlanMetricsSet) -> Self {
+        Self {
+            fetch_time: MetricBuilder::new(metrics).subset_time("fetch_time", partition),
+            local_read_time: MetricBuilder::new(metrics)
+                .subset_time("local_read_time", partition),
+            permit_wait_time: MetricBuilder::new(metrics)
+                .subset_time("permit_wait_time", partition),
+            decoded_bytes: MetricBuilder::new(metrics)
+                .counter("decoded_bytes", partition),
+            fetch_requests: MetricBuilder::new(metrics)
+                .counter("fetch_requests", partition),
+            fetch_retries: MetricBuilder::new(metrics)
+                .counter("fetch_retries", partition),
+            local_partitions: MetricBuilder::new(metrics)
+                .counter("local_partitions", partition),
+            remote_partitions: MetricBuilder::new(metrics)
+                .counter("remote_partitions", partition),
+        }
+    }
+}
+
 fn send_fetch_partitions(
     work_dir: &str,
     partition_locations: Vec<PartitionLocation>,
     config: &SessionConfig,
     client_pool: Option<Arc<dyn BallistaClientPool>>,
+    read_metrics: ShuffleReadMetrics,
 ) -> AbortableReceiverStream {
-    let max_request_num = config.ballista_shuffle_reader_maximum_concurrent_requests();
+    let ballista_config = config.ballista_config();
+    let max_reqs = config.ballista_shuffle_reader_maximum_concurrent_requests();
+    let max_bytes = ballista_config.shuffle_reader_max_bytes_in_flight();
+    let max_blocks_per_addr =
+        ballista_config.shuffle_reader_max_blocks_in_flight_per_address();
+    let default_block_size = ballista_config.shuffle_reader_default_block_size_bytes();
     let sort_shuffle_enabled = config.ballista_sort_shuffle_enabled();
 
-    let (response_sender, response_receiver) = mpsc::channel(max_request_num);
-    let semaphore = Arc::new(Semaphore::new(max_request_num));
+    let (response_sender, response_receiver) = mpsc::channel(max_reqs.max(1));
+
+    // Reduce-side in-flight governor. Each remote fetch acquires:
+    //   * `byte_sem`  — min(block_size, max_bytes) permits (in-flight-bytes budget)
+    //   * `req_sem`   — 1 permit (in-flight-request count)
+    //   * an addr semaphore — 1 permit (per-address in-flight cap)
+    // All three are held for the lifetime of the returned stream (see
+    // GovernedStream), so budget is released only when the body is fully
+    // consumed. Because total in-flight bytes stay <= the sized h2 window, the
+    // governor — not the 64 KB transport window — is the binding backpressure,
+    // which is what makes multiplexing over few connections safe.
+    let byte_sem = Arc::new(Semaphore::new(byte_permits_cap(max_bytes)));
+    let req_sem = Arc::new(Semaphore::new(max_reqs.max(1)));
+    let addr_sems: Arc<std::sync::Mutex<HashMap<String, Arc<Semaphore>>>> =
+        Arc::new(std::sync::Mutex::new(HashMap::new()));
+
     let mut spawned_tasks: Vec<SpawnedTask<()>> = vec![];
 
     let (local_locations, remote_locations): (Vec<_>, Vec<_>) = local_remote_read_split(
@@ -660,17 +804,20 @@ fn send_fetch_partitions(
         remote_locations.len()
     );
 
+    read_metrics.local_partitions.add(local_locations.len());
+    read_metrics.remote_partitions.add(remote_locations.len());
+
     // keep local shuffle files reading in serial order for memory control.
     let response_sender_c = response_sender.clone();
-
-    //
-    // fetching local partitions (read from file)
-    //
     let work_dir = work_dir.to_string();
+    let local_read_time = read_metrics.local_read_time.clone();
     spawned_tasks.push(SpawnedTask::spawn_blocking({
         move || {
             for p in local_locations {
-                let r = fetch_partition_local(&work_dir, &p, sort_shuffle_enabled);
+                let r = {
+                    let _timer = local_read_time.timer();
+                    fetch_partition_local(&work_dir, &p, sort_shuffle_enabled)
+                };
                 if let Err(e) = response_sender_c.blocking_send(r) {
                     error!("Fail to send response event to the channel due to {e}");
                 }
@@ -678,38 +825,92 @@ fn send_fetch_partitions(
         }
     }));
 
-    //
-    // fetching remote partitions (uses grpc flight protocol)
-    //
     let grpc_config: Arc<GrpcClientConfig> = Arc::new((&config.ballista_config()).into());
     let customize_endpoint = config.ballista_override_create_grpc_client_endpoint();
     let prefer_flight = config.ballista_shuffle_reader_remote_prefer_flight();
 
+    // The reduce-side with_retry owns fetch retries; disable the client's inner
+    // establish-retry loop (set to a single attempt) so the two layers don't
+    // multiply. Read the retry budget BEFORE overriding it for the client.
+    let outer_retries = grpc_config.io_retries_times;
+    let io_wait = grpc_config.io_retry_wait_time_ms;
+    let client_grpc_config: Arc<GrpcClientConfig> = {
+        let mut c = (*grpc_config).clone();
+        c.io_retries_times = 1;
+        Arc::new(c)
+    };
+
     for p in remote_locations.into_iter() {
-        let semaphore = semaphore.clone();
+        let byte_sem = byte_sem.clone();
+        let req_sem = req_sem.clone();
+        let addr_sems = addr_sems.clone();
         let response_sender = response_sender.clone();
+        let customize_endpoint = customize_endpoint.clone();
+        let client_grpc_config = client_grpc_config.clone();
+        let client_pool = client_pool.clone();
+        let read_metrics = read_metrics.clone();
+
+        spawned_tasks.push(SpawnedTask::spawn(async move {
+            let addr = p.executor_meta.id.clone();
+            let size = block_size(&p, default_block_size);
+
+            // Time spent blocked acquiring the three governor permits (#1951).
+            let (req_permit, addr_permit, byte_permit) = {
+                let _permit_timer = read_metrics.permit_wait_time.timer();
+                let req_permit = req_sem.acquire_owned().await.unwrap();
+                let addr_sem = {
+                    let mut map = addr_sems.lock().unwrap();
+                    map.entry(addr.clone())
+                        .or_insert_with(|| {
+                            Arc::new(Semaphore::new(max_blocks_per_addr.max(1)))
+                        })
+                        .clone()
+                };
+                let addr_permit = addr_sem.acquire_owned().await.unwrap();
+                let byte_permit = byte_sem
+                    .acquire_many_owned(byte_permits_for(size, max_bytes))
+                    .await
+                    .unwrap();
+                (req_permit, addr_permit, byte_permit)
+            };
 
-        spawned_tasks.push(SpawnedTask::spawn({
-            let customize_endpoint = customize_endpoint.clone();
-            let grpc_config = grpc_config.clone();
-            let client_pool = client_pool.clone();
-            async move {
-                // Block if exceeds max request number.
-                let permit = semaphore.acquire_owned().await.unwrap();
-                let r = fetch_partition_remote(
-                    &p,
-                    grpc_config,
-                    prefer_flight,
-                    customize_endpoint,
-                    client_pool,
-                )
-                .await;
-                // Block if the channel buffer is full.
-                if let Err(e) = response_sender.send(r).await {
-                    error!("Fail to send response event to the channel due to {e}");
-                }
-                // Increase semaphore by dropping existing permits.
-                drop(permit);
+            let mut attempts = 0usize;
+            // Cloned (rather than used by reference) to avoid a partial move of
+            // `read_metrics`, which is still needed below for
+            // `fetch_requests`/`fetch_retries`.
+            let decoded_bytes = read_metrics.decoded_bytes.clone();
+            let r = {
+                let _fetch_timer = read_metrics.fetch_time.timer();
+                with_retry(outer_retries, io_wait, is_retriable_fetch_error, || {
+                    attempts += 1;
+                    fetch_partition_buffered(
+                        &p,
+                        client_grpc_config.clone(),
+                        prefer_flight,
+                        customize_endpoint.clone(),
+                        client_pool.clone(),
+                    )
+                })
+                .await
+            }
+            .map(|(schema, batches)| {
+                let bytes: usize =
+                    batches.iter().map(|b| b.get_array_memory_size()).sum();
+                decoded_bytes.add(bytes);
+                Box::pin(GovernedStream::new(
+                    buffered_stream(schema, batches),
+                    byte_permit,
+                    req_permit,
+                    addr_permit,
+                )) as SendableRecordBatchStream
+            });
+            // Total wire attempts (initial + retries), recorded only after
+            // `with_retry` has finished retrying.
+            read_metrics.fetch_requests.add(attempts);
+            read_metrics.fetch_retries.add(attempts.saturating_sub(1));
+
+            if let Err(e) = response_sender.send(r).await {
+                error!("Fail to send response event to the channel due to {e}");
             }
         }));
     }
@@ -717,6 +918,101 @@ fn send_fetch_partitions(
     AbortableReceiverStream::create(response_receiver, spawned_tasks)
 }
 
+/// Retry an idempotent async operation up to `retries` times after the initial
+/// attempt, sleeping `wait_ms` between tries. Shuffle partition fetches are
+/// idempotent (the server re-reads the file), so a transport error — including
+/// one mid-body — can be recovered by refetching the whole partition. Only
+/// errors accepted by `should_retry` are retried; anything else is returned
+/// immediately on the first failure.
+async fn with_retry<T, F, Fut>(
+    retries: u8,
+    wait_ms: u64,
+    should_retry: impl Fn(&BallistaError) -> bool,
+    mut f: F,
+) -> result::Result<T, BallistaError>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = result::Result<T, BallistaError>>,
+{
+    let mut attempt: u8 = 0;
+    loop {
+        match f().await {
+            Ok(v) => return Ok(v),
+            Err(e) => {
+                if attempt >= retries || !should_retry(&e) {
+                    return Err(e);
+                }
+                attempt += 1;
+                debug!(
+                    "retrying shuffle fetch (attempt {attempt}/{retries}) after error: {e}"
+                );
+                if wait_ms > 0 {
+                    tokio::time::sleep(Duration::from_millis(wait_ms)).await;
+                }
+            }
+        }
+    }
+}
+
+/// Transport/fetch failures worth refetching an idempotent shuffle block for.
+/// Deterministic failures (bad config, schema/decoding logic errors) are not
+/// retried. Mirrors the transport-only retry policy of the client's inner loop.
+fn is_retriable_fetch_error(e: &BallistaError) -> bool {
+    matches!(
+        e,
+        BallistaError::GrpcConnectionError(_) | BallistaError::FetchFailed(..)
+    )
+}
+
+/// Fetch a remote partition and buffer its entire body into memory, turning the
+/// open-ended stream into a discrete, refetchable unit. Buffering is what makes
+/// a mid-body transport failure retriable without emitting duplicate batches
+/// downstream. The governor charges each block its compressed (serialized) size,
+/// so the byte budget bounds concurrent in-flight **wire** bytes — which is the
+/// quantity the h2 window must accommodate. The decoded in-memory footprint is
+/// larger by the Arrow/compression expansion ratio, so peak buffered RAM exceeds
+/// the byte budget by that factor. Bounding decoded memory precisely is the
+/// deferred disk-spill work.
+async fn fetch_partition_buffered(
+    location: &PartitionLocation,
+    config: Arc<GrpcClientConfig>,
+    prefer_flight: bool,
+    customize_endpoint: Option<Arc<BallistaConfigGrpcEndpoint>>,
+    client_pool: Option<Arc<dyn BallistaClientPool>>,
+) -> result::Result<(SchemaRef, Vec<RecordBatch>), BallistaError> {
+    let stream = fetch_partition_remote(
+        location,
+        config,
+        prefer_flight,
+        customize_endpoint,
+        client_pool,
+    )
+    .await?;
+    let schema = stream.schema();
+    let metadata = &location.executor_meta;
+    let partition_id = &location.partition_id;
+    let batches = stream.try_collect::<Vec<_>>().await.map_err(|e| {
+        BallistaError::FetchFailed(
+            metadata.id.clone(),
+            partition_id.stage_id,
+            partition_id.partition_id,
+            e.to_string(),
+        )
+    })?;
+    Ok((schema, batches))
+}
+
+/// Build an in-memory `SendableRecordBatchStream` from already-buffered batches.
+fn buffered_stream(
+    schema: SchemaRef,
+    batches: Vec<RecordBatch>,
+) -> SendableRecordBatchStream {
+    Box::pin(RecordBatchStreamAdapter::new(
+        schema,
+        futures::stream::iter(batches.into_iter().map(Ok)),
+    ))
+}
+
 async fn new_ballista_client(
     host: &str,
     port: u16,
@@ -736,6 +1032,8 @@ async fn new_ballista_client(
         customize_endpoint,
         io_retries_times,
         io_retry_wait_time_ms,
+        config.initial_connection_window_size,
+        config.initial_stream_window_size,
     )
     .await
 }
@@ -1027,8 +1325,7 @@ mod tests {
                         host: "executor_1".to_string(),
                         port: 7070,
                         grpc_port: 8080,
-                        specification: ExecutorSpecification::default()
-                            .with_task_slots(1),
+                        specification: ExecutorSpecification::default().with_vcores(1),
                         os_info: ExecutorOperatingSystemSpecification::default(),
                     },
                     partition_stats: PartitionStats {
@@ -1146,7 +1443,7 @@ mod tests {
                     host: "executor_1".to_string(),
                     port: 7070,
                     grpc_port: 8080,
-                    specification: ExecutorSpecification::default().with_task_slots(1),
+                    specification: ExecutorSpecification::default().with_vcores(1),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 partition_stats: PartitionStats {
@@ -1197,7 +1494,7 @@ mod tests {
                     host: "executor_1".to_string(),
                     port: 7070,
                     grpc_port: 8080,
-                    specification: ExecutorSpecification::default().with_task_slots(1),
+                    specification: ExecutorSpecification::default().with_vcores(1),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 partition_stats: PartitionStats {
@@ -1249,7 +1546,7 @@ mod tests {
                     host: "executor_1".to_string(),
                     port: 7070,
                     grpc_port: 8080,
-                    specification: ExecutorSpecification::default().with_task_slots(1),
+                    specification: ExecutorSpecification::default().with_vcores(1),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 partition_stats: PartitionStats {
@@ -1301,7 +1598,7 @@ mod tests {
                     host: "executor_1".to_string(),
                     port: 7070,
                     grpc_port: 8080,
-                    specification: ExecutorSpecification::default().with_task_slots(1),
+                    specification: ExecutorSpecification::default().with_vcores(1),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 partition_stats: Default::default(),
@@ -1334,6 +1631,99 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn fetch_partitions_error_path_records_metrics() -> Result<()> {
+        // A single remote partition location pointing at a host with no Flight
+        // server listening, so every wire attempt fails with a
+        // `GrpcConnectionError` (remapped to `FetchFailed` by
+        // `fetch_partition_remote`), which `is_retriable_fetch_error` treats
+        // as retriable. Unlike `test_fetch_partitions_error_mapping` (which
+        // fans out 4 upstream locations into 4 concurrent remote tasks), this
+        // test uses exactly one location so there is a single fetch task and
+        // the attempt/retry counters are deterministic: with several
+        // concurrent tasks racing to error out first, the stream can return
+        // as soon as the fastest task fails, aborting the others mid-retry
+        // and making their counters nondeterministic.
+        let retries: usize = 2;
+        let config = SessionConfig::new_with_ballista()
+            .set_usize(crate::config::BALLISTA_CLIENT_IO_RETRIES_TIMES, retries)
+            .set_usize(crate::config::BALLISTA_CLIENT_IO_RETRY_WAIT_TIME_MS, 0);
+
+        let session_ctx = SessionContext::new_with_config(config);
+        let task_ctx = session_ctx.task_ctx();
+
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+            Field::new("c", DataType::Int32, false),
+        ]);
+
+        let job_id = "test_job_metrics";
+        let input_stage_id = 2;
+        let partition = PartitionLocation {
+            map_partition_id: 0,
+            partition_id: PartitionId {
+                job_id: job_id.into(),
+                stage_id: input_stage_id,
+                partition_id: 0,
+            },
+            executor_meta: ExecutorMetadata {
+                id: "executor_1".to_string(),
+                host: "executor_1".to_string(),
+                port: 7070,
+                grpc_port: 8080,
+                specification: ExecutorSpecification::default().with_vcores(1),
+                os_info: ExecutorOperatingSystemSpecification::default(),
+            },
+            partition_stats: Default::default(),
+            file_id: None,
+            is_sort_shuffle: false,
+        };
+        let work_dir = TempDir::new().unwrap();
+        let work_dir = work_dir.path().to_str().unwrap().to_owned();
+
+        let shuffle_reader_exec = ShuffleReaderExec::try_new(
+            input_stage_id,
+            vec![vec![partition]],
+            Arc::new(schema),
+            Partitioning::UnknownPartitioning(1),
+        )?
+        .with_work_dir(work_dir);
+
+        let mut stream = shuffle_reader_exec.execute(0, task_ctx)?;
+        let batches = utils::collect_stream(&mut stream).await;
+        assert!(batches.is_err());
+        let ballista_error = batches.unwrap_err();
+        assert!(matches!(
+            ballista_error,
+            BallistaError::FetchFailed(_, _, _, _)
+        ));
+
+        // The injected error is retriable (see `is_retriable_fetch_error`), so
+        // `with_retry` runs the initial attempt plus `retries` retries before
+        // giving up: total wire attempts = 1 + retries.
+        let expected_attempts = retries + 1;
+        let expected_retries = retries;
+
+        let metrics = shuffle_reader_exec
+            .metrics()
+            .expect("ShuffleReaderExec should report metrics");
+        let count = |name: &str| metrics.sum_by_name(name).map(|v| v.as_usize());
+
+        assert_eq!(count("fetch_requests"), Some(expected_attempts));
+        assert_eq!(count("fetch_retries"), Some(expected_retries));
+        assert!(
+            metrics.sum_by_name("fetch_time").is_some(),
+            "fetch_time metric should be registered"
+        );
+        assert!(
+            metrics.sum_by_name("permit_wait_time").is_some(),
+            "permit_wait_time metric should be registered"
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_send_fetch_partitions_1() {
         test_send_fetch_partitions(1, 10).await;
@@ -1382,7 +1772,10 @@ mod tests {
             .map_err(|e| DataFusionError::Execution(format!("{e:?}")))
             .unwrap();
 
-        assert_eq!(result.len(), 2);
+        // Writer's K=1 hash routes every row to partition 0. Its coordinator
+        // reads its full input slice (2 partitions × 2 batches = 4 batches),
+        // all routed to the single output file.
+        assert_eq!(result.len(), 4);
         for b in result {
             assert_eq!(b, create_test_batch())
         }
@@ -1459,11 +1852,13 @@ mod tests {
         let config = SessionConfig::new_with_ballista()
             .with_ballista_shuffle_reader_maximum_concurrent_requests(max_request_num);
 
+        let metrics_set = ExecutionPlanMetricsSet::new();
         let response_receiver = send_fetch_partitions(
             &work_dir.to_string_lossy(),
             partition_locations,
             &config,
             None,
+            ShuffleReadMetrics::new(0, &metrics_set),
         );
 
         let stream = RecordBatchStreamAdapter::new(
@@ -1475,6 +1870,90 @@ mod tests {
         assert_eq!(partition_num, result.len());
     }
 
+    #[tokio::test]
+    async fn send_fetch_partitions_records_local_metrics() {
+        let schema = get_test_partition_schema();
+        let data_array = Int32Array::from(vec![1]);
+        let batch =
+            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(data_array)])
+                .unwrap();
+        let tmp_dir = tempdir().unwrap();
+        let work_dir = tmp_dir.path();
+        let partition_num = 3usize;
+        for p in 0..partition_num {
+            let file_path =
+                create_shuffle_path(work_dir, &"job".into(), 1, p, None, false).unwrap();
+            std::fs::create_dir_all(file_path.parent().unwrap()).unwrap();
+            let file = File::create(&file_path).unwrap();
+            let mut writer = StreamWriter::try_new(file, &schema).unwrap();
+            writer.write(&batch).unwrap();
+            writer.finish().unwrap();
+        }
+        let partition_locations = get_test_partition_locations(partition_num, None);
+        let config = SessionConfig::new_with_ballista();
+        let metrics_set = ExecutionPlanMetricsSet::new();
+        let read_metrics = ShuffleReadMetrics::new(0, &metrics_set);
+
+        let response_receiver = send_fetch_partitions(
+            &work_dir.to_string_lossy(),
+            partition_locations,
+            &config,
+            None,
+            read_metrics,
+        );
+        let stream = RecordBatchStreamAdapter::new(
+            Arc::new(schema),
+            response_receiver.try_flatten(),
+        );
+        let result = common::collect(Box::pin(stream)).await.unwrap();
+        assert_eq!(partition_num, result.len());
+
+        let metrics = metrics_set.clone_inner();
+        let count =
+            |name: &str| metrics.sum_by_name(name).map(|v| v.as_usize()).unwrap_or(0);
+        assert_eq!(count("local_partitions"), partition_num);
+        assert_eq!(count("remote_partitions"), 0);
+        assert!(
+            metrics.sum_by_name("local_read_time").is_some(),
+            "local_read_time metric should be registered"
+        );
+    }
+
+    #[tokio::test]
+    async fn send_fetch_partitions_counts_remote_split() {
+        let tmp_dir = tempdir().unwrap();
+        let work_dir = tmp_dir.path();
+        // No files on disk => all partitions are treated as remote.
+        let partition_locations = get_test_partition_locations(2, None);
+        let config = SessionConfig::new_with_ballista();
+        let metrics_set = ExecutionPlanMetricsSet::new();
+        let read_metrics = ShuffleReadMetrics::new(0, &metrics_set);
+
+        // Drop the receiver without polling; the split counts are recorded
+        // synchronously inside send_fetch_partitions.
+        let _rx = send_fetch_partitions(
+            &work_dir.to_string_lossy(),
+            partition_locations,
+            &config,
+            None,
+            read_metrics,
+        );
+
+        let metrics = metrics_set.clone_inner();
+        assert_eq!(
+            metrics
+                .sum_by_name("remote_partitions")
+                .map(|v| v.as_usize()),
+            Some(2)
+        );
+        assert_eq!(
+            metrics
+                .sum_by_name("local_partitions")
+                .map(|v| v.as_usize()),
+            Some(0)
+        );
+    }
+
     fn get_test_partition_locations(
         n: usize,
         file_id: Option<u64>,
@@ -1492,7 +1971,7 @@ mod tests {
                     host: "localhost".to_string(),
                     port: 50051,
                     grpc_port: 50052,
-                    specification: ExecutorSpecification::default().with_task_slots(12),
+                    specification: ExecutorSpecification::default().with_vcores(12),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 partition_stats: Default::default(),
@@ -1850,3 +2329,104 @@ mod tests {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod governor_tests {
+    use super::*;
+    use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+
+    #[test]
+    fn byte_permits_for_caps_at_budget() {
+        // A block larger than the budget requests exactly the whole budget,
+        // so it can only run when all other fetches have drained.
+        assert_eq!(byte_permits_for(10, 100), 10);
+        assert_eq!(byte_permits_for(500, 100), 100);
+        // Never zero (a zero acquire is a no-op and would under-account).
+        assert_eq!(byte_permits_for(0, 100), 1);
+    }
+
+    #[test]
+    fn byte_permits_cap_is_nonzero_and_bounded() {
+        assert_eq!(byte_permits_cap(0), 1);
+        assert_eq!(byte_permits_cap(48 * 1024 * 1024), 48 * 1024 * 1024);
+    }
+
+    #[tokio::test]
+    async fn governed_stream_releases_permits_on_drop() {
+        let byte_sem = Arc::new(Semaphore::new(100));
+        let req_sem = Arc::new(Semaphore::new(4));
+        let addr_sem = Arc::new(Semaphore::new(2));
+
+        let byte = byte_sem.clone().acquire_many_owned(30).await.unwrap();
+        let req = req_sem.clone().acquire_owned().await.unwrap();
+        let addr = addr_sem.clone().acquire_owned().await.unwrap();
+        assert_eq!(byte_sem.available_permits(), 70);
+        assert_eq!(req_sem.available_permits(), 3);
+        assert_eq!(addr_sem.available_permits(), 1);
+
+        let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+        let empty = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::empty(),
+        )) as SendableRecordBatchStream;
+        let governed = GovernedStream::new(empty, byte, req, addr);
+        drop(governed);
+
+        assert_eq!(byte_sem.available_permits(), 100);
+        assert_eq!(req_sem.available_permits(), 4);
+        assert_eq!(addr_sem.available_permits(), 2);
+    }
+
+    #[tokio::test]
+    async fn with_retry_succeeds_after_transient_failures() {
+        use std::sync::atomic::{AtomicU8, Ordering};
+        let calls = AtomicU8::new(0);
+        let result: result::Result<u32, BallistaError> =
+            with_retry(3, 0, is_retriable_fetch_error, || {
+                let n = calls.fetch_add(1, Ordering::SeqCst);
+                async move {
+                    if n < 2 {
+                        Err(BallistaError::GrpcConnectionError("transient".to_string()))
+                    } else {
+                        Ok(42)
+                    }
+                }
+            })
+            .await;
+        assert_eq!(result.unwrap(), 42);
+        assert_eq!(calls.load(Ordering::SeqCst), 3);
+    }
+
+    #[tokio::test]
+    async fn with_retry_gives_up_after_max_attempts() {
+        use std::sync::atomic::{AtomicU8, Ordering};
+        let calls = AtomicU8::new(0);
+        let result: result::Result<u32, BallistaError> =
+            with_retry(2, 0, is_retriable_fetch_error, || {
+                calls.fetch_add(1, Ordering::SeqCst);
+                async move {
+                    Err(BallistaError::GrpcConnectionError("always".to_string()))
+                }
+            })
+            .await;
+        assert!(result.is_err());
+        // initial attempt + 2 retries = 3 calls
+        assert_eq!(calls.load(Ordering::SeqCst), 3);
+    }
+
+    #[tokio::test]
+    async fn with_retry_does_not_retry_non_retriable() {
+        use std::sync::atomic::{AtomicU8, Ordering};
+        let calls = AtomicU8::new(0);
+        let result: result::Result<u32, BallistaError> =
+            with_retry(3, 0, is_retriable_fetch_error, || {
+                calls.fetch_add(1, Ordering::SeqCst);
+                async move { Err(BallistaError::General("deterministic".to_string())) }
+            })
+            .await;
+        assert!(result.is_err());
+        // A non-retriable error must not trigger any retry attempts.
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+}
diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs
index a75925d6df..56d3787406 100644
--- a/ballista/core/src/execution_plans/shuffle_writer.rs
+++ b/ballista/core/src/execution_plans/shuffle_writer.rs
@@ -20,11 +20,7 @@
 //! partition is re-partitioned and streamed to disk in Arrow IPC format. Future stages of the query
 //! will use the ShuffleReaderExec to read these results.
 
-use datafusion::arrow::ipc::CompressionType;
-use datafusion::arrow::ipc::writer::IpcWriteOptions;
-
 use datafusion::arrow::ipc::writer::StreamWriter;
-use std::any::Any;
 use std::fmt::Debug;
 use std::fs;
 use std::fs::File;
@@ -36,12 +32,14 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use crate::JobId;
+use crate::execution_plans::SortShuffleWriterExec;
 use crate::execution_plans::create_shuffle_path;
 use crate::extension::SessionConfigExt;
 use crate::utils;
 
 use crate::serde::protobuf::ShuffleWritePartition;
 use crate::serde::scheduler::PartitionStats;
+use crate::utils::create_write_options;
 use datafusion::arrow::array::{
     ArrayBuilder, ArrayRef, StringBuilder, StructBuilder, UInt32Builder, UInt64Builder,
 };
@@ -54,28 +52,282 @@ use datafusion::physical_plan::metrics::{
     self, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet,
 };
 
+use datafusion::physical_plan::display::DisplayableExecutionPlan;
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
-    SendableRecordBatchStream, Statistics, displayable,
+    SendableRecordBatchStream, Statistics,
 };
-use futures::{StreamExt, TryFutureExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 
 use datafusion::arrow::error::ArrowError;
 use datafusion::execution::context::TaskContext;
-use datafusion::physical_plan::repartition::BatchPartitioner;
+use datafusion::physical_plan::repartition::{BatchPartitioner, RepartitionExec};
+use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use log::{debug, error};
+use std::sync::Mutex;
+use tokio::sync::oneshot;
 
 use super::shuffle_writer_trait::ShuffleWriter;
 
+/// Rule for translating a passthrough writer's local output-partition index
+/// (`0..plan.output_partitioning().partition_count()`) to a global partition
+/// id — the thing that ends up in `ShuffleWritePartition.partition_id` and
+/// keys `PartitionLocation`s downstream.
+#[derive(Debug, Clone)]
+enum GlobalPartitionMap {
+    /// The plan collapses to a single output partition (e.g.
+    /// `SortPreservingMergeExec`). Every local index → global partition 0.
+    Collapsed,
+    /// The plan re-establishes a hash-partition K-space (e.g.
+    /// `RepartitionExec::Hash(_, K)`). Local index == global (0..K).
+    HashSpace,
+    /// Nothing between the writer and the leaves rewrites partitioning —
+    /// local index `i` is `slice[i]` globally. Empty when no slice was
+    /// stamped, in which case local is used as-is (identity).
+    PassThrough(Vec<usize>),
+}
+
+impl GlobalPartitionMap {
+    fn resolve(&self, local: usize) -> u64 {
+        match self {
+            GlobalPartitionMap::Collapsed => 0,
+            GlobalPartitionMap::HashSpace => local as u64,
+            GlobalPartitionMap::PassThrough(slice) => {
+                slice.get(local).copied().unwrap_or(local) as u64
+            }
+        }
+    }
+}
+
+/// Walk from `plan` toward the leaves, stopping at the first operator that
+/// determines the output partitioning shape:
+///
+/// - `SortPreservingMergeExec` → `Collapsed` (fan-in to 1).
+/// - `RepartitionExec` producing a hash / round-robin K-space → `HashSpace`.
+/// - Otherwise recurse into the sole child (Filter/Sort/Projection/… are
+///   partitioning-preserving passthroughs).
+/// - If we hit a leaf or a fan-in without recognising it, treat it as
+///   passthrough (the caller passes `global_output_partition_ids`).
+fn walk_child_partition_mapping(
+    plan: &Arc<dyn ExecutionPlan>,
+    global_output_partition_ids: &[usize],
+) -> GlobalPartitionMap {
+    if plan.downcast_ref::<SortPreservingMergeExec>().is_some() {
+        return GlobalPartitionMap::Collapsed;
+    }
+    if let Some(repart) = plan.downcast_ref::<RepartitionExec>() {
+        match repart.partitioning() {
+            Partitioning::Hash(_, _) | Partitioning::RoundRobinBatch(_) => {
+                return GlobalPartitionMap::HashSpace;
+            }
+            Partitioning::UnknownPartitioning(_) => {
+                // RepartitionExec still exchanges rows and freshly numbers
+                // its K outputs, so global_output_partition_ids[local] would be a
+                // meaningless mapping. DataFusion's BatchPartitioner also
+                // rejects this scheme (`not_impl_err!`), so reaching this
+                // arm means an upstream invariant has broken — fail loudly.
+                panic!(
+                    "unexpected RepartitionExec::UnknownPartitioning \
+                     in shuffle-writer child plan"
+                );
+            }
+        }
+    }
+    let children = plan.children();
+    if children.len() == 1 {
+        return walk_child_partition_mapping(children[0], global_output_partition_ids);
+    }
+    GlobalPartitionMap::PassThrough(global_output_partition_ids.to_vec())
+}
+
+/// Compute the set of global output partition ids a task's writer will
+/// produce, given the global *input* partition ids the task consumes.
+///
+/// This is called on the scheduler when preparing a `TaskDefinition` for
+/// the wire. The executor-side writer then uses these ids directly rather
+/// than re-walking the plan.
+///
+/// Three cases:
+///
+/// - `SortShuffleWriter(Hash(K))` — HashSpace by construction; the K-space
+///   `[0..K-1]` is intrinsic. Input ids are irrelevant.
+/// - `ShuffleWriter(Hash(K)|RoundRobin(K))` — same K-space semantics.
+/// - `ShuffleWriter(None)` — passthrough; the child's plan shape decides:
+///   - `SortPreservingMergeExec` in the child chain → `[0]` (collapse).
+///   - `RepartitionExec(Hash|RoundRobin)` in the child chain → `[0..K-1]`.
+///   - Otherwise (leaf-preserved partitioning) → the input ids as-is.
+///
+/// Any other plan root is treated as a passthrough of the input ids —
+/// this is the compat crutch for tests and codepaths that call the
+/// helper without a real writer at the root.
+pub fn compute_global_output_partition_ids(
+    stage_plan: &Arc<dyn ExecutionPlan>,
+    global_input_partition_ids: &[usize],
+) -> Vec<usize> {
+    if let Some(w) = stage_plan.downcast_ref::<SortShuffleWriterExec>() {
+        let Partitioning::Hash(_, k) = w.shuffle_output_partitioning() else {
+            unreachable!("SortShuffleWriterExec is Hash-partitioned by construction");
+        };
+        return (0..*k).collect();
+    }
+    if let Some(w) = stage_plan.downcast_ref::<ShuffleWriterExec>() {
+        match w.shuffle_output_partitioning() {
+            Some(Partitioning::Hash(_, k)) | Some(Partitioning::RoundRobinBatch(k)) => {
+                return (0..*k).collect();
+            }
+            None => {
+                let children = stage_plan.children();
+                let [child] = children.as_slice() else {
+                    unreachable!("ShuffleWriterExec always has exactly one child");
+                };
+                return match walk_child_partition_mapping(
+                    child,
+                    global_input_partition_ids,
+                ) {
+                    GlobalPartitionMap::Collapsed => vec![0],
+                    GlobalPartitionMap::HashSpace => {
+                        let k =
+                            child.properties().output_partitioning().partition_count();
+                        (0..k).collect()
+                    }
+                    GlobalPartitionMap::PassThrough(ids) => ids,
+                };
+            }
+            Some(Partitioning::UnknownPartitioning(_)) => {
+                panic!(
+                    "ShuffleWriterExec::UnknownPartitioning has no meaningful output-id \
+                     space; upstream planner should never emit this shape"
+                );
+            }
+        }
+    }
+    global_input_partition_ids.to_vec()
+}
+
 /// Default bounded-channel capacity for the async-to-blocking I/O bridge.
 pub const DEFAULT_SHUFFLE_CHANNEL_CAPACITY: usize = 8;
 
+/// Shared state for the coordinator+handoff pattern (see range_repartition.rs
+/// upstream for the reference design). The first `execute(N)` call spawns a
+/// single coordinator task that owns all K writes; every `execute(N)` call
+/// takes its own `oneshot::Receiver` and awaits the summaries for partition N.
+///
+/// The receiver payload is a `Vec` so writers that produce multiple files per
+/// output partition (e.g. `SortShuffleWriterExec` under a multi-partition
+/// task, where each slice member yields one file each containing K logical
+/// partitions) can hand the full set to a single `execute(N)` stream.
+/// Passthrough / hash-repart writers produce at most one summary per slot.
+struct WriterState {
+    initialized: bool,
+    /// One receiver per output partition. `execute(N)` takes `handoffs[N]`;
+    /// the coordinator holds the matching sender and pushes the summaries
+    /// once partition N's files are closed.
+    handoffs: Vec<Option<oneshot::Receiver<Result<Vec<ShuffleWritePartition>>>>>,
+}
+
+impl Debug for WriterState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("WriterState")
+            .field("initialized", &self.initialized)
+            .field("handoffs", &self.handoffs.len())
+            .finish()
+    }
+}
+
 /// ShuffleWriterExec represents a section of a query plan that has consistent partitioning and
 /// can be executed as one unit with each partition being executed in parallel. The output of each
 /// partition is re-partitioned and streamed to disk in Arrow IPC format. Future stages of the query
 /// will use the ShuffleReaderExec to read these results.
-#[derive(Debug, Clone)]
+///
+/// # Threading (passthrough / `None` branch)
+///
+/// One Ballista task holds one `ShuffleWriterExec`. The first `execute(k)`
+/// call from the executor initializes K oneshot handoffs and spawns
+/// `run_coordinator`, which invokes `execute_shuffle_write`. In the
+/// passthrough branch (`shuffle_output_partitioning == None`, i.e. the
+/// child already has the target K partitions), `execute_shuffle_write`
+/// spawns K concurrent tokio tasks — one per output partition — each
+/// pulling `child.execute(k)` and streaming directly to
+/// `data-{task_id}.arrow` for partition k. Each drain emits one summary;
+/// the coordinator routes it to the oneshot whose slot matches the output
+/// partition. `execute(k)` returns a stream that awaits its oneshot and
+/// emits one metadata batch pointing at that single file.
+///
+/// K concurrent per-output drainers is the same fan-out shape as
+/// DataFusion's `RepartitionExec`, applied at the writer boundary: the
+/// writer never coordinates across output partitions in-process, and any
+/// upstream operator (e.g. `DynamicRangeRepartitionExec`) that pushes to
+/// all K senders must see all K drainers running concurrently — draining
+/// one to EOF before the next would fill the undrained channels and
+/// deadlock the scatter side.
+///
+/// Data flows bottom → top (child produces, executor consumes), matching
+/// DataFusion's convention. Under passthrough M = K (child preserves
+/// partition count). Example: K=3.
+///
+/// ```text
+///                     K=3 output partitions (pulled by executor)
+///
+///        writer.execute(0)  writer.execute(1)  writer.execute(2)
+///                ▲                 ▲                 ▲
+///                │                 │                 │
+///           ┌──────────┐      ┌──────────┐      ┌──────────┐
+///           │ oneshot  │      │ oneshot  │      │ oneshot  │
+///           │ (out=0)  │      │ (out=1)  │      │ (out=2)  │
+///           └──────────┘      └──────────┘      └──────────┘
+///                ▲                 ▲                 ▲
+///                │       each oneshot: 1 summary
+///                │       pointing at ONE data file
+///                └─────────────────┼─────────────────┘
+///                                  │  route each summary
+///                                  │  to its output slot
+///                         ┌─────────────────┐
+///                         │ run_coordinator │
+///                         └─────────────────┘
+///                                  ▲
+///              ┌───────────────────┼───────────────────┐
+///              │                   │                   │
+///     ┌────────────────┐  ┌────────────────┐  ┌────────────────┐
+///     │  drain task    │  │  drain task    │  │  drain task    │
+///     │   (out=0)      │  │   (out=1)      │  │   (out=2)      │
+///     │ stream → disk  │  │ stream → disk  │  │ stream → disk  │
+///     └────────────────┘  └────────────────┘  └────────────────┘
+///              ▲                   ▲                   ▲
+///              │                   │                   │
+///      child.execute(0)    child.execute(1)    child.execute(2)
+///
+///                     K=3 child partitions
+///          (passthrough: child preserves K, no repartitioning)
+/// ```
+///
+/// File layout — K files, one per output partition:
+///
+/// ```text
+///           .../{stage_id}/{partition_k}/data-{task_id}.arrow
+/// ```
+///
+/// Contrast with [`SortShuffleWriterExec`]:
+/// its M concurrent per-input writers produce M files, each holding K
+/// buckets internally. Here K concurrent per-output drainers produce K
+/// files, each holding exactly one output partition. Both writers expose
+/// the same K-summary contract to the framework — the on-disk shape is
+/// what differs, and downstream `ShuffleReaderExec` uses the summaries to
+/// open whichever set of files is right.
+///
+/// The `Hash` branch (`shuffle_output_partitioning == Some(Hash)`) is a
+/// legacy shape kept for correctness while the planner transitions to
+/// `RepartitionExec(Hash) → ShuffleWriterExec(None)`; not diagrammed here.
+///
+/// The coordinator + oneshot plumbing is a shared idiom with
+/// [`SortShuffleWriterExec`] and
+/// with the `Hash` branch; passthrough alone doesn't structurally need it —
+/// only the K concurrent drains, which are the real deadlock guard. Once
+/// `Hash` is retired in favor of DataFusion's `RepartitionExec(Hash)`, this
+/// branch could collapse to per-`execute(k)` eager K-spawn with
+/// `JoinHandle` handoff, keeping the coordinator idiom only where it does
+/// real work (SortShuffle's M×K summary re-bucketing).
+#[derive(Debug)]
 pub struct ShuffleWriterExec {
     /// Unique ID for the job (query) that this stage is a part of
     job_id: JobId,
@@ -88,15 +340,53 @@ pub struct ShuffleWriterExec {
     /// Optional shuffle output partitioning.
     /// If it's none, it means there's no need to do repartitioning.
     shuffle_output_partitioning: Option<Partitioning>,
+    /// Task id (the task's append-order slot in `RunningStage.task_infos`)
+    /// used as `file_id` in shuffle paths so files from different tasks
+    /// (including retries) don't collide. Seeded by the executor's
+    /// `create_query_stage_exec`; defaults to 0 for plans that arrive from
+    /// `try_new` directly (proto decode before executor stamping, or tests).
+    task_id: usize,
+    /// Global partition ids this task's restricted plan covers, in slice
+    /// order. Position `i` in the child plan corresponds to
+    /// `global_output_partition_ids[i]` globally.
+    ///
+    /// Consumed by the passthrough (`None`) branch when the plan is a straight
+    /// pass-through of the slice. The `Hash` branch ignores it — the hash
+    /// K-space is its own global identity — and even for `None`, if the child
+    /// plan contains a partitioning-resetting operator (SPM → 1 output,
+    /// RepartitionExec::Hash → 0..K) the writer detects that at path-build
+    /// time and uses `local` directly instead of `global_output_partition_ids[local]`.
+    global_output_partition_ids: Vec<usize>,
     /// Execution metrics
     metrics: ExecutionPlanMetricsSet,
     /// Plan properties
     properties: Arc<PlanProperties>,
+    /// Shared coordinator handoff state. Clones share the same Arc, so a
+    /// clone of the writer produced by `with_new_children` participates in
+    /// the same coordinator.
+    state: Arc<Mutex<WriterState>>,
+}
+
+impl Clone for ShuffleWriterExec {
+    fn clone(&self) -> Self {
+        Self {
+            job_id: self.job_id.clone(),
+            stage_id: self.stage_id,
+            plan: self.plan.clone(),
+            work_dir: self.work_dir.clone(),
+            shuffle_output_partitioning: self.shuffle_output_partitioning.clone(),
+            task_id: self.task_id,
+            global_output_partition_ids: self.global_output_partition_ids.clone(),
+            metrics: self.metrics.clone(),
+            properties: self.properties.clone(),
+            state: self.state.clone(),
+        }
+    }
 }
 
 impl std::fmt::Display for ShuffleWriterExec {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        let printable_plan = displayable(self.plan.as_ref())
+        let printable_plan = DisplayableExecutionPlan::with_metrics(self.plan.as_ref())
             .set_show_statistics(true)
             .indent(false);
         write!(
@@ -128,14 +418,23 @@ struct ShuffleWriteMetrics {
 }
 
 impl ShuffleWriteMetrics {
-    fn new(partition: usize, metrics: &ExecutionPlanMetricsSet) -> Self {
-        let write_time = MetricBuilder::new(metrics).subset_time("write_time", partition);
+    /// `input_partition` is the operator-local input partition index this
+    /// bucket tracks (0..slice_len). Under multi-partition tasks, a single
+    /// task builds K such buckets — one per operator-local input partition
+    /// it drains — so the stage still sees K per-partition buckets total,
+    /// exactly as it did before K-drain (K tasks × 1 bucket = 1 task × K
+    /// buckets). The scheduler maps `input_partition` to a stage-global
+    /// input partition id via `TaskDescription.global_input_partition_ids`.
+    fn new(input_partition: usize, metrics: &ExecutionPlanMetricsSet) -> Self {
+        let write_time =
+            MetricBuilder::new(metrics).subset_time("write_time", input_partition);
         let repart_time =
-            MetricBuilder::new(metrics).subset_time("repart_time", partition);
+            MetricBuilder::new(metrics).subset_time("repart_time", input_partition);
 
-        let input_rows = MetricBuilder::new(metrics).counter("input_rows", partition);
+        let input_rows =
+            MetricBuilder::new(metrics).counter("input_rows", input_partition);
 
-        let output_rows = MetricBuilder::new(metrics).output_rows(partition);
+        let output_rows = MetricBuilder::new(metrics).output_rows(input_partition);
 
         Self {
             write_time,
@@ -147,7 +446,9 @@ impl ShuffleWriteMetrics {
 }
 
 impl ShuffleWriterExec {
-    /// Create a new shuffle writer
+    /// Create a new shuffle writer. `task_id` defaults to 0; the executor
+    /// stamps the real value via [`Self::with_task_id`] at
+    /// `create_query_stage_exec` time.
     pub fn try_new(
         job_id: JobId,
         stage_id: usize,
@@ -160,23 +461,66 @@ impl ShuffleWriterExec {
         let partitioning = shuffle_output_partitioning
             .clone()
             .unwrap_or_else(|| plan.properties().output_partitioning().clone());
+        let output_partition_count = partitioning.partition_count();
         let properties = Arc::new(PlanProperties::new(
             datafusion::physical_expr::EquivalenceProperties::new(plan.schema()),
             partitioning,
             datafusion::physical_plan::execution_plan::EmissionType::Incremental,
             datafusion::physical_plan::execution_plan::Boundedness::Bounded,
         ));
+        // TODO: kill the default-identity slice. It's a compat crutch for
+        // plans decoded straight from proto (before executor stamping) and
+        // unit tests that don't set a slice. Once every construction path
+        // calls `with_global_output_partition_ids`, drop the default and fold it into
+        // `try_new`'s signature.
+        let child_partition_count =
+            plan.properties().output_partitioning().partition_count();
+        let default_partition_slice: Vec<usize> = (0..child_partition_count).collect();
         Ok(Self {
             job_id,
             stage_id,
             plan,
             work_dir,
             shuffle_output_partitioning,
+            task_id: 0,
+            global_output_partition_ids: default_partition_slice,
             metrics: ExecutionPlanMetricsSet::new(),
             properties,
+            state: Arc::new(Mutex::new(WriterState {
+                initialized: false,
+                handoffs: (0..output_partition_count).map(|_| None).collect(),
+            })),
         })
     }
 
+    /// Bind this writer to a specific task_id. Called by the executor
+    /// after decoding the plan so shuffle files from different tasks in
+    /// the same stage don't collide on file_id.
+    pub fn with_task_id(mut self, task_id: usize) -> Self {
+        self.task_id = task_id;
+        self
+    }
+
+    /// Task id (append-order slot within the stage) this writer instance
+    /// is bound to.
+    pub fn task_id(&self) -> usize {
+        self.task_id
+    }
+
+    /// Bind this writer to the task's assigned global partition slice.
+    pub fn with_global_output_partition_ids(
+        mut self,
+        global_output_partition_ids: Vec<usize>,
+    ) -> Self {
+        self.global_output_partition_ids = global_output_partition_ids;
+        self
+    }
+
+    /// Global partition slice this writer instance is bound to.
+    pub fn global_output_partition_ids(&self) -> &[usize] {
+        &self.global_output_partition_ids
+    }
+
     /// Get the Job ID for this query stage
     pub fn job_id(&self) -> &JobId {
         &self.job_id
@@ -200,81 +544,160 @@ impl ShuffleWriterExec {
         self.shuffle_output_partitioning.as_ref()
     }
 
-    /// Executes the shuffle write operation for a single input partition.
+    /// Executes the shuffle write operation for this task.
+    ///
+    /// Returns `(handoff_idx, summary)` pairs. `handoff_idx` indexes into the
+    /// coordinator's per-output-partition oneshot slots (0..K in this
+    /// operator's output_partitioning). `summary.partition_id` is the
+    /// **global** output partition id downstream will address:
+    ///
+    /// - Passthrough (`None`): computed via `walk_child_partition_mapping`
+    ///   over the child plan — either `global_output_partition_ids[local]`, `local` (hash
+    ///   K-space), or `0` (collapsed / SPM).
+    /// - Hash: hash bucket space is global; global = local.
     pub fn execute_shuffle_write(
         self,
-        input_partition: usize,
         context: Arc<TaskContext>,
-    ) -> impl Future<Output = Result<Vec<ShuffleWritePartition>>> {
-        let write_metrics = ShuffleWriteMetrics::new(input_partition, &self.metrics);
+    ) -> impl Future<Output = Result<Vec<(usize, ShuffleWritePartition)>>> {
+        let task_id = self.task_id;
         let output_partitioning = self.shuffle_output_partitioning.clone();
         let plan = self.plan.clone();
+        let partition_map =
+            walk_child_partition_mapping(&plan, &self.global_output_partition_ids);
+        let metrics = self.metrics.clone();
 
         async move {
             let now = Instant::now();
-            let channel_capacity = context
-                .session_config()
-                .ballista_config()
-                .shuffle_writer_channel_capacity();
-            let mut stream = plan.execute(input_partition, context)?;
+            let config = context.session_config().ballista_config();
+            let compression_type = config.shuffle_compression_codec()?;
+            let channel_capacity = config.shuffle_writer_channel_capacity();
 
             match output_partitioning {
                 None => {
-                    let path = create_shuffle_path(
-                        &self.work_dir,
-                        &self.job_id,
-                        self.stage_id,
-                        input_partition,
-                        None,
-                        false,
-                    )?;
-
-                    if let Some(parent) = path.parent() {
-                        std::fs::create_dir_all(parent)?;
-                    }
-
-                    debug!("Writing results to {path:?}");
-
-                    let stats = utils::write_stream_to_disk(
-                        &mut stream,
-                        path.as_path(),
-                        &write_metrics.write_time,
-                        channel_capacity,
-                    )
-                    .await
-                    .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
+                    // Passthrough shuffle: drain each of the child's output
+                    // partitions into its own file. All K must drain
+                    // CONCURRENTLY, not sequentially — coordinating operators
+                    // below (DynamicRangeRepartitionExec) push to all K
+                    // senders from shared scatter tasks; draining one to EOF
+                    // before the next starts fills up the undrained channel
+                    // and deadlocks the scatter side.
+                    let num_partitions =
+                        plan.properties().output_partitioning().partition_count();
+                    let mut handles = Vec::with_capacity(num_partitions);
+                    for local_input_partition in 0..num_partitions {
+                        // Each drain owns its own metric bucket, keyed by the
+                        // operator-local input partition it drains. Passthrough
+                        // is 1:1 so local input == local output here.
+                        let write_metrics =
+                            ShuffleWriteMetrics::new(local_input_partition, &metrics);
+                        let global_partition =
+                            partition_map.resolve(local_input_partition) as usize;
+                        let path = create_shuffle_path(
+                            &self.work_dir,
+                            &self.job_id,
+                            self.stage_id,
+                            global_partition,
+                            Some(task_id as u64),
+                            false,
+                        )?;
+
+                        if let Some(parent) = path.parent() {
+                            std::fs::create_dir_all(parent)?;
+                        }
 
-                    write_metrics
-                        .input_rows
-                        .add(stats.num_rows.unwrap_or(0) as usize);
-                    write_metrics
-                        .output_rows
-                        .add(stats.num_rows.unwrap_or(0) as usize);
+                        debug!("Writing results to {path:?}");
+
+                        let mut stream =
+                            plan.execute(local_input_partition, context.clone())?;
+                        handles.push(tokio::spawn(async move {
+                            let stats = utils::write_stream_to_disk(
+                                &mut stream,
+                                path.as_path(),
+                                &write_metrics.write_time,
+                                channel_capacity,
+                                compression_type,
+                            )
+                            .await
+                            .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
+                            let rows = stats.num_rows.unwrap_or(0) as usize;
+                            write_metrics.input_rows.add(rows);
+                            write_metrics.output_rows.add(rows);
+                            Ok::<_, DataFusionError>((local_input_partition, stats))
+                        }));
+                    }
 
+                    let mut results = Vec::with_capacity(num_partitions);
+                    for handle in handles {
+                        let (local_input_partition, stats) =
+                            handle.await.map_err(|e| {
+                                DataFusionError::Execution(format!(
+                                    "shuffle-write drain task panicked: {e}"
+                                ))
+                            })??;
+                        results.push((
+                            local_input_partition,
+                            ShuffleWritePartition {
+                                partition_id: partition_map
+                                    .resolve(local_input_partition),
+                                num_batches: stats.num_batches.unwrap_or(0),
+                                num_rows: stats.num_rows.unwrap_or(0),
+                                num_bytes: stats.num_bytes.unwrap_or(0),
+                                file_id: Some(task_id as u64),
+                                is_sort_shuffle: false,
+                            },
+                        ));
+                    }
                     debug!(
-                        "Executed partition {} in {} seconds. Statistics: {}",
-                        input_partition,
-                        now.elapsed().as_secs(),
-                        stats
+                        "task_id {} drained {} partitions in {}s",
+                        task_id,
+                        num_partitions,
+                        now.elapsed().as_secs()
                     );
-
-                    Ok(vec![ShuffleWritePartition {
-                        partition_id: input_partition as u64,
-                        num_batches: stats.num_batches.unwrap_or(0),
-                        num_rows: stats.num_rows.unwrap_or(0),
-                        num_bytes: stats.num_bytes.unwrap_or(0),
-                        file_id: None,
-                        is_sort_shuffle: false,
-                    }])
+                    Ok(results)
                 }
-
+                // TODO: just use Datafusion's hash partition operator and remove this branch
                 Some(Partitioning::Hash(exprs, num_output_partitions)) => {
-                    let schema = stream.schema();
-                    let (tx, mut rx) =
-                        tokio::sync::mpsc::channel::<RecordBatch>(channel_capacity);
-                    let write_time = write_metrics.write_time.clone();
-                    let repart_time = write_metrics.repart_time.clone();
-                    let output_rows = write_metrics.output_rows.clone();
+                    // Task drains ALL of the child's (already slice-restricted)
+                    // input partitions and routes rows into K writers by hash.
+                    // file_id = task_id so files from different tasks
+                    // (including retries) don't collide. Hash buckets 0..K are
+                    // a global K-space and aren't further sliced.
+                    //
+                    // One metric bucket per operator-local input partition,
+                    // matching the pre-K-drain model (before: N tasks × 1
+                    // bucket; now: 1 task × N buckets). One `BatchPartitioner`
+                    // per input partition so `repart_time` disaggregates the
+                    // same way — total partitioner count across the stage is
+                    // unchanged.
+                    let num_input_partitions =
+                        plan.properties().output_partitioning().partition_count();
+                    let write_metrics_per_input: Vec<ShuffleWriteMetrics> = (0
+                        ..num_input_partitions)
+                        .map(|i| ShuffleWriteMetrics::new(i, &metrics))
+                        .collect();
+                    let schema = plan.schema();
+                    let (tx, mut rx) = tokio::sync::mpsc::channel::<(usize, RecordBatch)>(
+                        channel_capacity,
+                    );
+                    let write_times: Vec<metrics::Time> = write_metrics_per_input
+                        .iter()
+                        .map(|m| m.write_time.clone())
+                        .collect();
+                    let output_rows_per_input: Vec<metrics::Count> =
+                        write_metrics_per_input
+                            .iter()
+                            .map(|m| m.output_rows.clone())
+                            .collect();
+                    let partitioners: Vec<BatchPartitioner> = write_metrics_per_input
+                        .iter()
+                        .map(|m| {
+                            BatchPartitioner::new_hash_partitioner(
+                                exprs.clone(),
+                                num_output_partitions,
+                                m.repart_time.clone(),
+                            )
+                        })
+                        .collect::<Result<Vec<_>, _>>()?;
                     let work_dir = self.work_dir.clone();
                     let job_id = self.job_id.clone();
                     let stage_id = self.stage_id;
@@ -282,14 +705,15 @@ impl ShuffleWriterExec {
                     let handle = tokio::task::spawn_blocking(move || {
                         let mut writers: Vec<Option<WriteTracker>> =
                             (0..num_output_partitions).map(|_| None).collect();
-                        let mut partitioner = BatchPartitioner::new_hash_partitioner(
-                            exprs,
-                            num_output_partitions,
-                            repart_time,
-                        );
-
-                        while let Some(input_batch) = rx.blocking_recv() {
-                            partitioner.partition(
+                        let mut partitioners = partitioners;
+
+                        while let Some((local_input_partition, input_batch)) =
+                            rx.blocking_recv()
+                        {
+                            let write_time = &write_times[local_input_partition];
+                            let output_rows =
+                                &output_rows_per_input[local_input_partition];
+                            partitioners[local_input_partition].partition(
                                 input_batch,
                                 |output_partition, output_batch| {
                                     let timer = write_time.timer();
@@ -305,7 +729,7 @@ impl ShuffleWriterExec {
                                                 &job_id,
                                                 stage_id,
                                                 output_partition,
-                                                Some(input_partition as u64),
+                                                Some(task_id as u64),
                                                 false,
                                             )?;
 
@@ -315,10 +739,9 @@ impl ShuffleWriterExec {
 
                                             debug!("Writing results to {p:?}");
 
-                                            let options = IpcWriteOptions::default()
-                                                .try_with_compression(Some(
-                                                    CompressionType::LZ4_FRAME,
-                                                ))?;
+                                            let options =
+                                                create_write_options(compression_type)?;
+
                                             let file =
                                                 BufWriter::new(File::create(p.clone())?);
                                             let mut writer =
@@ -353,31 +776,48 @@ impl ShuffleWriterExec {
                                     "Finished writing shuffle partition {} at {:?}. Batches: {}. Rows: {}. Bytes: {}.",
                                     i, w.path, w.num_batches, w.num_rows, num_bytes
                                 );
-                                part_locs.push(ShuffleWritePartition {
-                                    partition_id: i as u64,
-                                    num_batches: w.num_batches as u64,
-                                    num_rows: w.num_rows as u64,
-                                    num_bytes,
-                                    file_id: Some(input_partition as u64),
-                                    is_sort_shuffle: false,
-                                });
+                                part_locs.push((
+                                    i,
+                                    ShuffleWritePartition {
+                                        partition_id: i as u64,
+                                        num_batches: w.num_batches as u64,
+                                        num_rows: w.num_rows as u64,
+                                        num_bytes,
+                                        file_id: Some(task_id as u64),
+                                        is_sort_shuffle: false,
+                                    },
+                                ));
                             }
                         }
                         Ok(part_locs)
                     });
 
-                    let stream_err = loop {
-                        match stream.next().await {
-                            Some(Ok(batch)) => {
-                                write_metrics.input_rows.add(batch.num_rows());
-                                if tx.send(batch).await.is_err() {
-                                    break None;
+                    let mut stream_err = None;
+                    'outer: for (local_input_partition, per_input_metrics) in
+                        write_metrics_per_input.iter().enumerate()
+                    {
+                        let mut stream =
+                            plan.execute(local_input_partition, context.clone())?;
+                        loop {
+                            match stream.next().await {
+                                Some(Ok(batch)) => {
+                                    per_input_metrics.input_rows.add(batch.num_rows());
+                                    if tx
+                                        .send((local_input_partition, batch))
+                                        .await
+                                        .is_err()
+                                    {
+                                        break 'outer;
+                                    }
                                 }
+                                Some(Err(e)) => {
+                                    stream_err = Some(e);
+                                    break 'outer;
+                                }
+                                None => break,
                             }
-                            Some(Err(e)) => break Some(e),
-                            None => break None,
                         }
-                    };
+                    }
                     drop(tx);
 
                     let write_result = handle.await.map_err(|e| {
@@ -438,10 +878,6 @@ impl ExecutionPlan for ShuffleWriterExec {
         "ShuffleWriterExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.plan.schema()
     }
@@ -465,13 +901,19 @@ impl ExecutionPlan for ShuffleWriterExec {
                 )
             })?;
 
-            Ok(Arc::new(ShuffleWriterExec::try_new(
-                self.job_id.clone(),
-                self.stage_id,
-                input,
-                self.work_dir.clone(),
-                self.shuffle_output_partitioning.clone(),
-            )?))
+            Ok(Arc::new(
+                ShuffleWriterExec::try_new(
+                    self.job_id.clone(),
+                    self.stage_id,
+                    input,
+                    self.work_dir.clone(),
+                    self.shuffle_output_partitioning.clone(),
+                )?
+                .with_task_id(self.task_id)
+                .with_global_output_partition_ids(
+                    self.global_output_partition_ids.clone(),
+                ),
+            ))
         } else {
             Err(DataFusionError::Plan(
                 "Ballista ShuffleWriterExec expects single child".to_owned(),
@@ -479,6 +921,17 @@ impl ExecutionPlan for ShuffleWriterExec {
         }
     }
 
+    /// Return the stream for output partition `partition`.
+    ///
+    /// The first call locks the shared state, initializes K oneshot
+    /// channels, and spawns a single coordinator task that drives the
+    /// write work for all K output partitions and sends each summary to
+    /// its matching partition's oneshot sender. Subsequent calls take
+    /// their receiver and return a stream that awaits it.
+    ///
+    /// Caller (typically `DefaultQueryStageExec::execute_query_stage`)
+    /// should spawn all K `execute(N, ctx)` calls concurrently so the
+    /// stream drains don't serialize.
     fn execute(
         &self,
         partition: usize,
@@ -486,74 +939,66 @@ impl ExecutionPlan for ShuffleWriterExec {
     ) -> Result<SendableRecordBatchStream> {
         let schema = result_schema();
 
-        let schema_captured = schema.clone();
-        let job_id = self.job_id.clone();
-        let work_dir = self.work_dir.to_string();
-        let stage_id = self.stage_id;
-        let fut_stream = self
-            .clone()
-            .execute_shuffle_write(partition, context)
-            .and_then(move |part_loc| async move {
-                // build metadata result batch
-                let num_writers = part_loc.len();
-                let mut partition_builder = UInt32Builder::with_capacity(num_writers);
-                let mut path_builder =
-                    StringBuilder::with_capacity(num_writers, num_writers * 100);
-                let mut num_rows_builder = UInt64Builder::with_capacity(num_writers);
-                let mut num_batches_builder = UInt64Builder::with_capacity(num_writers);
-                let mut num_bytes_builder = UInt64Builder::with_capacity(num_writers);
-
-                for loc in &part_loc {
-                    let path = create_shuffle_path(
-                        &work_dir,
-                        &job_id,
-                        stage_id,
-                        loc.partition_id as usize,
-                        loc.file_id,
-                        false,
-                    )?
-                    .to_string_lossy()
-                    .to_string();
-
-                    path_builder.append_value(path);
-                    partition_builder.append_value(loc.partition_id as u32);
-                    num_rows_builder.append_value(loc.num_rows);
-                    num_batches_builder.append_value(loc.num_batches);
-                    num_bytes_builder.append_value(loc.num_bytes);
-                }
-
-                // build arrays
-                let partition_num: ArrayRef = Arc::new(partition_builder.finish());
-                let path: ArrayRef = Arc::new(path_builder.finish());
-                let field_builders: Vec<Box<dyn ArrayBuilder>> = vec![
-                    Box::new(num_rows_builder),
-                    Box::new(num_batches_builder),
-                    Box::new(num_bytes_builder),
-                ];
-                let mut stats_builder = StructBuilder::new(
-                    PartitionStats::default().arrow_struct_fields(),
-                    field_builders,
-                );
-                for _ in 0..num_writers {
-                    stats_builder.append(true);
-                }
-                let stats = Arc::new(stats_builder.finish());
-
-                // build result batch containing metadata
-                let batch = RecordBatch::try_new(
-                    schema_captured.clone(),
-                    vec![partition_num, path, stats],
-                )?;
+        let mut state = self.state.lock().map_err(|_| {
+            DataFusionError::Internal("ShuffleWriterExec state mutex poisoned".to_owned())
+        })?;
+
+        if !state.initialized {
+            state.initialized = true;
+            let k = state.handoffs.len();
+            let mut senders: Vec<oneshot::Sender<Result<Vec<ShuffleWritePartition>>>> =
+                Vec::with_capacity(k);
+            for slot in state.handoffs.iter_mut() {
+                let (tx, rx) = oneshot::channel();
+                senders.push(tx);
+                *slot = Some(rx);
+            }
+            let writer = self.clone();
+            let ctx = context.clone();
+            tokio::spawn(async move {
+                run_coordinator(writer, ctx, senders).await;
+            });
+        }
 
-                debug!("RESULTS METADATA:\n{batch:?}");
+        let rx = state
+            .handoffs
+            .get_mut(partition)
+            .and_then(Option::take)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "ShuffleWriterExec: execute({partition}) called twice or out of range (have {} partitions)",
+                    state.handoffs.len()
+                ))
+            })?;
+        drop(state);
 
-                MemoryStream::try_new(vec![batch], schema_captured, None)
-            })
-            .map_err(|e| ArrowError::ExternalError(Box::new(e)));
+        let work_dir = self.work_dir.clone();
+        let job_id = self.job_id.clone();
+        let stage_id = self.stage_id;
+        let schema_captured = schema.clone();
 
         Ok(Box::pin(RecordBatchStreamAdapter::new(
             schema,
-            futures::stream::once(fut_stream).try_flatten(),
+            futures::stream::once(async move {
+                let summaries = rx
+                    .await
+                    .map_err(|_| {
+                        DataFusionError::Internal(
+                            "ShuffleWriterExec coordinator dropped without sending"
+                                .to_owned(),
+                        )
+                    })?
+                    .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
+                summaries_to_batch(
+                    summaries,
+                    schema_captured,
+                    &work_dir,
+                    &job_id,
+                    stage_id,
+                )
+                .map_err(|e| ArrowError::ExternalError(Box::new(e)))
+            })
+            .try_flatten(),
         )))
     }
 
@@ -561,7 +1006,7 @@ impl ExecutionPlan for ShuffleWriterExec {
         Some(self.metrics.clone_inner())
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         self.plan.partition_statistics(partition)
     }
 }
@@ -591,15 +1036,143 @@ impl ShuffleWriter for ShuffleWriterExec {
     }
 }
 
-fn result_schema() -> SchemaRef {
+/// Internal metadata schema handed from a writer's coordinator up to
+/// `drive_shuffle_writer_stage`. Purely a handoff shape — not persisted, not
+/// sent to the scheduler. Shared between `ShuffleWriterExec` and
+/// `SortShuffleWriterExec` so the executor can drive both via the same
+/// `execute(N)` contract.
+pub(crate) fn result_schema() -> SchemaRef {
     let stats = PartitionStats::default();
     Arc::new(Schema::new(vec![
         Field::new("partition", DataType::UInt32, false),
         Field::new("path", DataType::Utf8, false),
+        Field::new("file_id", DataType::UInt64, true),
         stats.arrow_struct_repr(),
     ]))
 }
 
+/// Drives the shared write work for all K output partitions. Runs
+/// `execute_shuffle_write` once, then routes each `(handoff_idx, summary)`
+/// pair to the matching sender. Slots that never receive a summary (partition
+/// produced no rows) are filled with an empty sentinel so their `execute(N)`
+/// stream terminates cleanly.
+///
+/// On failure, sends the error to every sender so no waiting stream hangs.
+async fn run_coordinator(
+    writer: ShuffleWriterExec,
+    ctx: Arc<TaskContext>,
+    senders: Vec<oneshot::Sender<Result<Vec<ShuffleWritePartition>>>>,
+) {
+    let k = senders.len();
+    let mut senders: Vec<Option<oneshot::Sender<Result<Vec<ShuffleWritePartition>>>>> =
+        senders.into_iter().map(Some).collect();
+
+    match writer.execute_shuffle_write(ctx).await {
+        Ok(summaries) => {
+            // Bucket per-file summaries by their handoff slot. Passthrough and
+            // hash writers put at most one summary per slot; sort-based
+            // writers put up to `slice.len()` (one per input file in the
+            // slice). Slots with no summaries stay empty and the receiver's
+            // execute(N) stream ends up emitting an empty metadata batch.
+            let mut grouped: Vec<Vec<ShuffleWritePartition>> =
+                (0..k).map(|_| Vec::new()).collect();
+            for (handoff_idx, summary) in summaries {
+                if handoff_idx < k {
+                    grouped[handoff_idx].push(summary);
+                }
+            }
+            for (slot, bucket) in senders.iter_mut().zip(grouped) {
+                if let Some(sender) = slot.take() {
+                    let _ = sender.send(Ok(bucket));
+                }
+            }
+        }
+        Err(e) => {
+            let msg = format!("{e:?}");
+            for (i, slot) in senders.iter_mut().enumerate() {
+                if let Some(sender) = slot.take() {
+                    let err_msg = if i == 0 {
+                        msg.clone()
+                    } else {
+                        format!("shuffle writer failed: {msg}")
+                    };
+                    let _ = sender.send(Err(DataFusionError::Execution(err_msg)));
+                }
+            }
+        }
+    }
+}
+
+/// Build a metadata `MemoryStream` describing the files a single output
+/// partition was written to (produced by the coordinator via oneshot). One
+/// row per summary — sort-based writers put multiple summaries in a single
+/// slot (one per input file the task drained), passthrough / hash writers put
+/// at most one.
+pub(crate) fn summaries_to_batch(
+    summaries: Vec<ShuffleWritePartition>,
+    schema: SchemaRef,
+    work_dir: &str,
+    job_id: &JobId,
+    stage_id: usize,
+) -> Result<MemoryStream> {
+    let cap = summaries.len().max(1);
+    let mut partition_builder = UInt32Builder::with_capacity(cap);
+    let mut path_builder = StringBuilder::with_capacity(cap, cap * 64);
+    let mut file_id_builder = UInt64Builder::with_capacity(cap);
+    let mut num_rows_builder = UInt64Builder::with_capacity(cap);
+    let mut num_batches_builder = UInt64Builder::with_capacity(cap);
+    let mut num_bytes_builder = UInt64Builder::with_capacity(cap);
+
+    for summary in &summaries {
+        let path = create_shuffle_path(
+            work_dir,
+            job_id,
+            stage_id,
+            summary.partition_id as usize,
+            summary.file_id,
+            summary.is_sort_shuffle,
+        )?
+        .to_string_lossy()
+        .to_string();
+
+        partition_builder.append_value(summary.partition_id as u32);
+        path_builder.append_value(path);
+        match summary.file_id {
+            Some(fid) => file_id_builder.append_value(fid),
+            None => file_id_builder.append_null(),
+        }
+        num_rows_builder.append_value(summary.num_rows);
+        num_batches_builder.append_value(summary.num_batches);
+        num_bytes_builder.append_value(summary.num_bytes);
+    }
+
+    let partition_num: ArrayRef = Arc::new(partition_builder.finish());
+    let path_arr: ArrayRef = Arc::new(path_builder.finish());
+    let file_id_arr: ArrayRef = Arc::new(file_id_builder.finish());
+    let field_builders: Vec<Box<dyn ArrayBuilder>> = vec![
+        Box::new(num_rows_builder),
+        Box::new(num_batches_builder),
+        Box::new(num_bytes_builder),
+    ];
+    let mut stats_builder = StructBuilder::new(
+        PartitionStats::default().arrow_struct_fields(),
+        field_builders,
+    );
+    for _ in &summaries {
+        stats_builder.append(true);
+    }
+    let stats = Arc::new(stats_builder.finish());
+
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![partition_num, path_arr, file_id_arr, stats],
+    )?;
+
+    debug!("SHUFFLE PARTITION METADATA:\n{batch:?}");
+
+    MemoryStream::try_new(vec![batch], schema, None)
+}
+
 #[cfg(test)]
 #[allow(dead_code, unused_imports)] // clippy false positive with local imports
 mod tests {
@@ -612,6 +1185,37 @@ mod tests {
     use datafusion::prelude::SessionContext;
     use tempfile::TempDir;
 
+    /// Spawn K parallel `plan.execute(N, ctx)` calls, collect each stream's
+    /// batches, and return them concatenated in partition order. Mirrors
+    /// what `DefaultQueryStageExec::execute_query_stage` does in production:
+    /// the writer's coordinator only runs once every oneshot receiver has
+    /// been taken.
+    async fn drive_all_partitions(
+        plan: Arc<ShuffleWriterExec>,
+        task_ctx: Arc<TaskContext>,
+    ) -> Result<Vec<RecordBatch>> {
+        let k = plan.properties().output_partitioning().partition_count();
+        let mut handles = Vec::with_capacity(k);
+        for n in 0..k {
+            let plan = plan.clone();
+            let ctx = task_ctx.clone();
+            handles.push(tokio::spawn(async move {
+                let mut stream = plan.execute(n, ctx)?;
+                utils::collect_stream(&mut stream)
+                    .await
+                    .map_err(|e| DataFusionError::Execution(format!("{e:?}")))
+            }));
+        }
+        let mut all = Vec::new();
+        for h in handles {
+            let batches = h.await.map_err(|e| {
+                DataFusionError::Execution(format!("drive_all_partitions panic: {e}"))
+            })??;
+            all.extend(batches);
+        }
+        Ok(all)
+    }
+
     #[tokio::test]
     // number of rows in each partition is a function of the hash output, so don't test here
     #[cfg(not(feature = "force_hash_collisions"))]
@@ -621,52 +1225,84 @@ mod tests {
 
         let input_plan = Arc::new(CoalescePartitionsExec::new(create_input_plan()?));
         let work_dir = TempDir::new()?;
-        let query_stage = ShuffleWriterExec::try_new(
+        let query_stage = Arc::new(ShuffleWriterExec::try_new(
             JobId::new("jobOne"),
             1,
             input_plan,
             work_dir.path().to_str().unwrap().to_owned(),
             Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)),
+        )?);
+        let batches = drive_all_partitions(query_stage, task_ctx).await?;
+        // K=2 output partitions -> one metadata batch per execute(N) call
+        // (drive_all_partitions collects one per K). Empty output slots
+        // now emit 0-row batches (no summary), non-empty ones emit 1-row.
+        assert_eq!(2, batches.len());
+        for batch in &batches {
+            assert_eq!(4, batch.num_columns());
+            let path = batch
+                .column(1)
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            for row in 0..batch.num_rows() {
+                let f = path.value(row);
+                assert!(
+                    (0..2).any(|p| f.ends_with(&format!("/jobOne/1/{p}/data-0.arrow"))
+                        || f.ends_with(&format!("\\jobOne\\1\\{p}\\data-0.arrow"))),
+                    "unexpected shuffle file path: {f}"
+                );
+            }
+        }
+
+        let total: u64 = batches
+            .iter()
+            .flat_map(|b| {
+                let stats = b.column(3).as_any().downcast_ref::<StructArray>().unwrap();
+                let num_rows = stats
+                    .column_by_name("num_rows")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .clone();
+                (0..b.num_rows()).map(move |i| num_rows.value(i))
+            })
+            .sum();
+        // Row conservation: 2 input partitions × 2 batches × 2 rows = 8.
+        assert_eq!(8, total);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[cfg(not(feature = "force_hash_collisions"))]
+    async fn display_renders_child_operator_metrics() -> Result<()> {
+        let session_ctx = SessionContext::new();
+        let task_ctx = session_ctx.task_ctx();
+
+        let input_plan = Arc::new(CoalescePartitionsExec::new(create_input_plan()?));
+        let work_dir = TempDir::new()?;
+        let query_stage = ShuffleWriterExec::try_new(
+            JobId::new("jobDisplay"),
+            1,
+            input_plan,
+            work_dir.path().to_str().unwrap().to_owned(),
+            Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)),
         )?;
         let mut stream = query_stage.execute(0, task_ctx)?;
-        let batches = utils::collect_stream(&mut stream)
+        let _ = utils::collect_stream(&mut stream)
             .await
             .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
-        assert_eq!(1, batches.len());
-        let batch = &batches[0];
-        assert_eq!(3, batch.num_columns());
-        assert_eq!(2, batch.num_rows());
-        let path = batch.columns()[1]
-            .as_any()
-            .downcast_ref::<StringArray>()
-            .unwrap();
-
-        let file0 = path.value(0);
+
+        let rendered = format!("{query_stage}");
         assert!(
-            file0.ends_with("/jobOne/1/0/data-0.arrow")
-                || file0.ends_with("\\jobOne\\1\\0\\data-0.arrow")
+            rendered.contains("metrics="),
+            "expected child-operator metrics in rendered plan:\n{rendered}"
         );
-        let file1 = path.value(1);
-
         assert!(
-            file1.ends_with("/jobOne/1/1/data-0.arrow")
-                || file1.ends_with("\\jobOne\\1\\1\\data-0.arrow")
+            rendered.contains("elapsed_compute"),
+            "expected populated elapsed_compute metric in rendered plan:\n{rendered}"
         );
-
-        let stats = batch.columns()[2]
-            .as_any()
-            .downcast_ref::<StructArray>()
-            .unwrap();
-
-        let num_rows = stats
-            .column_by_name("num_rows")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<UInt64Array>()
-            .unwrap();
-        assert_eq!(4, num_rows.value(0));
-        assert_eq!(4, num_rows.value(1));
-
         Ok(())
     }
 
@@ -679,33 +1315,32 @@ mod tests {
 
         let input_plan = create_input_plan()?;
         let work_dir = TempDir::new()?;
-        let query_stage = ShuffleWriterExec::try_new(
+        let query_stage = Arc::new(ShuffleWriterExec::try_new(
             JobId::new("jobOne"),
             1,
             input_plan,
             work_dir.path().to_str().unwrap().to_owned(),
             Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)),
-        )?;
-        let mut stream = query_stage.execute(0, task_ctx)?;
-        let batches = utils::collect_stream(&mut stream)
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
-        assert_eq!(1, batches.len());
-        let batch = &batches[0];
-        assert_eq!(3, batch.num_columns());
-        assert_eq!(2, batch.num_rows());
-        let stats = batch.columns()[2]
-            .as_any()
-            .downcast_ref::<StructArray>()
-            .unwrap();
-        let num_rows = stats
-            .column_by_name("num_rows")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<UInt64Array>()
-            .unwrap();
-        assert_eq!(2, num_rows.value(0));
-        assert_eq!(2, num_rows.value(1));
+        )?);
+        let batches = drive_all_partitions(query_stage, task_ctx).await?;
+        assert_eq!(2, batches.len());
+        let total: u64 = batches
+            .iter()
+            .flat_map(|b| {
+                let stats = b.column(3).as_any().downcast_ref::<StructArray>().unwrap();
+                let num_rows = stats
+                    .column_by_name("num_rows")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .clone();
+                (0..b.num_rows()).map(move |i| num_rows.value(i))
+            })
+            .sum();
+        // Row conservation across the K output partitions: writer reads its
+        // full input slice (2 partitions × 2 batches × 2 rows = 8).
+        assert_eq!(8, total);
 
         Ok(())
     }
@@ -715,24 +1350,29 @@ mod tests {
         let session_ctx = SessionContext::new();
         let task_ctx = session_ctx.task_ctx();
 
-        // Place a directory at the data.arrow path so File::create fails
-        // inside write_stream_to_disk, not at create_dir_all.
-        // Path structure: work_dir / job_id / stage_id / partition / data.arrow
+        // Place a directory at the data-{file_id}.arrow path so File::create
+        // fails inside write_stream_to_disk, not at create_dir_all.
+        // Path: work_dir / job_id / stage_id / partition / data-{file_id}.arrow
+        // task_slot defaults to 0 in try_new, so file_id is 0.
         let tmp = tempfile::TempDir::new().unwrap();
         let data_arrow_dir = tmp
             .path()
             .join("jobOne")
             .join("1")
             .join("0")
-            .join("data.arrow");
+            .join("data-0.arrow");
         std::fs::create_dir_all(&data_arrow_dir).unwrap();
         let work_dir = tmp.path().to_str().unwrap().to_owned();
 
         let input_plan = Arc::new(CoalescePartitionsExec::new(create_input_plan()?));
-        let query_stage =
-            ShuffleWriterExec::try_new("jobOne".into(), 1, input_plan, work_dir, None)?;
-        let mut stream = query_stage.execute(0, task_ctx)?;
-        let result = utils::collect_stream(&mut stream).await;
+        let query_stage = Arc::new(ShuffleWriterExec::try_new(
+            "jobOne".into(),
+            1,
+            input_plan,
+            work_dir,
+            None,
+        )?);
+        let result = drive_all_partitions(query_stage, task_ctx).await;
         assert!(
             result.is_err(),
             "expected File::create failure in write_stream_to_disk to propagate"
@@ -757,15 +1397,14 @@ mod tests {
         let work_dir = tmp.path().to_str().unwrap().to_owned();
 
         let input_plan = Arc::new(CoalescePartitionsExec::new(create_input_plan()?));
-        let query_stage = ShuffleWriterExec::try_new(
+        let query_stage = Arc::new(ShuffleWriterExec::try_new(
             "jobOne".into(),
             1,
             input_plan,
             work_dir,
             Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)),
-        )?;
-        let mut stream = query_stage.execute(0, task_ctx)?;
-        let result = utils::collect_stream(&mut stream).await;
+        )?);
+        let result = drive_all_partitions(query_stage, task_ctx).await;
         assert!(
             result.is_err(),
             "expected create_dir_all failure in hash writer to propagate"
diff --git a/ballista/core/src/execution_plans/sort_shuffle/buffer.rs b/ballista/core/src/execution_plans/sort_shuffle/buffer.rs
index cc3ea53f85..b5dc81abb3 100644
--- a/ballista/core/src/execution_plans/sort_shuffle/buffer.rs
+++ b/ballista/core/src/execution_plans/sort_shuffle/buffer.rs
@@ -25,6 +25,9 @@
 use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::arrow::record_batch::RecordBatch;
 
+/// Return type of [`BufferedBatches::take`]: `(batches, per-partition indices)`.
+pub type BufferedTake = (Vec<RecordBatch>, Vec<Vec<(u32, u32)>>);
+
 /// Holds whole input `RecordBatch`es and per-partition `(batch_idx, row_idx)`
 /// index lists. Rows are not copied at insertion time — only the indices are
 /// recorded. Materialization happens through `PartitionedBatchIterator` at
@@ -109,7 +112,7 @@ impl BufferedBatches {
 
     /// Drains all state, returning the buffered batches and per-partition
     /// index lists. After this call the buffer is empty.
-    pub fn take(&mut self) -> (Vec<RecordBatch>, Vec<Vec<(u32, u32)>>) {
+    pub fn take(&mut self) -> BufferedTake {
         self.num_buffered_rows = 0;
         let batches = std::mem::take(&mut self.batches);
         // Preserve the partition count by replacing each inner vec with empty
diff --git a/ballista/core/src/execution_plans/sort_shuffle/config.rs b/ballista/core/src/execution_plans/sort_shuffle/config.rs
index 8c901f5dc1..5dd4b15095 100644
--- a/ballista/core/src/execution_plans/sort_shuffle/config.rs
+++ b/ballista/core/src/execution_plans/sort_shuffle/config.rs
@@ -17,15 +17,11 @@
 
 //! Configuration for sort-based shuffle.
 
-use datafusion::arrow::ipc::CompressionType;
-
 /// Configuration for sort-based shuffle.
 #[derive(Debug, Clone)]
 pub struct SortShuffleConfig {
     /// Whether sort-based shuffle is enabled (default: false).
     pub enabled: bool,
-    /// Compression codec for shuffle data (default: LZ4_FRAME).
-    pub compression: CompressionType,
     /// Target batch size in rows when materializing buffered indices via
     /// `interleave_record_batch` (default: 8192).
     pub batch_size: usize,
@@ -39,7 +35,6 @@ impl Default for SortShuffleConfig {
     fn default() -> Self {
         Self {
             enabled: false,
-            compression: CompressionType::LZ4_FRAME,
             batch_size: 8192,
             memory_limit_per_task_bytes: 256 * 1024 * 1024,
         }
@@ -48,10 +43,9 @@ impl Default for SortShuffleConfig {
 
 impl SortShuffleConfig {
     /// Creates a new configuration with the default per-task memory limit.
-    pub fn new(enabled: bool, compression: CompressionType, batch_size: usize) -> Self {
+    pub fn new(enabled: bool, batch_size: usize) -> Self {
         Self {
             enabled,
-            compression,
             batch_size,
             memory_limit_per_task_bytes: Self::default().memory_limit_per_task_bytes,
         }
@@ -72,16 +66,14 @@ mod tests {
     fn test_default_config() {
         let config = SortShuffleConfig::default();
         assert!(!config.enabled);
-        assert!(matches!(config.compression, CompressionType::LZ4_FRAME));
         assert_eq!(config.batch_size, 8192);
         assert_eq!(config.memory_limit_per_task_bytes, 256 * 1024 * 1024);
     }
 
     #[test]
     fn test_new() {
-        let config = SortShuffleConfig::new(true, CompressionType::LZ4_FRAME, 4096);
+        let config = SortShuffleConfig::new(true, 4096);
         assert!(config.enabled);
-        assert!(matches!(config.compression, CompressionType::LZ4_FRAME));
         assert_eq!(config.batch_size, 4096);
         assert_eq!(config.memory_limit_per_task_bytes, 256 * 1024 * 1024);
     }
diff --git a/ballista/core/src/execution_plans/sort_shuffle/spill.rs b/ballista/core/src/execution_plans/sort_shuffle/spill.rs
index c1da095dd7..6f768d86f6 100644
--- a/ballista/core/src/execution_plans/sort_shuffle/spill.rs
+++ b/ballista/core/src/execution_plans/sort_shuffle/spill.rs
@@ -51,7 +51,7 @@ pub struct SpillManager {
     /// Active writers per partition, kept open for appending
     active_writers: HashMap<usize, StreamWriter<BufWriter<File>>>,
     /// Compression codec for spill files
-    compression: CompressionType,
+    compression: Option<CompressionType>,
     /// Total number of batches written across all spill files. One call to
     /// `spill_all_partitions` typically increments this multiple times (once
     /// per partition that had buffered rows).
@@ -78,7 +78,7 @@ impl SpillManager {
         stage_id: usize,
         input_partition: usize,
         schema: SchemaRef,
-        compression: CompressionType,
+        compression: Option<CompressionType>,
     ) -> Result<Self> {
         let mut spill_dir = PathBuf::from(work_dir);
         spill_dir.push(job_id.as_str());
@@ -119,8 +119,8 @@ impl SpillManager {
             let file = File::create(&spill_path).map_err(BallistaError::IoError)?;
             let buffered = BufWriter::new(file);
 
-            let options = IpcWriteOptions::default()
-                .try_with_compression(Some(self.compression))?;
+            let options =
+                IpcWriteOptions::default().try_with_compression(self.compression)?;
 
             let writer =
                 StreamWriter::try_new_with_options(buffered, &self.schema, options)?;
@@ -278,7 +278,7 @@ mod tests {
             1,
             0,
             schema.clone(),
-            CompressionType::LZ4_FRAME,
+            Some(CompressionType::LZ4_FRAME),
         )?;
 
         let b1 = create_test_batch(&schema, vec![1, 2, 3]);
@@ -313,7 +313,7 @@ mod tests {
             1,
             0,
             schema.clone(),
-            CompressionType::LZ4_FRAME,
+            Some(CompressionType::LZ4_FRAME),
         )?;
 
         manager.spill(0, &create_test_batch(&schema, vec![1, 2]))?;
@@ -350,7 +350,7 @@ mod tests {
             1,
             0,
             schema.clone(),
-            CompressionType::LZ4_FRAME,
+            Some(CompressionType::LZ4_FRAME),
         )?;
 
         manager.spill(0, &create_test_batch(&schema, vec![1, 2, 3]))?;
@@ -396,7 +396,7 @@ mod tests {
             1,
             0,
             schema.clone(),
-            CompressionType::LZ4_FRAME,
+            Some(CompressionType::LZ4_FRAME),
         )?;
 
         manager.spill(0, &create_test_batch(&schema, vec![1, 2]))?;
diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs b/ballista/core/src/execution_plans/sort_shuffle/writer.rs
index 81b201680f..eb9b43896d 100644
--- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs
+++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs
@@ -21,14 +21,15 @@
 //! per input partition, along with an index file mapping partition IDs to
 //! byte offsets.
 
-use std::any::Any;
+use std::fmt::Debug;
 use std::fs::File;
-use std::future::Future;
 use std::io::{BufWriter, Seek, Write};
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use std::time::Instant;
+use tokio::sync::oneshot;
 
+use super::super::shuffle_writer::{result_schema, summaries_to_batch};
 use super::super::shuffle_writer_trait::ShuffleWriter;
 use super::buffer::BufferedBatches;
 use super::config::SortShuffleConfig;
@@ -36,15 +37,14 @@ use super::index::ShuffleIndex;
 use super::partitioned_batch_iterator::PartitionedBatchIterator;
 use super::spill::SpillManager;
 use crate::JobId;
-use crate::execution_plans::create_shuffle_path;
+use crate::extension::SessionConfigExt;
 use crate::serde::protobuf::ShuffleWritePartition;
 
-use datafusion::arrow::array::{
-    ArrayBuilder, ArrayRef, StringBuilder, StructBuilder, UInt32Builder, UInt64Builder,
-};
-use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use crate::utils::create_write_options;
+use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::arrow::error::ArrowError;
-use datafusion::arrow::ipc::writer::{IpcWriteOptions, StreamWriter};
+use datafusion::arrow::ipc::CompressionType;
+use datafusion::arrow::ipc::writer::StreamWriter;
 use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::common::hash_utils::create_hashes;
 use datafusion::error::{DataFusionError, Result};
@@ -52,7 +52,6 @@ use datafusion::execution::context::TaskContext;
 use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};
 use datafusion::physical_expr::PhysicalExpr;
 use datafusion::physical_expr_common::utils::evaluate_expressions_to_arrays;
-use datafusion::physical_plan::memory::MemoryStream;
 use datafusion::physical_plan::metrics::{
     self, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet,
 };
@@ -62,18 +61,121 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
     SendableRecordBatchStream, Statistics, displayable,
 };
-use futures::{StreamExt, TryFutureExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use log::debug;
 
-use crate::serde::scheduler::PartitionStats;
-
 /// Result of finalizing shuffle output: (data_path, index_path, partition_write_stats)
 /// where partition_write_stats is (partition_id, num_batches, num_rows, num_bytes)
 type FinalizeResult = (PathBuf, PathBuf, Vec<(usize, u64, u64, u64)>);
 
+/// Coordinator handoff state — same shape as `ShuffleWriterExec`. Each output
+/// hash partition (0..K) gets one `oneshot::Receiver`; the first `execute(N)`
+/// call spawns the coordinator, subsequent calls take their receiver and
+/// await the summaries for their output partition.
+struct WriterState {
+    initialized: bool,
+    handoffs: Vec<Option<oneshot::Receiver<Result<Vec<ShuffleWritePartition>>>>>,
+}
+
+impl Debug for WriterState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("WriterState")
+            .field("initialized", &self.initialized)
+            .field("handoffs", &self.handoffs.len())
+            .finish()
+    }
+}
+
 /// Sort-based shuffle writer that produces a single consolidated output file
 /// per input partition with an index file for partition offsets.
-#[derive(Debug, Clone)]
+///
+/// Under the multi-partition-task model each task's restricted plan carries
+/// input partitions 0..N_local, and different tasks in the same stage share
+/// those local indices. To keep shuffle files unique across tasks the writer
+/// stamps output paths with the task's `global_output_partition_ids` (bound
+/// via [`Self::with_global_output_partition_ids`]) — position `i` of the
+/// local plan maps to global id `global_output_partition_ids[i]`, which is
+/// what gets embedded in the file path.
+///
+/// # Threading
+///
+/// One Ballista task holds one `SortShuffleWriterExec`. The first
+/// `execute(k)` call from the executor initializes K oneshot handoffs (one
+/// per output partition) and spawns `run_coordinator`, which spawns M
+/// concurrent tokio tasks — one per local input partition in the task's
+/// slice. Each input writer pulls `child.execute(i)`, hash-buckets rows
+/// into K sub-slices, sorts and spills, and produces one indexed data file.
+/// The coordinator collects M×K per-bucket summaries and re-buckets them
+/// by output partition so each of the K oneshots receives the M summaries
+/// for its output. Each `execute(k)` returns a stream that awaits its
+/// oneshot and emits a single metadata batch pointing into the M files.
+///
+/// M concurrent per-input producers is the same fan-out shape as
+/// DataFusion's `RepartitionExec` (see `pull_from_input` in
+/// `datafusion/physical-plan/src/repartition/mod.rs`): N producers, each
+/// running `plan.execute(i)`, fanning to K sinks. Here the K sinks are
+/// bucketed sub-slices in one indexed file per input instead of K mpsc
+/// senders per producer.
+///
+/// Data flows bottom → top (child produces, executor consumes), matching
+/// DataFusion's diagram convention for `RepartitionExec`. Arrows point up
+/// with the data. **M and K are independent**: M = the task's slice of the
+/// child's partitions, K = the hash-partition target from the planner.
+/// Example: M=2, K=3.
+///
+/// ```text
+///                     K=3 output partitions (pulled by executor)
+///
+///        writer.execute(0)  writer.execute(1)  writer.execute(2)
+///                ▲                 ▲                 ▲
+///                │                 │                 │
+///           ┌──────────┐      ┌──────────┐      ┌──────────┐
+///           │ oneshot  │      │ oneshot  │      │ oneshot  │
+///           │ (out=0)  │      │ (out=1)  │      │ (out=2)  │
+///           └──────────┘      └──────────┘      └──────────┘
+///                ▲                 ▲                 ▲
+///                │       each oneshot carries a summary
+///                │       pointing at bucket-k slices in
+///                │       ALL M data files
+///                └─────────────────┼─────────────────┘
+///                                  │  re-bucket M×K per-bucket
+///                                  │  summaries by output part.
+///                         ┌─────────────────┐
+///                         │ run_coordinator │
+///                         └─────────────────┘
+///                                  ▲
+///                    ┌─────────────┴─────────────┐
+///                    │ K per-bucket summaries    │ K per-bucket summaries
+///                    │                           │
+///           ┌────────────────┐           ┌────────────────┐
+///           │  input writer  │           │  input writer  │
+///           │      (0)       │           │      (1)       │
+///           │ sort by hash   │           │ sort by hash   │
+///           │ bucket, spill  │           │ bucket, spill  │
+///           │ if needed →    │           │ if needed →    │
+///           │ ONE data file  │           │ ONE data file  │
+///           │ + ONE index    │           │ + ONE index    │
+///           └────────────────┘           └────────────────┘
+///                    ▲                           ▲
+///                    │                           │
+///            child.execute(0)            child.execute(1)
+///
+///                     M=2 input partitions
+///                  (task's slice of child plan)
+/// ```
+///
+/// File layout — one pair per input writer, K buckets concatenated in one
+/// file:
+///
+/// ```text
+///           data.arrow:        [ bucket 0 ][ bucket 1 ][ bucket 2 ]
+///           data.arrow.index:  { 0 → off0, 1 → off1, 2 → off2 }
+/// ```
+///
+/// A reader for output partition k opens each of the M `data.arrow` files,
+/// seeks to `index[k]`, and reads bucket-k's slice — so K downstream
+/// readers × M files, driven by the summaries the oneshots hand back.
+#[derive(Debug)]
 pub struct SortShuffleWriterExec {
     /// Unique ID for the job (query) that this stage is a part of
     job_id: JobId,
@@ -87,10 +189,45 @@ pub struct SortShuffleWriterExec {
     shuffle_output_partitioning: Partitioning,
     /// Sort shuffle configuration
     config: SortShuffleConfig,
+    /// Task id (the task's append-order slot in
+    /// `RunningStage.task_infos`). Stamped by the executor's
+    /// `create_query_stage_exec`; defaults to 0 for plans that arrive from
+    /// `try_new` directly (proto decode before executor stamping, or
+    /// tests). Kept for symmetry with `ShuffleWriterExec`.
+    task_id: usize,
+    /// Global input-partition ids this task's restricted plan covers, in
+    /// slice order. Position `i` of the local plan maps to
+    /// `global_output_partition_ids[i]` globally — used in file paths and
+    /// reported summaries so files from different tasks in the same stage
+    /// don't collide (task slices are disjoint by construction).
+    global_output_partition_ids: Vec<usize>,
     /// Execution metrics
     metrics: ExecutionPlanMetricsSet,
     /// Plan properties
     properties: Arc<PlanProperties>,
+    /// Shared coordinator handoff state. Clones share the same Arc so a
+    /// clone produced by `with_new_children` (or by the coordinator when
+    /// it spawns per-input-partition write tasks) participates in the
+    /// same coordinator.
+    state: Arc<Mutex<WriterState>>,
+}
+
+impl Clone for SortShuffleWriterExec {
+    fn clone(&self) -> Self {
+        Self {
+            job_id: self.job_id.clone(),
+            stage_id: self.stage_id,
+            plan: self.plan.clone(),
+            work_dir: self.work_dir.clone(),
+            shuffle_output_partitioning: self.shuffle_output_partitioning.clone(),
+            config: self.config.clone(),
+            task_id: self.task_id,
+            global_output_partition_ids: self.global_output_partition_ids.clone(),
+            metrics: self.metrics.clone(),
+            properties: self.properties.clone(),
+            state: self.state.clone(),
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -156,6 +293,14 @@ impl SortShuffleWriterExec {
             datafusion::physical_plan::execution_plan::Boundedness::Bounded,
         ));
 
+        // Default identity slice — one global id per local input partition.
+        // The executor's `create_query_stage_exec` stamps the real slice via
+        // `with_global_output_partition_ids` before dispatch.
+        let child_partition_count =
+            plan.properties().output_partitioning().partition_count();
+        let default_partition_slice: Vec<usize> = (0..child_partition_count).collect();
+        let output_partition_count = properties.output_partitioning().partition_count();
+
         Ok(Self {
             job_id,
             stage_id,
@@ -163,11 +308,48 @@ impl SortShuffleWriterExec {
             work_dir,
             shuffle_output_partitioning,
             config,
+            task_id: 0,
+            global_output_partition_ids: default_partition_slice,
             metrics: ExecutionPlanMetricsSet::new(),
             properties,
+            state: Arc::new(Mutex::new(WriterState {
+                initialized: false,
+                handoffs: (0..output_partition_count).map(|_| None).collect(),
+            })),
         })
     }
 
+    /// Bind this writer to a specific task_id. Called by the executor
+    /// after decoding the plan so writer instances from different tasks in
+    /// the same stage carry their identity for logging / debugging.
+    pub fn with_task_id(mut self, task_id: usize) -> Self {
+        self.task_id = task_id;
+        self
+    }
+
+    /// Task id (append-order slot within the stage) this writer instance
+    /// is bound to.
+    pub fn task_id(&self) -> usize {
+        self.task_id
+    }
+
+    /// Bind this writer to the task's assigned global partition slice.
+    /// Position `i` of the local plan corresponds to `slice[i]` globally,
+    /// which is used to name shuffle files so different tasks in the same
+    /// stage don't collide on file paths.
+    pub fn with_global_output_partition_ids(
+        mut self,
+        global_output_partition_ids: Vec<usize>,
+    ) -> Self {
+        self.global_output_partition_ids = global_output_partition_ids;
+        self
+    }
+
+    /// Global partition ids this task's restricted plan covers.
+    pub fn global_output_partition_ids(&self) -> &[usize] {
+        &self.global_output_partition_ids
+    }
+
     /// Get the Job ID for this query stage
     pub fn job_id(&self) -> &JobId {
         &self.job_id
@@ -197,6 +379,14 @@ impl SortShuffleWriterExec {
     }
 
     /// Execute the sort-based shuffle write for a single input partition.
+    ///
+    /// `input_partition` is a **local** index into this task's restricted
+    /// plan (0..N_local). All on-disk paths and the returned `file_id`
+    /// bit-pack `(task_id, input_partition)` into one u64 so files from
+    /// different tasks (and different local inputs within a task) never
+    /// collide. `global_output_partition_ids` is not used for file naming
+    /// here — for `SortShuffleWriterExec` it holds the K-space (0..K),
+    /// which is intrinsic to hash shuffle and not per-file-unique.
     pub fn execute_shuffle_write(
         self,
         input_partition: usize,
@@ -209,11 +399,19 @@ impl SortShuffleWriterExec {
         let job_id = self.job_id.clone();
         let stage_id = self.stage_id;
         let partitioning = self.shuffle_output_partitioning.clone();
+        // Pack (task_id, input_partition) into a single u64 so file paths
+        // are globally unique. Reader also uses this opaquely — path is
+        // `.../{file_id}/data.arrow` and the reader gets `file_id` back
+        // through `ShuffleWritePartition`.
+        let file_id: u64 = ((self.task_id as u64) << 32) | (input_partition as u64);
+        let file_id_usize = file_id as usize;
 
         async move {
             let now = Instant::now();
             let mut stream = plan.execute(input_partition, context.clone())?;
             let schema = stream.schema();
+            let ballista_config = Arc::new(context.session_config().ballista_config());
+            let compression_type = ballista_config.shuffle_compression_codec()?;
 
             let Partitioning::Hash(exprs, num_output_partitions) = partitioning else {
                 return Err(DataFusionError::Internal(
@@ -225,9 +423,9 @@ impl SortShuffleWriterExec {
                 &work_dir,
                 &job_id,
                 stage_id,
-                input_partition,
+                file_id_usize,
                 schema.clone(),
-                config.compression,
+                compression_type,
             )
             .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
 
@@ -269,13 +467,16 @@ impl SortShuffleWriterExec {
                 growth += after.saturating_sub(before);
 
                 // Mirror the growth in the runtime pool reservation so the pool
-                // sees this writer's memory usage. Best-effort: if the pool is
-                // bounded and rejects the grow, that's fine — the absolute
-                // counter below still triggers a spill.
-                let _ = reservation.try_grow(growth);
+                // sees this writer's memory usage. A rejected grow means the
+                // pool is under pressure and has *not* accounted for the batch
+                // just buffered, so spill instead of holding memory the pool
+                // believes is free. Spilling flushes that batch to disk and
+                // frees the reservation, which is the response a spillable
+                // consumer owes the pool.
+                let pool_rejected_growth = reservation.try_grow(growth).is_err();
                 buffered_bytes = buffered_bytes.saturating_add(growth);
 
-                if buffered_bytes >= memory_limit {
+                if pool_rejected_growth || buffered_bytes >= memory_limit {
                     let spill_timer = metrics.spill_time.timer();
                     let (event_batches, event_bytes) = spill_all_partitions(
                         &mut buffered,
@@ -314,11 +515,12 @@ impl SortShuffleWriterExec {
                 &work_dir,
                 &job_id,
                 stage_id,
-                input_partition,
+                file_id_usize,
                 &mut buffered,
                 &mut spill_manager,
                 &schema,
                 &config,
+                compression_type,
             )?;
             timer.done();
 
@@ -363,7 +565,7 @@ impl SortShuffleWriterExec {
                         num_batches,
                         num_rows,
                         num_bytes,
-                        file_id: Some(input_partition as u64),
+                        file_id: Some(file_id),
                         is_sort_shuffle: true,
                     });
                 }
@@ -426,6 +628,7 @@ fn finalize_output(
     spill_manager: &mut SpillManager,
     schema: &SchemaRef,
     config: &SortShuffleConfig,
+    compression_type: Option<CompressionType>,
 ) -> Result<FinalizeResult> {
     let num_partitions = buffered.num_partitions();
     let mut index = ShuffleIndex::new(num_partitions);
@@ -445,8 +648,7 @@ fn finalize_output(
     let file = File::create(&data_path)?;
     let mut output = BufWriter::new(file);
 
-    let opts =
-        IpcWriteOptions::default().try_with_compression(Some(config.compression))?;
+    let opts = create_write_options(compression_type)?;
 
     // Leading schema-header stream (schema message + EOS, no batches) so the
     // reader can recover the schema even when the requested partition is empty.
@@ -539,10 +741,6 @@ impl ExecutionPlan for SortShuffleWriterExec {
         "SortShuffleWriterExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.plan.schema()
     }
@@ -581,6 +779,16 @@ impl ExecutionPlan for SortShuffleWriterExec {
         }
     }
 
+    /// Return the stream for output partition `partition`.
+    ///
+    /// The first call locks the shared state, initializes K oneshot channels
+    /// (one per output partition), and spawns a coordinator that drives the
+    /// per-input-partition writes for all M local input partitions of the
+    /// task's restricted plan. Each input write emits K per-bucket
+    /// summaries pointing at that input's on-disk file; the coordinator
+    /// re-buckets them by output partition and sends each bucket's
+    /// summaries to the matching oneshot. Subsequent calls take their
+    /// receiver and return a stream that awaits it.
     fn execute(
         &self,
         partition: usize,
@@ -588,74 +796,68 @@ impl ExecutionPlan for SortShuffleWriterExec {
     ) -> Result<SendableRecordBatchStream> {
         let schema = result_schema();
 
-        let schema_captured = schema.clone();
-        let job_id = self.job_id.clone();
-        let work_dir = self.work_dir.to_string();
-        let stage_id = self.stage_id;
-        let fut_stream = self
-            .clone()
-            .execute_shuffle_write(partition, context)
-            .and_then(move |part_loc| async move {
-                // Build metadata result batch
-                let num_writers = part_loc.len();
-                let mut partition_builder = UInt32Builder::with_capacity(num_writers);
-                let mut path_builder =
-                    StringBuilder::with_capacity(num_writers, num_writers * 100);
-                let mut num_rows_builder = UInt64Builder::with_capacity(num_writers);
-                let mut num_batches_builder = UInt64Builder::with_capacity(num_writers);
-                let mut num_bytes_builder = UInt64Builder::with_capacity(num_writers);
-
-                for loc in &part_loc {
-                    path_builder.append_value(
-                        create_shuffle_path(
-                            &work_dir,
-                            &job_id,
-                            stage_id,
-                            partition,
-                            loc.file_id,
-                            true,
-                        )?
-                        .to_string_lossy(),
-                    );
-
-                    partition_builder.append_value(loc.partition_id as u32);
-                    num_rows_builder.append_value(loc.num_rows);
-                    num_batches_builder.append_value(loc.num_batches);
-                    num_bytes_builder.append_value(loc.num_bytes);
-                }
-
-                // Build arrays
-                let partition_num: ArrayRef = Arc::new(partition_builder.finish());
-                let path: ArrayRef = Arc::new(path_builder.finish());
-                let field_builders: Vec<Box<dyn ArrayBuilder>> = vec![
-                    Box::new(num_rows_builder),
-                    Box::new(num_batches_builder),
-                    Box::new(num_bytes_builder),
-                ];
-                let mut stats_builder = StructBuilder::new(
-                    PartitionStats::default().arrow_struct_fields(),
-                    field_builders,
-                );
-                for _ in 0..num_writers {
-                    stats_builder.append(true);
-                }
-                let stats = Arc::new(stats_builder.finish());
-
-                // Build result batch containing metadata
-                let batch = RecordBatch::try_new(
-                    schema_captured.clone(),
-                    vec![partition_num, path, stats],
-                )?;
+        let mut state = self.state.lock().map_err(|_| {
+            DataFusionError::Internal(
+                "SortShuffleWriterExec state mutex poisoned".to_owned(),
+            )
+        })?;
+
+        if !state.initialized {
+            state.initialized = true;
+            let k = state.handoffs.len();
+            let mut senders: Vec<oneshot::Sender<Result<Vec<ShuffleWritePartition>>>> =
+                Vec::with_capacity(k);
+            for slot in state.handoffs.iter_mut() {
+                let (tx, rx) = oneshot::channel();
+                senders.push(tx);
+                *slot = Some(rx);
+            }
+            let writer = self.clone();
+            let ctx = context.clone();
+            tokio::spawn(async move {
+                run_coordinator(writer, ctx, senders).await;
+            });
+        }
 
-                debug!("SORT SHUFFLE RESULTS METADATA:\n{batch:?}");
+        let rx = state
+            .handoffs
+            .get_mut(partition)
+            .and_then(Option::take)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "SortShuffleWriterExec: execute({partition}) called twice or out of range (have {} partitions)",
+                    state.handoffs.len()
+                ))
+            })?;
+        drop(state);
 
-                MemoryStream::try_new(vec![batch], schema_captured, None)
-            })
-            .map_err(|e| ArrowError::ExternalError(Box::new(e)));
+        let work_dir = self.work_dir.clone();
+        let job_id = self.job_id.clone();
+        let stage_id = self.stage_id;
+        let schema_captured = schema.clone();
 
         Ok(Box::pin(RecordBatchStreamAdapter::new(
             schema,
-            futures::stream::once(fut_stream).try_flatten(),
+            futures::stream::once(async move {
+                let summaries = rx
+                    .await
+                    .map_err(|_| {
+                        DataFusionError::Internal(
+                            "SortShuffleWriterExec coordinator dropped without sending"
+                                .to_owned(),
+                        )
+                    })?
+                    .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
+                summaries_to_batch(
+                    summaries,
+                    schema_captured,
+                    &work_dir,
+                    &job_id,
+                    stage_id,
+                )
+                .map_err(|e| ArrowError::ExternalError(Box::new(e)))
+            })
+            .try_flatten(),
         )))
     }
 
@@ -663,7 +865,7 @@ impl ExecutionPlan for SortShuffleWriterExec {
         Some(self.metrics.clone_inner())
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         self.plan.partition_statistics(partition)
     }
 }
@@ -693,13 +895,81 @@ impl ShuffleWriter for SortShuffleWriterExec {
     }
 }
 
-fn result_schema() -> SchemaRef {
-    let stats = PartitionStats::default();
-    Arc::new(Schema::new(vec![
-        Field::new("partition", DataType::UInt32, false),
-        Field::new("path", DataType::Utf8, false),
-        stats.arrow_struct_repr(),
-    ]))
+/// Drives per-input-partition writes for the task's restricted plan, then
+/// re-buckets summaries by output partition and sends each bucket's
+/// summaries to the matching oneshot. Runs the M input writes concurrently
+/// (one tokio task each) so shuffle-write parallelism matches what main's
+/// design achieved by having DataFusion drive `execute(0..M)` in parallel.
+///
+/// On failure, sends the error to every sender so no waiting stream hangs.
+async fn run_coordinator(
+    writer: SortShuffleWriterExec,
+    ctx: Arc<TaskContext>,
+    senders: Vec<oneshot::Sender<Result<Vec<ShuffleWritePartition>>>>,
+) {
+    let k = senders.len();
+    let mut senders: Vec<Option<oneshot::Sender<Result<Vec<ShuffleWritePartition>>>>> =
+        senders.into_iter().map(Some).collect();
+
+    let num_input_partitions = writer
+        .plan
+        .properties()
+        .output_partitioning()
+        .partition_count();
+
+    // Spawn per-input-partition write tasks concurrently — this matches the
+    // parallelism main's design achieved by having DataFusion drive
+    // `execute(0..M)` concurrently.
+    let handles: Vec<_> = (0..num_input_partitions)
+        .map(|input_partition| {
+            let w = writer.clone();
+            let c = ctx.clone();
+            tokio::spawn(async move { w.execute_shuffle_write(input_partition, c).await })
+        })
+        .collect();
+
+    let mut grouped: Vec<Vec<ShuffleWritePartition>> =
+        (0..k).map(|_| Vec::new()).collect();
+    let mut first_error: Option<String> = None;
+    for handle in handles {
+        match handle.await {
+            Ok(Ok(summaries)) => {
+                for summary in summaries {
+                    let idx = summary.partition_id as usize;
+                    if idx < k {
+                        grouped[idx].push(summary);
+                    }
+                }
+            }
+            Ok(Err(e)) => {
+                first_error.get_or_insert_with(|| format!("{e:?}"));
+            }
+            Err(join_err) => {
+                first_error
+                    .get_or_insert_with(|| format!("write task panicked: {join_err}"));
+            }
+        }
+    }
+
+    if let Some(msg) = first_error {
+        for (i, slot) in senders.iter_mut().enumerate() {
+            if let Some(sender) = slot.take() {
+                let err_msg = if i == 0 {
+                    msg.clone()
+                } else {
+                    format!("sort shuffle writer failed: {msg}")
+                };
+                let _ = sender.send(Err(DataFusionError::Execution(err_msg)));
+            }
+        }
+        return;
+    }
+
+    for (slot, bucket) in senders.iter_mut().zip(grouped) {
+        if let Some(sender) = slot.take() {
+            let _ = sender.send(Ok(bucket));
+        }
+    }
 }
 
 impl std::fmt::Display for SortShuffleWriterExec {
@@ -753,6 +1023,8 @@ fn compute_partition_indices(
 mod tests {
     use super::*;
     use datafusion::arrow::array::{StringArray, UInt32Array};
+    use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::arrow::record_batch::RecordBatch;
     use datafusion::datasource::memory::MemorySourceConfig;
     use datafusion::datasource::source::DataSourceExec;
     use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
@@ -857,7 +1129,9 @@ mod tests {
 
         assert_eq!(batches.len(), 1);
         let batch = &batches[0];
-        assert_eq!(batch.num_columns(), 3);
+        // partition | path | file_id | stats — shared with `ShuffleWriterExec`
+        // so the reader path can consume both writers uniformly.
+        assert_eq!(batch.num_columns(), 4);
 
         // Verify output files exist
         let output_dir = work_dir.path().join("job1").join("1").join("0");
@@ -871,7 +1145,8 @@ mod tests {
     /// `rows_per_batch` rows each with schema `(k: Int64, v: Int64)`, writes
     /// them through `SortShuffleWriterExec` into `num_partitions` output
     /// partitions with a per-task buffered-bytes budget of
-    /// `sort_shuffle_memory_limit_bytes`, reads every partition back via
+    /// `sort_shuffle_memory_limit_bytes` against a `FairSpillPool` of
+    /// `pool_bytes`, reads every partition back via
     /// `stream_sort_shuffle_partition`, and asserts:
     ///   - Every input key `0..total_rows` is present exactly once in the union
     ///     of partition outputs.
@@ -882,6 +1157,7 @@ mod tests {
         rows_per_batch: usize,
         num_partitions: usize,
         sort_shuffle_memory_limit_bytes: usize,
+        pool_bytes: usize,
         expect_spills: bool,
     ) -> Result<()> {
         use super::super::reader::stream_sort_shuffle_partition;
@@ -919,13 +1195,9 @@ mod tests {
         let input: Arc<dyn ExecutionPlan> =
             Arc::new(DataSourceExec::new(memory_data_source));
 
-        // The runtime pool is sized generously: spill decisions are now driven
-        // by `SortShuffleConfig::memory_limit_per_task_bytes`, but we keep a
-        // bounded pool so the post-test `pool.reserved() == 0` assertion still
-        // checks that the writer releases its `MemoryReservation`.
         let runtime_env = Arc::new(
             RuntimeEnvBuilder::new()
-                .with_memory_pool(Arc::new(FairSpillPool::new(64 * 1024 * 1024)))
+                .with_memory_pool(Arc::new(FairSpillPool::new(pool_bytes)))
                 .build()?,
         );
         let session_ctx = SessionContext::new_with_config_rt(
@@ -1011,15 +1283,35 @@ mod tests {
         Ok(())
     }
 
+    // The writer has two independent spill triggers, and these tests arm exactly
+    // one at a time so a failure names its cause:
+    //
+    //   1. the shared `MemoryPool` rejecting a `try_grow` — sized by `pool_bytes`
+    //   2. the private per-task budget `memory_limit_per_task_bytes` being
+    //      reached — sized by `sort_shuffle_memory_limit_bytes`
+    //
+    // The `*_DISARMED` constants below are the "this trigger is not what is
+    // under test" settings. A test payload batch is 8192 rows x 2 Int64 columns,
+    // i.e. ~128 KiB.
+
+    /// Pool far larger than any test payload, so `try_grow` always succeeds and
+    /// trigger 1 never fires. Still bounded, so the `pool.reserved() == 0`
+    /// assertion after the run stays meaningful.
+    const POOL_NEVER_REJECTS: usize = 64 * 1024 * 1024;
+
+    /// Per-task budget far above any test payload, so trigger 2 never fires.
+    const BUDGET_NEVER_REACHED: usize = 1024 * 1024 * 1024;
+
     #[tokio::test]
-    async fn spills_under_memory_pressure_and_round_trips() -> Result<()> {
-        // 10 batches of 8192 rows each. With a 512 KiB per-task buffer budget,
-        // this forces spilling.
+    async fn spills_when_per_task_budget_reached_and_round_trips() -> Result<()> {
+        // Trigger 2 only: 10 batches (~1.28 MiB) against a 512 KiB per-task
+        // budget forces spilling, while the pool stays out of the picture.
         run_round_trip(
             /* num_batches */ 10,
             /* rows_per_batch */ 8192,
             /* num_partitions */ 4,
             /* sort_shuffle_memory_limit_bytes */ 512 * 1024,
+            /* pool_bytes */ POOL_NEVER_REJECTS,
             /* expect_spills */ true,
         )
         .await
@@ -1027,16 +1319,17 @@ mod tests {
 
     #[tokio::test]
     async fn multi_spill_round_trips() -> Result<()> {
-        // Larger total payload + tighter buffer budget than the baseline test,
-        // so we expect multiple spill events per partition. The byte-copy
-        // finalize path should produce a partition section that is
-        // `spill_stream || in_memory_stream` (two concatenated IPC streams),
-        // and the reader must yield every input row regardless.
+        // Trigger 2 only, with a larger total payload and a tighter budget than
+        // the baseline test, so we expect multiple spill events per partition.
+        // The byte-copy finalize path should produce a partition section that is
+        // `spill_stream || in_memory_stream` (two concatenated IPC streams), and
+        // the reader must yield every input row regardless.
         run_round_trip(
             /* num_batches */ 20,
             /* rows_per_batch */ 8192,
             /* num_partitions */ 2,
             /* sort_shuffle_memory_limit_bytes */ 256 * 1024,
+            /* pool_bytes */ POOL_NEVER_REJECTS,
             /* expect_spills */ true,
         )
         .await
@@ -1044,19 +1337,39 @@ mod tests {
 
     #[tokio::test]
     async fn in_memory_only_round_trips() -> Result<()> {
-        // Generous per-task buffer budget so no partition spills. Validates
-        // the path where each partition section is exactly one in-memory IPC
-        // stream.
+        // Neither trigger armed, so nothing spills. Validates the path where
+        // each partition section is exactly one in-memory IPC stream.
         run_round_trip(
             /* num_batches */ 4,
             /* rows_per_batch */ 1024,
             /* num_partitions */ 4,
-            /* sort_shuffle_memory_limit_bytes */ 64 * 1024 * 1024,
+            /* sort_shuffle_memory_limit_bytes */ BUDGET_NEVER_REACHED,
+            /* pool_bytes */ POOL_NEVER_REJECTS,
             /* expect_spills */ false,
         )
         .await
     }
 
+    #[tokio::test]
+    async fn spills_when_memory_pool_rejects_growth() -> Result<()> {
+        // Trigger 1 only: a 64 KiB pool cannot admit even one ~128 KiB batch, so
+        // every `try_grow` is rejected, while the per-task budget can never fire.
+        // Pool rejection is therefore the only signal that can produce a spill.
+        //
+        // A rejected grow means the pool has not accounted for the batch just
+        // buffered. The writer must spill in response rather than keep buffering
+        // memory the pool believes is free.
+        run_round_trip(
+            /* num_batches */ 8,
+            /* rows_per_batch */ 8192,
+            /* num_partitions */ 2,
+            /* sort_shuffle_memory_limit_bytes */ BUDGET_NEVER_REACHED,
+            /* pool_bytes */ 64 * 1024,
+            /* expect_spills */ true,
+        )
+        .await
+    }
+
     #[tokio::test]
     async fn empty_partitions_round_trip() -> Result<()> {
         use super::super::reader::stream_sort_shuffle_partition;
@@ -1164,7 +1477,8 @@ mod tests {
 
         // Reference: DataFusion's BatchPartitioner::new_hash_partitioner
         let mut ref_partitioner =
-            BatchPartitioner::new_hash_partitioner(exprs.clone(), 4, Time::default());
+            BatchPartitioner::new_hash_partitioner(exprs.clone(), 4, Time::default())
+                .unwrap();
         let mut ref_assignments = [usize::MAX; 10];
         ref_partitioner
             .partition(batch.clone(), |partition, sub_batch| {
diff --git a/ballista/core/src/execution_plans/unresolved_shuffle.rs b/ballista/core/src/execution_plans/unresolved_shuffle.rs
index 3f3567b6a9..f544f0b0a4 100644
--- a/ballista/core/src/execution_plans/unresolved_shuffle.rs
+++ b/ballista/core/src/execution_plans/unresolved_shuffle.rs
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::any::Any;
 use std::sync::Arc;
 
 use datafusion::arrow::datatypes::SchemaRef;
@@ -156,13 +155,14 @@ impl DisplayAs for UnresolvedShuffleExec {
                 if self.broadcast {
                     write!(
                         f,
-                        "UnresolvedShuffleExec: broadcast=true, upstream_partitions: {}",
-                        self.upstream_partition_count,
+                        "UnresolvedShuffleExec: stage={}, broadcast=true, upstream_partitions: {}",
+                        self.stage_id, self.upstream_partition_count,
                     )
                 } else {
                     write!(
                         f,
-                        "UnresolvedShuffleExec: partitioning: {}",
+                        "UnresolvedShuffleExec: stage={}, partitioning: {}",
+                        self.stage_id,
                         self.properties().output_partitioning()
                     )?;
                     if let Some(c) = &self.coalesce {
@@ -192,10 +192,6 @@ impl ExecutionPlan for UnresolvedShuffleExec {
         "UnresolvedShuffleExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -271,4 +267,19 @@ mod tests {
             "expected ', coalesce: 3 of 8' annotation; got: {s}"
         );
     }
+
+    #[test]
+    fn display_includes_stage_id() {
+        let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+        let exec =
+            UnresolvedShuffleExec::new(3, schema, Partitioning::UnknownPartitioning(4));
+        let s = format!(
+            "{}",
+            datafusion::physical_plan::displayable(&exec).indent(false)
+        );
+        assert!(
+            s.contains("stage=3"),
+            "expected stage id in display, got: {s}"
+        );
+    }
 }
diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs
index a9da57c786..b0a58289b9 100644
--- a/ballista/core/src/extension.rs
+++ b/ballista/core/src/extension.rs
@@ -16,13 +16,13 @@
 // under the License.
 
 use crate::config::{
-    BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES, BALLISTA_CLIENT_GRPC_MAX_MESSAGE_SIZE,
-    BALLISTA_CLIENT_USE_TLS, BALLISTA_COALESCE_ENABLED,
-    BALLISTA_COALESCE_MERGED_PARTITION_FACTOR, BALLISTA_COALESCE_SMALL_PARTITION_FACTOR,
-    BALLISTA_COALESCE_TARGET_PARTITION_BYTES, BALLISTA_JOB_NAME,
-    BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ, BALLISTA_SHUFFLE_READER_MAX_REQUESTS,
-    BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT, BALLISTA_STANDALONE_PARALLELISM,
-    BallistaConfig,
+    BALLISTA_BROADCAST_JOIN_THRESHOLD_BYTES, BALLISTA_BROADCAST_JOIN_THRESHOLD_ROWS,
+    BALLISTA_CLIENT_GRPC_MAX_MESSAGE_SIZE, BALLISTA_CLIENT_USE_TLS,
+    BALLISTA_COALESCE_ENABLED, BALLISTA_COALESCE_MERGED_PARTITION_FACTOR,
+    BALLISTA_COALESCE_SMALL_PARTITION_FACTOR, BALLISTA_COALESCE_TARGET_PARTITION_BYTES,
+    BALLISTA_JOB_NAME, BALLISTA_SHUFFLE_READER_FORCE_REMOTE_READ,
+    BALLISTA_SHUFFLE_READER_MAX_REQUESTS, BALLISTA_SHUFFLE_READER_REMOTE_PREFER_FLIGHT,
+    BALLISTA_STANDALONE_PARALLELISM, BallistaConfig,
 };
 use crate::planner::BallistaQueryPlanner;
 use crate::serde::protobuf::KeyValuePair;
@@ -192,6 +192,18 @@ pub trait SessionConfigExt {
     fn with_ballista_broadcast_join_threshold_bytes(self, threshold_bytes: usize)
     -> Self;
 
+    /// retrieves the row-count threshold below which a hash join's smaller side
+    /// is promoted to `CollectLeft` and lowered via the broadcast pattern, used
+    /// as a fallback when byte-size statistics are unavailable. `0` disables
+    /// promotion via the row-count path.
+    fn ballista_broadcast_join_threshold_rows(&self) -> usize;
+
+    /// Sets the row-count threshold below which a hash join's smaller side is
+    /// promoted to `CollectLeft` and lowered via the broadcast pattern, used as
+    /// a fallback when byte-size statistics are unavailable. Setting `0`
+    /// disables promotion via the row-count path.
+    fn with_ballista_broadcast_join_threshold_rows(self, threshold_rows: usize) -> Self;
+
     /// retrieves grpc client max message size
     fn ballista_grpc_client_max_message_size(&self) -> usize;
 
@@ -355,15 +367,22 @@ impl SessionConfigExt for SessionConfig {
     }
 
     fn upgrade_for_ballista(self) -> SessionConfig {
-        // if ballista config is not provided
-        // one is created and session state is updated
+        // Ballista's opinionated DataFusion defaults are applied once, when a
+        // plain config is first upgraded. The `BallistaConfig` extension marks a
+        // config that has already been through this path; re-applying the
+        // defaults on such a config would overwrite any values the user set
+        // afterwards (e.g. via `-c`/`SET`), so they are left untouched.
+        let already_upgraded =
+            self.options().extensions.get::<BallistaConfig>().is_some();
+
         let ballista_config = self.ballista_config();
+        let config = self.with_option_extension(ballista_config);
 
-        // session config has ballista config extension and
-        // default datafusion configuration is altered
-        // to fit ballista execution
-        self.with_option_extension(ballista_config)
-            .ballista_restricted_configuration()
+        if already_upgraded {
+            config
+        } else {
+            config.ballista_restricted_configuration()
+        }
     }
 
     fn ballista_config(&self) -> BallistaConfig {
@@ -477,6 +496,23 @@ impl SessionConfigExt for SessionConfig {
         }
     }
 
+    fn ballista_broadcast_join_threshold_rows(&self) -> usize {
+        self.options()
+            .extensions
+            .get::<BallistaConfig>()
+            .map(|c| c.broadcast_join_threshold_rows())
+            .unwrap_or_else(|| BallistaConfig::default().broadcast_join_threshold_rows())
+    }
+
+    fn with_ballista_broadcast_join_threshold_rows(self, threshold_rows: usize) -> Self {
+        if self.options().extensions.get::<BallistaConfig>().is_some() {
+            self.set_usize(BALLISTA_BROADCAST_JOIN_THRESHOLD_ROWS, threshold_rows)
+        } else {
+            self.with_option_extension(BallistaConfig::default())
+                .set_usize(BALLISTA_BROADCAST_JOIN_THRESHOLD_ROWS, threshold_rows)
+        }
+    }
+
     fn ballista_shuffle_reader_maximum_concurrent_requests(&self) -> usize {
         self.options()
             .extensions
@@ -738,6 +774,7 @@ impl SessionConfigHelperExt for SessionConfig {
     }
 
     fn ballista_restricted_configuration(self) -> Self {
+        let ballista_defaults = BallistaConfig::default();
         self
             // round robbin repartition does not work well with ballista.
             // this setting it will also be enforced by the scheduler
@@ -760,18 +797,19 @@ impl SessionConfigHelperExt for SessionConfig {
             // same like previous comment
             .set_bool("datafusion.sql_parser.map_string_types_to_utf8view", false)
             //
-            // As mentioned in https://github.com/apache/datafusion-ballista/issues/1055
-            // "Left/full outer join incorrect for CollectLeft / broadcast"
-            //
-            // In order to make correct results (decreasing performance) CollectLeft
-            // has been disabled until fixed
+            // A build side smaller than these thresholds is collected into a
+            // CollectLeft (broadcast) hash join rather than being repartitioned.
+            // The values mirror Ballista's own broadcast thresholds so a single
+            // set of `ballista.optimizer.broadcast_join_threshold_*` defaults
+            // drives both DataFusion's built-in JoinSelection (static planner)
+            // and Ballista's AQE join selection.
             .set_u64(
                 "datafusion.optimizer.hash_join_single_partition_threshold",
-                0,
+                ballista_defaults.broadcast_join_threshold_bytes() as u64,
             )
             .set_u64(
                 "datafusion.optimizer.hash_join_single_partition_threshold_rows",
-                0,
+                ballista_defaults.broadcast_join_threshold_rows() as u64,
             )
             //
             // DataFusion's hash join has no spill support, so each parallel
@@ -782,6 +820,35 @@ impl SessionConfigHelperExt for SessionConfig {
             //
             // See https://github.com/apache/datafusion-ballista/issues/1648
             .set_bool("datafusion.optimizer.prefer_hash_join", false)
+            //
+            // DataFusion 54 plans uncorrelated scalar subqueries as a physical
+            // `ScalarSubqueryExec` wrapping a `ScalarSubqueryExpr` that reads an
+            // in-process shared results container. That container cannot cross
+            // process or stage boundaries, and `datafusion-proto` can only
+            // deserialize the expr inside its surrounding exec, so when Ballista
+            // splits a plan into stages the expr is serialized without its exec
+            // and the executor fails to decode it. Disabling this option makes
+            // the optimizer rewrite uncorrelated scalar subqueries to joins,
+            // which Ballista distributes correctly.
+            //
+            // See https://github.com/apache/datafusion-ballista/issues/1909
+            .set_bool(
+                "datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery",
+                false,
+            )
+            //
+            // DataFusion's dynamic filters are populated at runtime by an
+            // upstream operator (a hash join build side, a TopK heap, a partial
+            // aggregate) and read by a downstream scan within the same plan.
+            // Ballista splits a plan into stages at shuffle and broadcast
+            // boundaries that run as independent tasks, so when the producing
+            // operator and the consuming scan land in different stages the
+            // filter is never populated across the boundary and the scan blocks
+            // forever. Disable dynamic filter pushdown until Ballista can carry
+            // dynamic filters across stage boundaries.
+            //
+            // See https://github.com/apache/datafusion-ballista/issues/1375
+            .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", false)
     }
 }
 
@@ -1018,6 +1085,77 @@ mod test {
 
         assert!(!state.config().round_robin_repartition());
     }
+
+    // User overrides of Ballista's soft defaults must survive `upgrade_for_ballista`;
+    // re-applying the defaults would discard them. See #1901.
+    #[test]
+    fn should_preserve_user_overrides_on_upgrade() {
+        // Ballista defaults these to prefer_hash_join=false and the threshold to
+        // 10 MB. The overrides below differ from those defaults so the assertions
+        // prove the user's values survived `upgrade_for_ballista`.
+        let mut config = SessionConfig::new_with_ballista();
+        config
+            .options_mut()
+            .set("datafusion.optimizer.prefer_hash_join", "true")
+            .unwrap();
+        config
+            .options_mut()
+            .set(
+                "datafusion.optimizer.hash_join_single_partition_threshold",
+                "5242880",
+            )
+            .unwrap();
+
+        let upgraded = config.upgrade_for_ballista();
+
+        assert!(upgraded.options().optimizer.prefer_hash_join);
+        assert_eq!(
+            upgraded
+                .options()
+                .optimizer
+                .hash_join_single_partition_threshold,
+            5242880
+        );
+    }
+
+    // A plain (non-Ballista) config still receives Ballista's opinionated
+    // defaults when upgraded.
+    #[test]
+    fn should_apply_defaults_when_upgrading_plain_config() {
+        let config = SessionConfig::new().upgrade_for_ballista();
+
+        assert!(!config.options().optimizer.prefer_hash_join);
+        assert_eq!(
+            config
+                .options()
+                .optimizer
+                .hash_join_single_partition_threshold,
+            10 * 1024 * 1024
+        );
+        assert_eq!(
+            config
+                .options()
+                .optimizer
+                .hash_join_single_partition_threshold_rows,
+            1_000_000
+        );
+    }
+
+    // Uncorrelated scalar subqueries must be rewritten to joins rather than
+    // planned as a physical `ScalarSubqueryExec`, whose `ScalarSubqueryExpr`
+    // cannot be deserialized once Ballista splits the plan into stages. See
+    // #1909.
+    #[test]
+    fn should_disable_physical_uncorrelated_scalar_subquery() {
+        let config = SessionConfig::new().upgrade_for_ballista();
+        assert!(
+            !config
+                .options()
+                .optimizer
+                .enable_physical_uncorrelated_scalar_subquery
+        );
+    }
+
     #[test]
     fn should_convert_to_key_value_pairs() {
         // key value pairs should contain datafusion and ballista values
diff --git a/ballista/core/src/lib.rs b/ballista/core/src/lib.rs
index 08cbf09dc5..e8364ada38 100644
--- a/ballista/core/src/lib.rs
+++ b/ballista/core/src/lib.rs
@@ -29,6 +29,18 @@ pub use crate::ids::JobId;
 /// The current version of Ballista, derived from the Cargo package version.
 pub const BALLISTA_VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Wire-protocol version negotiated between scheduler and executor.
+///
+/// The scheduler compares this against `ExecutorRegistration.ballista_protocol_version`
+/// on every registration and heartbeat; mismatched executors are rejected with
+/// `Status::failed_precondition` and never receive work. Bump this on any release
+/// that changes the executor↔scheduler wire format (proto shape, task/plan
+/// encoding, launch/heartbeat semantics). Most releases will not bump it.
+///
+/// Zero is reserved as the proto-default "unset" value, produced by executors
+/// that predate this field — it never matches a real scheduler version.
+pub const BALLISTA_PROTOCOL_VERSION: u32 = 1;
+
 /// Prints the current Ballista version to stdout.
 pub fn print_version() {
     println!("Ballista version: {BALLISTA_VERSION}")
diff --git a/ballista/core/src/object_store.rs b/ballista/core/src/object_store.rs
index b26e0f193e..ce0ea01fa0 100644
--- a/ballista/core/src/object_store.rs
+++ b/ballista/core/src/object_store.rs
@@ -145,7 +145,7 @@ impl ObjectStoreRegistry for CustomObjectStoreRegistry {
 
     fn get_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {
         let scheme = url.scheme();
-        log::trace!("get_store: {:?}", &self.s3options.config.read());
+        log::trace!("get_store: {:?}", self.s3options.config.read());
         match scheme {
             "" | "file" => Ok(self.local.clone()),
             "http" | "https" => Ok(Arc::new(
diff --git a/ballista/core/src/planner.rs b/ballista/core/src/planner.rs
index e1ed75998c..ecf41733b8 100644
--- a/ballista/core/src/planner.rs
+++ b/ballista/core/src/planner.rs
@@ -307,12 +307,10 @@ mod test {
 
         assert!(matches!(analyze_df.logical_plan(), LogicalPlan::Analyze(_)));
         let explain = plan
-            .as_any()
             .downcast_ref::<DistributedExplainAnalyzeExec<LogicalPlanNode>>()
             .unwrap();
         assert!(
             explain.children()[0]
-                .as_any()
                 .downcast_ref::<DistributedQueryExec<LogicalPlanNode>>()
                 .is_some()
         );
diff --git a/ballista/core/src/registry.rs b/ballista/core/src/registry.rs
index 1d45e7b58a..8ec1d97539 100644
--- a/ballista/core/src/registry.rs
+++ b/ballista/core/src/registry.rs
@@ -19,7 +19,9 @@ use datafusion::common::DataFusionError;
 use datafusion::execution::{FunctionRegistry, SessionState};
 use datafusion::functions::all_default_functions;
 use datafusion::functions_aggregate::all_default_aggregate_functions;
+use datafusion::functions_nested::all_default_higher_order_functions;
 use datafusion::functions_window::all_default_window_functions;
+use datafusion::logical_expr::HigherOrderUDF;
 use datafusion::logical_expr::planner::ExprPlanner;
 use datafusion::logical_expr::{AggregateUDF, ScalarUDF, WindowUDF};
 use std::collections::{HashMap, HashSet};
@@ -41,6 +43,8 @@ pub struct BallistaFunctionRegistry {
     pub aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
     /// Window user-defined functions.
     pub window_functions: HashMap<String, Arc<WindowUDF>>,
+    /// Higher-order (lambda) user-defined functions.
+    pub higher_order_functions: HashMap<String, Arc<HigherOrderUDF>>,
 }
 
 impl Default for BallistaFunctionRegistry {
@@ -62,6 +66,12 @@ impl Default for BallistaFunctionRegistry {
                 .map(|f| (f.name().to_string(), f))
                 .collect();
 
+        let higher_order_functions: HashMap<String, Arc<HigherOrderUDF>> =
+            all_default_higher_order_functions()
+                .into_iter()
+                .map(|f| (f.name().to_string(), f))
+                .collect();
+
         #[cfg(feature = "spark-compat")]
         let (scalar_functions, aggregate_functions, window_functions) = {
             let mut scalar_functions = scalar_functions;
@@ -85,6 +95,7 @@ impl Default for BallistaFunctionRegistry {
             scalar_functions,
             aggregate_functions,
             window_functions,
+            higher_order_functions,
         }
     }
 }
@@ -106,6 +117,21 @@ impl FunctionRegistry for BallistaFunctionRegistry {
         self.window_functions.keys().cloned().collect()
     }
 
+    fn higher_order_function_names(&self) -> HashSet<String> {
+        self.higher_order_functions.keys().cloned().collect()
+    }
+
+    fn higher_order_function(
+        &self,
+        name: &str,
+    ) -> datafusion::common::Result<Arc<HigherOrderUDF>> {
+        self.higher_order_functions.get(name).cloned().ok_or_else(|| {
+            DataFusionError::Internal(format!(
+                "There is no higher-order function named \"{name}\" in the TaskContext"
+            ))
+        })
+    }
+
     fn udf(&self, name: &str) -> datafusion::common::Result<Arc<ScalarUDF>> {
         let result = self.scalar_functions.get(name);
 
@@ -142,11 +168,13 @@ impl From<&SessionState> for BallistaFunctionRegistry {
         let scalar_functions = state.scalar_functions().clone();
         let aggregate_functions = state.aggregate_functions().clone();
         let window_functions = state.window_functions().clone();
+        let higher_order_functions = state.higher_order_functions().clone();
 
         Self {
             scalar_functions,
             aggregate_functions,
             window_functions,
+            higher_order_functions,
         }
     }
 }
@@ -164,6 +192,26 @@ mod tests {
         assert!(registry.udfs().contains("abs"));
     }
 
+    #[test]
+    fn test_higher_order_functions_available() {
+        let registry = BallistaFunctionRegistry::default();
+
+        // DataFusion ships built-in higher-order functions (e.g. array_filter);
+        // the registry the executor runs with must carry them or those queries
+        // fail to plan on the executor.
+        let names = registry.higher_order_function_names();
+        assert!(
+            !names.is_empty(),
+            "registry must expose DataFusion's built-in higher-order functions"
+        );
+        for name in &names {
+            assert!(
+                registry.higher_order_function(name).is_ok(),
+                "named higher-order function {name} must be retrievable"
+            );
+        }
+    }
+
     #[test]
     #[cfg(not(feature = "spark-compat"))]
     fn test_spark_functions_unavailable_without_feature() {
diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs
index 269827145e..9f9efa8f1d 100644
--- a/ballista/core/src/serde/generated/ballista.rs
+++ b/ballista/core/src/serde/generated/ballista.rs
@@ -172,8 +172,6 @@ pub struct ExecutionGraph {
     pub output_locations: ::prost::alloc::vec::Vec<PartitionLocation>,
     #[prost(string, tag = "7")]
     pub scheduler_id: ::prost::alloc::string::String,
-    #[prost(uint32, tag = "8")]
-    pub task_id_gen: u32,
     #[prost(message, repeated, tag = "9")]
     pub failed_attempts: ::prost::alloc::vec::Vec<StageAttempts>,
     #[prost(string, tag = "10")]
@@ -424,14 +422,35 @@ pub struct PartitionId {
     #[prost(uint32, tag = "4")]
     pub partition_id: u32,
 }
-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+/// Within-stage task identity (paired with job_id + stage_id on the parent
+/// MultiTaskDefinition). One task processes a slice of partitions;
+/// `task_id` names the task within the stage (it's the task's append-order
+/// slot in `RunningStage.task_infos`, so (job_id, stage_id, task_id) is
+/// globally unique). `global_output_partition_ids` gives the concrete
+/// global partition ids the task's restricted plan is covering. Writers
+/// use these to name shuffle files with global identity (via
+/// `create_shuffle_path`) so downstream reads have a stable, canonical
+/// address regardless of how the scheduler split the work.
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct TaskId {
     #[prost(uint32, tag = "1")]
     pub task_id: u32,
     #[prost(uint32, tag = "2")]
     pub task_attempt_num: u32,
-    #[prost(uint32, tag = "3")]
-    pub partition_id: u32,
+    /// Global partition ids covered by this task, in slice order. Position i in
+    /// the restricted plan corresponds to `global_output_partition_ids\[i\]` globally.
+    /// `\[packed=false\]` keeps the single-partition case wire-compatible with the
+    /// legacy `uint32 partition_id = 3` field: one element writes as a single
+    /// varint at tag 3, exactly matching the old scalar encoding.
+    #[prost(uint32, repeated, packed = "false", tag = "3")]
+    pub global_output_partition_ids: ::prost::alloc::vec::Vec<u32>,
+    /// Vcores this task consumed from the executor's budget at bind time.
+    /// The executor uses this to size the task's memory pool proportionally
+    /// (`pool = total_memory * vcores_consumed / total_vcores`), so a
+    /// multi-partition task claiming N vcores gets N/total of the executor's
+    /// memory budget rather than a single per-vcore share.
+    #[prost(uint32, tag = "5")]
+    pub vcores_consumed: u32,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct PartitionStats {
@@ -501,6 +520,11 @@ pub struct NamedRatio {
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct OperatorMetric {
+    /// Local partition index within the reporting task's plan (0..slice_len-1).
+    /// The scheduler maps this to a global partition id via the task's
+    /// `global_input_partition_ids` when folding metrics into stage metrics.
+    #[prost(uint32, optional, tag = "16")]
+    pub partition: ::core::option::Option<u32>,
     #[prost(
         oneof = "operator_metric::Metric",
         tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15"
@@ -575,6 +599,12 @@ pub struct ExecutorRegistration {
     pub specification: ::core::option::Option<ExecutorSpecification>,
     #[prost(message, optional, tag = "6")]
     pub os_info: ::core::option::Option<ExecutorOperatingSystemSpecification>,
+    /// Wire-protocol version this executor was built against. The scheduler
+    /// rejects heartbeats/registrations whose version does not match its own
+    /// compiled-in BALLISTA_PROTOCOL_VERSION. Old executors that predate this
+    /// field default to 0, which is never a valid scheduler version.
+    #[prost(uint32, tag = "7")]
+    pub ballista_protocol_version: u32,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ExecutorHeartbeat {
@@ -643,6 +673,11 @@ pub struct ExecutorSpecification {
     #[prost(message, repeated, tag = "1")]
     pub resources: ::prost::alloc::vec::Vec<ExecutorResource>,
 }
+/// Virtual cores this executor has been assigned at runtime — analogous to
+/// YARN vcores (yarn.nodemanager.resource.cpu-vcores) or Spark's
+/// spark.executor.cores. Not physical `nproc`: may over- or under-subscribe.
+/// One task per executor drives this many DataFusion partitions in parallel
+/// through one plan-Arc.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ExecutorResource {
     #[prost(oneof = "executor_resource::Resource", tags = "1")]
@@ -653,7 +688,7 @@ pub mod executor_resource {
     #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)]
     pub enum Resource {
         #[prost(uint32, tag = "1")]
-        TaskSlots(u32),
+        Vcores(u32),
     }
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
@@ -677,17 +712,20 @@ pub struct ExecutorOperatingSystemSpecification {
     #[prost(uint64, tag = "9")]
     pub open_files_limit: u64,
 }
+/// One instance per executor. Tracks free vcores for scheduler-side task
+/// binding. See ExecutorResource for the vcore definition (YARN vcores or
+/// Spark's spark.executor.cores).
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
-pub struct AvailableTaskSlots {
+pub struct AvailableVcores {
     #[prost(string, tag = "1")]
     pub executor_id: ::prost::alloc::string::String,
     #[prost(uint32, tag = "2")]
-    pub slots: u32,
+    pub vcores: u32,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ExecutorTaskSlots {
+pub struct ExecutorVcores {
     #[prost(message, repeated, tag = "1")]
-    pub task_slots: ::prost::alloc::vec::Vec<AvailableTaskSlots>,
+    pub vcores: ::prost::alloc::vec::Vec<AvailableVcores>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ExecutorData {
@@ -784,6 +822,10 @@ pub struct ShuffleWritePartition {
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct TaskStatus {
+    /// Task's append-order slot in `RunningStage.task_infos`; (job_id,
+    /// stage_id, task_id) is globally unique. Under the multi-partition-task
+    /// model one task processes a partition slice, so this names the task,
+    /// not a single partition.
     #[prost(uint32, tag = "1")]
     pub task_id: u32,
     #[prost(string, tag = "2")]
@@ -792,8 +834,6 @@ pub struct TaskStatus {
     pub stage_id: u32,
     #[prost(uint32, tag = "4")]
     pub stage_attempt_num: u32,
-    #[prost(uint32, tag = "5")]
-    pub partition_id: u32,
     #[prost(uint64, tag = "6")]
     pub launch_time: u64,
     #[prost(uint64, tag = "7")]
@@ -822,11 +862,21 @@ pub struct PollWorkParams {
     #[prost(message, optional, tag = "1")]
     pub metadata: ::core::option::Option<ExecutorRegistration>,
     #[prost(uint32, tag = "2")]
-    pub num_free_slots: u32,
+    pub num_free_vcores: u32,
     /// All tasks must be reported until they reach the failed or completed state
     #[prost(message, repeated, tag = "3")]
     pub task_status: ::prost::alloc::vec::Vec<TaskStatus>,
 }
+/// Pull-based single-task dispatch. One task processes a slice of
+/// partitions; `task_id` names the task within the stage (its append-order
+/// slot in `RunningStage.task_infos`), `global_output_partition_ids` gives
+/// the concrete global partition ids it covers. The task's plan is
+/// scheduler-side shrink-restricted so its leaves report `slice.len()`
+/// partitions; the writer uses `global_output_partition_ids` to attach
+/// global identity to shuffle files (except in cases where the plan itself
+/// resets partitioning: the writer walks its child plan to detect
+/// SortPreservingMergeExec or RepartitionExec::Hash and picks the right
+/// global mapping).
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct TaskDefinition {
     #[prost(uint32, tag = "1")]
@@ -839,8 +889,6 @@ pub struct TaskDefinition {
     pub stage_id: u32,
     #[prost(uint32, tag = "5")]
     pub stage_attempt_num: u32,
-    #[prost(uint32, tag = "6")]
-    pub partition_id: u32,
     #[prost(bytes = "vec", tag = "7")]
     pub plan: ::prost::alloc::vec::Vec<u8>,
     #[prost(string, tag = "9")]
@@ -849,6 +897,12 @@ pub struct TaskDefinition {
     pub launch_time: u64,
     #[prost(message, repeated, tag = "11")]
     pub props: ::prost::alloc::vec::Vec<KeyValuePair>,
+    /// Global partition ids covered by this task, in slice order.
+    #[prost(uint32, repeated, tag = "12")]
+    pub global_output_partition_ids: ::prost::alloc::vec::Vec<u32>,
+    /// Vcores this task consumed at bind time — see TaskId.vcores_consumed.
+    #[prost(uint32, tag = "13")]
+    pub vcores_consumed: u32,
 }
 /// A set of tasks in the same stage
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -958,7 +1012,7 @@ pub struct ExecuteQueryParams {
     /// client and scheduler
     #[prost(string, tag = "5")]
     pub operation_id: ::prost::alloc::string::String,
-    #[prost(oneof = "execute_query_params::Query", tags = "1, 6")]
+    #[prost(oneof = "execute_query_params::Query", tags = "1, 6, 7")]
     pub query: ::core::option::Option<execute_query_params::Query>,
 }
 /// Nested message and enum types in `ExecuteQueryParams`.
@@ -969,6 +1023,12 @@ pub mod execute_query_params {
         LogicalPlan(::prost::alloc::vec::Vec<u8>),
         #[prost(bytes, tag = "6")]
         SubstraitPlan(::prost::alloc::vec::Vec<u8>),
+        /// An already-built physical plan, serialized with the scheduler's
+        /// physical extension codec. Lets clients bypass logical-plan creation
+        /// on the scheduler, e.g. for plans containing custom operators that
+        /// have no logical-plan representation.
+        #[prost(bytes, tag = "7")]
+        PhysicalPlan(::prost::alloc::vec::Vec<u8>),
     }
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -1182,6 +1242,10 @@ pub struct CancelJobResult {
 pub struct CleanJobDataParams {
     #[prost(string, tag = "1")]
     pub job_id: ::prost::alloc::string::String,
+    /// Specific stage ids to remove within the job dir.
+    /// Empty means remove the whole job dir (legacy behavior).
+    #[prost(uint32, repeated, tag = "2")]
+    pub remove_stage_ids: ::prost::alloc::vec::Vec<u32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct CleanJobDataResult {}
@@ -1227,19 +1291,23 @@ pub struct CancelTasksResult {
 pub struct RemoveJobDataParams {
     #[prost(string, tag = "1")]
     pub job_id: ::prost::alloc::string::String,
+    /// Specific stage ids to remove within the job dir.
+    /// Empty means remove the whole job dir (legacy behavior).
+    #[prost(uint32, repeated, tag = "2")]
+    pub remove_stage_ids: ::prost::alloc::vec::Vec<u32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct RemoveJobDataResult {}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct RunningTaskInfo {
+    /// Per-stage append-order slot; (job_id, stage_id, task_id) is globally
+    /// unique (see TaskStatus).
     #[prost(uint32, tag = "1")]
     pub task_id: u32,
     #[prost(string, tag = "2")]
     pub job_id: ::prost::alloc::string::String,
     #[prost(uint32, tag = "3")]
     pub stage_id: u32,
-    #[prost(uint32, tag = "4")]
-    pub partition_id: u32,
 }
 /// Generated client implementations.
 pub mod scheduler_grpc_client {
diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs
index 46e3430c38..045b303f01 100644
--- a/ballista/core/src/serde/mod.rs
+++ b/ballista/core/src/serde/mod.rs
@@ -36,6 +36,7 @@ use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning;
 use datafusion_proto::physical_plan::to_proto::serialize_partitioning;
 use datafusion_proto::physical_plan::{
     DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+    PhysicalPlanDecodeContext,
 };
 use datafusion_proto::protobuf::proto_error;
 use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode};
@@ -381,15 +382,15 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
                 )
             })?;
         let converter = DefaultPhysicalProtoConverter {};
+        let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self);
         match ballista_plan {
             PhysicalPlanType::ShuffleWriter(shuffle_writer) => {
                 let input = inputs[0].clone();
 
                 let shuffle_output_partitioning = parse_protobuf_hash_partitioning(
                     shuffle_writer.output_partitioning.as_ref(),
-                    ctx,
+                    &decode_ctx,
                     input.schema().as_ref(),
-                    self.default_codec.as_ref(),
                     &converter,
                 )?;
 
@@ -406,9 +407,8 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
 
                 let shuffle_output_partitioning = parse_protobuf_hash_partitioning(
                     sort_shuffle_writer.output_partitioning.as_ref(),
-                    ctx,
+                    &decode_ctx,
                     input.schema().as_ref(),
-                    self.default_codec.as_ref(),
                     &converter,
                 )?;
 
@@ -423,11 +423,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
                 } else {
                     8192
                 };
-                let config = SortShuffleConfig::new(
-                    true,
-                    datafusion::arrow::ipc::CompressionType::LZ4_FRAME,
-                    batch_size,
-                );
+                let config = SortShuffleConfig::new(true, batch_size);
 
                 Ok(Arc::new(SortShuffleWriterExec::try_new(
                     sort_shuffle_writer.job_id.clone().into(),
@@ -460,9 +456,8 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
                     .collect::<Result<Vec<_>, DataFusionError>>()?;
                 let partitioning = parse_protobuf_partitioning(
                     shuffle_reader.partitioning.as_ref(),
-                    ctx,
+                    &decode_ctx,
                     schema.as_ref(),
-                    self.default_codec.as_ref(),
                     &converter,
                 )?;
                 let partitioning = partitioning
@@ -503,9 +498,8 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
                     Arc::new(convert_required!(unresolved_shuffle.schema)?);
                 let partitioning = parse_protobuf_partitioning(
                     unresolved_shuffle.partitioning.as_ref(),
-                    ctx,
+                    &decode_ctx,
                     schema.as_ref(),
-                    self.default_codec.as_ref(),
                     &converter,
                 )?;
                 let partitioning = partitioning
@@ -557,7 +551,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
         node: Arc<dyn ExecutionPlan>,
         buf: &mut Vec<u8>,
     ) -> Result<(), DataFusionError> {
-        if let Some(exec) = node.as_any().downcast_ref::<ShuffleWriterExec>() {
+        if let Some(exec) = node.downcast_ref::<ShuffleWriterExec>() {
             // note that we use shuffle_output_partitioning() rather than output_partitioning()
             // to get the true output partitioning
             let output_partitioning = match exec.shuffle_output_partitioning() {
@@ -596,7 +590,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
             })?;
 
             Ok(())
-        } else if let Some(exec) = node.as_any().downcast_ref::<SortShuffleWriterExec>() {
+        } else if let Some(exec) = node.downcast_ref::<SortShuffleWriterExec>() {
             let output_partitioning = match exec.shuffle_output_partitioning() {
                 Partitioning::Hash(exprs, partition_count) => {
                     Some(datafusion_proto::protobuf::PhysicalHashRepartition {
@@ -639,7 +633,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
             })?;
 
             Ok(())
-        } else if let Some(exec) = node.as_any().downcast_ref::<ShuffleReaderExec>() {
+        } else if let Some(exec) = node.downcast_ref::<ShuffleReaderExec>() {
             let stage_id = exec.stage_id as u32;
             let mut partition = vec![];
             for location in &exec.partition {
@@ -682,7 +676,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
             })?;
 
             Ok(())
-        } else if let Some(exec) = node.as_any().downcast_ref::<UnresolvedShuffleExec>() {
+        } else if let Some(exec) = node.downcast_ref::<UnresolvedShuffleExec>() {
             let converter = DefaultPhysicalProtoConverter {};
             let partitioning = serialize_partitioning(
                 &exec.properties().partitioning,
@@ -708,7 +702,7 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
             })?;
 
             Ok(())
-        } else if let Some(exec) = node.as_any().downcast_ref::<ChaosExec>() {
+        } else if let Some(exec) = node.downcast_ref::<ChaosExec>() {
             let proto = protobuf::BallistaPhysicalPlanNode {
                 physical_plan_type: Some(PhysicalPlanType::ChaosExec(
                     protobuf::ChaosExecNode {
@@ -812,6 +806,94 @@ mod test {
         ]))
     }
 
+    // Regression coverage for #1838 and the removed `make_filter_projection_serde_safe`
+    // workaround: a `FilterExec` that projects to zero columns must survive physical
+    // plan serialization with its empty projection intact. datafusion-proto 53.1.0
+    // could not distinguish `Some(vec![])` (empty projection) from `None` (full
+    // projection) and decoded the former back as `None`, shifting column indices.
+    // DataFusion 54 preserves the distinction, so no Ballista-side rewrite is needed.
+    #[tokio::test]
+    async fn filter_exec_empty_projection_survives_physical_serde() {
+        use datafusion::physical_plan::empty::EmptyExec;
+        use datafusion::physical_plan::expressions::lit;
+        use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};
+        use datafusion_proto::physical_plan::{
+            AsExecutionPlan, DefaultPhysicalExtensionCodec,
+        };
+
+        let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+        let input: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(schema));
+        let filter = FilterExecBuilder::new(lit(true), input)
+            .apply_projection(Some(vec![]))
+            .unwrap()
+            .build()
+            .unwrap();
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(filter);
+        assert_eq!(
+            plan.schema().fields().len(),
+            0,
+            "precondition: filter projects to zero columns"
+        );
+
+        let codec = DefaultPhysicalExtensionCodec {};
+        let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec).unwrap();
+        let ctx = SessionContext::new().task_ctx();
+        let decoded = proto.try_into_physical_plan(&ctx, &codec).unwrap();
+
+        assert_eq!(
+            decoded.schema().fields().len(),
+            0,
+            "decoded FilterExec must still project zero columns"
+        );
+        let filter = decoded
+            .downcast_ref::<FilterExec>()
+            .expect("decoded plan must be a FilterExec");
+        assert_eq!(
+            filter.projection().as_ref().map(|p| p.is_empty()),
+            Some(true),
+            "empty projection must round-trip as Some(vec![]), not None"
+        );
+    }
+
+    // `datafusion-proto` encodes an `EmptyExec` as its schema alone, so a
+    // multi-partition `EmptyExec` decodes with a single partition (apache/datafusion
+    // #23642). `make_empty_exec_serde_safe` in the scheduler works around this by
+    // rewriting such nodes into a round-robin `RepartitionExec` before a stage plan
+    // goes on the wire.
+    //
+    // This test pins the upstream behaviour that makes the workaround necessary: when
+    // it starts failing, DataFusion preserves the partition count and the workaround
+    // can be deleted.
+    #[tokio::test]
+    async fn empty_exec_partition_count_is_lost_by_datafusion_proto() {
+        use datafusion::physical_plan::ExecutionPlanProperties;
+        use datafusion::physical_plan::empty::EmptyExec;
+        use datafusion_proto::physical_plan::{
+            AsExecutionPlan, DefaultPhysicalExtensionCodec,
+        };
+
+        let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+        let plan: Arc<dyn ExecutionPlan> =
+            Arc::new(EmptyExec::new(schema).with_partitions(4));
+        assert_eq!(
+            plan.output_partitioning().partition_count(),
+            4,
+            "precondition: EmptyExec reports 4 partitions"
+        );
+
+        let codec = DefaultPhysicalExtensionCodec {};
+        let proto = PhysicalPlanNode::try_from_physical_plan(plan, &codec).unwrap();
+        let ctx = SessionContext::new().task_ctx();
+        let decoded = proto.try_into_physical_plan(&ctx, &codec).unwrap();
+
+        assert_eq!(
+            decoded.output_partitioning().partition_count(),
+            1,
+            "EmptyExec partition count is dropped on the wire; if this now decodes as \
+             4, apache/datafusion#23642 is fixed and make_empty_exec_serde_safe can go"
+        );
+    }
+
     #[tokio::test]
     async fn test_unresolved_shuffle_exec_roundtrip() {
         let schema = create_test_schema();
@@ -834,7 +916,6 @@ mod test {
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
 
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<UnresolvedShuffleExec>()
             .expect("Expected UnresolvedShuffleExec");
 
@@ -871,7 +952,6 @@ mod test {
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
 
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<ShuffleReaderExec>()
             .expect("Expected ShuffleReaderExec");
 
@@ -914,7 +994,6 @@ mod test {
         let ctx = SessionContext::new().task_ctx();
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<ShuffleReaderExec>()
             .expect("Expected ShuffleReaderExec");
 
@@ -968,7 +1047,6 @@ mod test {
         let ctx = SessionContext::new().task_ctx();
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<ShuffleReaderExec>()
             .expect("Expected ShuffleReaderExec");
 
@@ -1014,7 +1092,6 @@ mod test {
         let ctx = SessionContext::new().task_ctx();
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<UnresolvedShuffleExec>()
             .expect("Expected UnresolvedShuffleExec");
 
@@ -1065,7 +1142,6 @@ mod test {
         let ctx = SessionContext::new().task_ctx();
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<ShuffleReaderExec>()
             .expect("Expected ShuffleReaderExec");
 
@@ -1163,7 +1239,6 @@ mod test {
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
 
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<UnresolvedShuffleExec>()
             .expect("Expected UnresolvedShuffleExec");
 
@@ -1190,7 +1265,6 @@ mod test {
         let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
 
         let decoded_exec = decoded_plan
-            .as_any()
             .downcast_ref::<ShuffleReaderExec>()
             .expect("Expected ShuffleReaderExec");
 
diff --git a/ballista/core/src/serde/scheduler/from_proto.rs b/ballista/core/src/serde/scheduler/from_proto.rs
index ba8b9b33a0..410e831a34 100644
--- a/ballista/core/src/serde/scheduler/from_proto.rs
+++ b/ballista/core/src/serde/scheduler/from_proto.rs
@@ -19,7 +19,7 @@ use chrono::{TimeZone, Utc};
 use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
 
 use datafusion::execution::TaskContext;
-use datafusion::logical_expr::{AggregateUDF, ScalarUDF, WindowUDF};
+use datafusion::logical_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
 use datafusion::physical_plan::metrics::{
     Count, Gauge, MetricValue, MetricsSet, PruningMetrics, RatioMetrics, Time, Timestamp,
 };
@@ -130,42 +130,42 @@ impl TryInto<PartitionLocation> for protobuf::PartitionLocation {
     }
 }
 
-impl TryInto<MetricValue> for protobuf::OperatorMetric {
+impl TryInto<MetricValue> for operator_metric::Metric {
     type Error = BallistaError;
 
     fn try_into(self) -> Result<MetricValue, Self::Error> {
-        match self.metric {
-            Some(operator_metric::Metric::OutputRows(value)) => {
+        match self {
+            operator_metric::Metric::OutputRows(value) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::OutputRows(count))
             }
-            Some(operator_metric::Metric::ElapseTime(value)) => {
+            operator_metric::Metric::ElapseTime(value) => {
                 let time = Time::new();
                 time.add_duration(Duration::from_nanos(value));
                 Ok(MetricValue::ElapsedCompute(time))
             }
-            Some(operator_metric::Metric::SpillCount(value)) => {
+            operator_metric::Metric::SpillCount(value) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::SpillCount(count))
             }
-            Some(operator_metric::Metric::SpilledBytes(value)) => {
+            operator_metric::Metric::SpilledBytes(value) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::SpilledBytes(count))
             }
-            Some(operator_metric::Metric::SpilledRows(value)) => {
+            operator_metric::Metric::SpilledRows(value) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::SpilledRows(count))
             }
-            Some(operator_metric::Metric::CurrentMemoryUsage(value)) => {
+            operator_metric::Metric::CurrentMemoryUsage(value) => {
                 let gauge = Gauge::new();
                 gauge.add(value as usize);
                 Ok(MetricValue::CurrentMemoryUsage(gauge))
             }
-            Some(operator_metric::Metric::Count(NamedCount { name, value })) => {
+            operator_metric::Metric::Count(NamedCount { name, value }) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::Count {
@@ -173,7 +173,7 @@ impl TryInto<MetricValue> for protobuf::OperatorMetric {
                     count,
                 })
             }
-            Some(operator_metric::Metric::Gauge(NamedGauge { name, value })) => {
+            operator_metric::Metric::Gauge(NamedGauge { name, value }) => {
                 let gauge = Gauge::new();
                 gauge.add(value as usize);
                 Ok(MetricValue::Gauge {
@@ -181,7 +181,7 @@ impl TryInto<MetricValue> for protobuf::OperatorMetric {
                     gauge,
                 })
             }
-            Some(operator_metric::Metric::Time(NamedTime { name, value })) => {
+            operator_metric::Metric::Time(NamedTime { name, value }) => {
                 let time = Time::new();
                 time.add_duration(Duration::from_nanos(value));
                 Ok(MetricValue::Time {
@@ -189,26 +189,26 @@ impl TryInto<MetricValue> for protobuf::OperatorMetric {
                     time,
                 })
             }
-            Some(operator_metric::Metric::StartTimestamp(value)) => {
+            operator_metric::Metric::StartTimestamp(value) => {
                 let timestamp = Timestamp::new();
                 timestamp.set(Utc.timestamp_nanos(value));
                 Ok(MetricValue::StartTimestamp(timestamp))
             }
-            Some(operator_metric::Metric::EndTimestamp(value)) => {
+            operator_metric::Metric::EndTimestamp(value) => {
                 let timestamp = Timestamp::new();
                 timestamp.set(Utc.timestamp_nanos(value));
                 Ok(MetricValue::EndTimestamp(timestamp))
             }
-            Some(operator_metric::Metric::OutputBytes(value)) => {
+            operator_metric::Metric::OutputBytes(value) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::OutputBytes(count))
             }
-            Some(operator_metric::Metric::PruningMetrics(NamedPruningMetrics {
+            operator_metric::Metric::PruningMetrics(NamedPruningMetrics {
                 name,
                 pruned,
                 matched,
-            })) => {
+            }) => {
                 let pruning_metrics = PruningMetrics::new();
                 pruning_metrics.add_pruned(pruned as usize);
                 pruning_metrics.add_matched(matched as usize);
@@ -217,7 +217,7 @@ impl TryInto<MetricValue> for protobuf::OperatorMetric {
                     pruning_metrics,
                 })
             }
-            Some(operator_metric::Metric::Ratio(NamedRatio { name, part, total })) => {
+            operator_metric::Metric::Ratio(NamedRatio { name, part, total }) => {
                 let ratio_metrics = RatioMetrics::new();
                 ratio_metrics.add_part(part as usize);
                 ratio_metrics.add_total(total as usize);
@@ -226,14 +226,11 @@ impl TryInto<MetricValue> for protobuf::OperatorMetric {
                     ratio_metrics,
                 })
             }
-            Some(operator_metric::Metric::OutputBatches(value)) => {
+            operator_metric::Metric::OutputBatches(value) => {
                 let count = Count::new();
                 count.add(value as usize);
                 Ok(MetricValue::OutputBatches(count))
             }
-            None => Err(BallistaError::General(
-                "scheduler::from_proto(OperatorMetric) metric is None.".to_owned(),
-            )),
         }
     }
 }
@@ -243,15 +240,15 @@ impl TryInto<MetricsSet> for protobuf::OperatorMetricsSet {
 
     fn try_into(self) -> Result<MetricsSet, Self::Error> {
         let mut ms = MetricsSet::new();
-        let metrics = self
-            .metrics
-            .into_iter()
-            .map(|m| m.try_into())
-            .collect::<Result<Vec<_>, BallistaError>>()?;
-
-        for value in metrics {
-            let new_metric = Arc::new(Metric::new(value, None));
-            ms.push(new_metric)
+        for proto_metric in self.metrics {
+            let partition = proto_metric.partition.map(|p| p as usize);
+            let inner = proto_metric.metric.ok_or_else(|| {
+                BallistaError::General(
+                    "scheduler::from_proto(OperatorMetric) metric is None.".to_owned(),
+                )
+            })?;
+            let value: MetricValue = inner.try_into()?;
+            ms.push(Arc::new(Metric::new(value, partition)));
         }
         Ok(ms)
     }
@@ -274,12 +271,12 @@ impl Into<ExecutorMetadata> for protobuf::ExecutorMetadata {
 #[allow(clippy::from_over_into)]
 impl Into<ExecutorSpecification> for protobuf::ExecutorSpecification {
     fn into(self) -> ExecutorSpecification {
-        let mut ret = ExecutorSpecification { task_slots: 0 };
+        let mut ret = ExecutorSpecification { vcores: 0 };
         for resource in self.resources {
             if let Some(resource_spec) = resource.resource {
                 match resource_spec {
-                    protobuf::executor_resource::Resource::TaskSlots(num_slots) => {
-                        ret.task_slots = num_slots
+                    protobuf::executor_resource::Resource::Vcores(vcores) => {
+                        ret.vcores = vcores
                     }
                 }
             }
@@ -313,21 +310,21 @@ impl Into<ExecutorData> for protobuf::ExecutorData {
     fn into(self) -> ExecutorData {
         let mut ret = ExecutorData {
             executor_id: self.executor_id,
-            total_task_slots: 0,
-            available_task_slots: 0,
+            total_vcores: 0,
+            available_vcores: 0,
         };
         for resource in self.resources {
-            if let Some(task_slots) = resource.total
-                && let Some(protobuf::executor_resource::Resource::TaskSlots(task_slots)) =
-                    task_slots.resource
+            if let Some(total) = resource.total
+                && let Some(protobuf::executor_resource::Resource::Vcores(vcores)) =
+                    total.resource
             {
-                ret.total_task_slots = task_slots
+                ret.total_vcores = vcores
             };
-            if let Some(task_slots) = resource.available
-                && let Some(protobuf::executor_resource::Resource::TaskSlots(task_slots)) =
-                    task_slots.resource
+            if let Some(available) = resource.available
+                && let Some(protobuf::executor_resource::Resource::Vcores(vcores)) =
+                    available.resource
             {
-                ret.available_task_slots = task_slots
+                ret.available_vcores = vcores
             };
         }
         ret
@@ -338,6 +335,7 @@ impl Into<ExecutorData> for protobuf::ExecutorData {
 ///
 /// This function deserializes the execution plan from the protobuf representation
 /// and constructs a complete task definition with the provided runtime configuration.
+#[allow(clippy::too_many_arguments)]
 pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>(
     task: protobuf::TaskDefinition,
     produce_runtime: RuntimeProducer,
@@ -345,6 +343,7 @@ pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
     scalar_functions: HashMap<String, Arc<ScalarUDF>>,
     aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
     window_functions: HashMap<String, Arc<WindowUDF>>,
+    higher_order_functions: HashMap<String, Arc<HigherOrderUDF>>,
     codec: BallistaCodec<T, U>,
 ) -> Result<TaskDefinition, BallistaError> {
     let session_config = session_config.update_from_key_value_pair(&task.props);
@@ -354,6 +353,7 @@ pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
         scalar_functions: scalar_functions.clone(),
         aggregate_functions: aggregate_functions.clone(),
         window_functions: window_functions.clone(),
+        higher_order_functions: higher_order_functions.clone(),
     });
 
     let ctx = TaskContext::new(
@@ -361,6 +361,7 @@ pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
         task.session_id.clone(),
         session_config.clone(),
         scalar_functions,
+        higher_order_functions,
         aggregate_functions,
         window_functions,
         runtime.clone(),
@@ -373,12 +374,16 @@ pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
 
     let job_id = task.job_id.into();
     let stage_id = task.stage_id as usize;
-    let partition_id = task.partition_id as usize;
     let task_attempt_num = task.task_attempt_num as usize;
     let stage_attempt_num = task.stage_attempt_num as usize;
     let launch_time = task.launch_time;
     let task_id = task.task_id as usize;
     let session_id = task.session_id;
+    let global_output_partition_ids = task
+        .global_output_partition_ids
+        .iter()
+        .map(|pid| *pid as usize)
+        .collect::<Vec<_>>();
 
     Ok(TaskDefinition {
         task_id,
@@ -386,7 +391,8 @@ pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
         job_id,
         stage_id,
         stage_attempt_num,
-        partition_id,
+        global_output_partition_ids,
+        vcores_consumed: task.vcores_consumed,
         plan,
         launch_time,
         session_id,
@@ -399,6 +405,7 @@ pub fn get_task_definition<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
 ///
 /// This function handles batch task definitions where multiple partitions share
 /// the same execution plan, creating individual task definitions for each partition.
+#[allow(clippy::too_many_arguments)]
 pub fn get_task_definition_vec<
     T: 'static + AsLogicalPlan,
     U: 'static + AsExecutionPlan,
@@ -409,6 +416,7 @@ pub fn get_task_definition_vec<
     scalar_functions: HashMap<String, Arc<ScalarUDF>>,
     aggregate_functions: HashMap<String, Arc<AggregateUDF>>,
     window_functions: HashMap<String, Arc<WindowUDF>>,
+    higher_order_functions: HashMap<String, Arc<HigherOrderUDF>>,
     codec: BallistaCodec<T, U>,
 ) -> Result<Vec<TaskDefinition>, BallistaError> {
     let session_config = session_config.update_from_key_value_pair(&multi_task.props);
@@ -418,6 +426,7 @@ pub fn get_task_definition_vec<
         scalar_functions: scalar_functions.clone(),
         aggregate_functions: aggregate_functions.clone(),
         window_functions: window_functions.clone(),
+        higher_order_functions: higher_order_functions.clone(),
     });
 
     let ctx = TaskContext::new(
@@ -425,6 +434,7 @@ pub fn get_task_definition_vec<
         uuid::Uuid::new_v4().to_string(),
         session_config.clone(),
         scalar_functions,
+        higher_order_functions,
         aggregate_functions,
         window_functions,
         runtime.clone(),
@@ -451,7 +461,12 @@ pub fn get_task_definition_vec<
                 job_id: job_id.clone(),
                 stage_id,
                 stage_attempt_num,
-                partition_id: task_id.partition_id as usize,
+                global_output_partition_ids: task_id
+                    .global_output_partition_ids
+                    .iter()
+                    .map(|pid| *pid as usize)
+                    .collect(),
+                vcores_consumed: task_id.vcores_consumed,
                 plan: reset_metrics_for_execution_plan(plan.clone())?,
                 launch_time,
                 session_id: session_id.clone(),
diff --git a/ballista/core/src/serde/scheduler/mod.rs b/ballista/core/src/serde/scheduler/mod.rs
index cb357e0037..970d3d2d07 100644
--- a/ballista/core/src/serde/scheduler/mod.rs
+++ b/ballista/core/src/serde/scheduler/mod.rs
@@ -69,6 +69,24 @@ pub struct PartitionId {
     pub partition_id: usize,
 }
 
+/// Locator for a task within an execution graph: (job, stage, task_id).
+///
+/// One task processes a partition slice, so the third component is
+/// `task_id` — the task's append-order slot in
+/// `RunningStage.task_infos`, not a partition index. Sibling to
+/// [`PartitionId`] — [`PartitionId`] identifies an operator output
+/// partition (shuffle location, etc.), [`TaskKey`] identifies a scheduled
+/// task attempt.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct TaskKey {
+    /// The job identifier.
+    pub job_id: JobId,
+    /// The stage identifier within the job.
+    pub stage_id: usize,
+    /// Task slot within the stage.
+    pub task_id: usize,
+}
+
 impl PartitionId {
     /// Creates a new partition ID with the given job, stage, and partition identifiers.
     pub fn new(job_id: &JobId, stage_id: usize, partition_id: usize) -> Self {
@@ -128,25 +146,41 @@ pub struct ExecutorMetadata {
     pub os_info: ExecutorOperatingSystemSpecification,
 }
 
-/// Specification of an executor, indicating executor resources, like total task slots.
+/// Specification of an executor, indicating its runtime-assigned vcore count.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 pub struct ExecutorSpecification {
-    /// Number of concurrent task slots available on this executor.
-    pub task_slots: u32,
+    /// Virtual cores assigned to this executor at runtime — analogous to YARN
+    /// vcores (`yarn.nodemanager.resource.cpu-vcores`) or Spark's
+    /// `spark.executor.cores`. Not physical `nproc`: may over- or
+    /// under-subscribe. One task per executor drives this many DataFusion
+    /// partitions in parallel through one plan-Arc.
+    pub vcores: u32,
 }
 
 impl Default for ExecutorSpecification {
     fn default() -> Self {
-        Self { task_slots: 1 }
+        Self { vcores: 1 }
     }
 }
 
 impl ExecutorSpecification {
-    /// Setting number of task slots (number of tasks that can be handled by this executor)
-    pub fn with_task_slots(mut self, task_slots: u32) -> Self {
-        self.task_slots = task_slots;
+    /// Set the vcore count. See [`ExecutorSpecification::vcores`].
+    pub fn with_vcores(mut self, vcores: u32) -> Self {
+        self.vcores = vcores;
         self
     }
+
+    /// Deprecated alias for [`Self::with_vcores`].
+    #[deprecated(note = "renamed to `with_vcores`")]
+    pub fn with_task_slots(self, task_slots: u32) -> Self {
+        self.with_vcores(task_slots)
+    }
+
+    /// Deprecated getter that returns [`Self::vcores`].
+    #[deprecated(note = "the `task_slots` field was renamed to `vcores`")]
+    pub fn task_slots(&self) -> u32 {
+        self.vcores
+    }
 }
 
 /// Operating system level specification of an executor
@@ -247,23 +281,48 @@ impl ExecutorOperatingSystemSpecification {
     }
 }
 
-/// Available resources for an executor, including total and available task slots.
+/// Executor vcore accounting: total assigned vs currently free.
 #[derive(Debug, Clone, Serialize)]
 pub struct ExecutorData {
     /// Unique executor identifier.
     pub executor_id: String,
-    /// Total number of task slots.
-    pub total_task_slots: u32,
-    /// Currently available task slots.
-    pub available_task_slots: u32,
+    /// Total vcores assigned to this executor. See
+    /// [`ExecutorSpecification::vcores`].
+    pub total_vcores: u32,
+    /// Currently free vcores (total minus reserved).
+    pub available_vcores: u32,
+}
+
+impl ExecutorData {
+    /// Deprecated getter that returns [`Self::total_vcores`].
+    #[deprecated(note = "the `total_task_slots` field was renamed to `total_vcores`")]
+    pub fn total_task_slots(&self) -> u32 {
+        self.total_vcores
+    }
+
+    /// Deprecated getter that returns [`Self::available_vcores`].
+    #[deprecated(
+        note = "the `available_task_slots` field was renamed to `available_vcores`"
+    )]
+    pub fn available_task_slots(&self) -> u32 {
+        self.available_vcores
+    }
 }
 
-/// Represents a change in executor task slot availability.
+/// Represents a change in an executor's free-vcore count.
 pub struct ExecutorDataChange {
     /// Unique executor identifier.
     pub executor_id: String,
-    /// Change in available task slots (positive or negative).
-    pub task_slots: i32,
+    /// Change in free vcores (positive to release, negative to reserve).
+    pub vcores: i32,
+}
+
+impl ExecutorDataChange {
+    /// Deprecated getter that returns [`Self::vcores`].
+    #[deprecated(note = "the `task_slots` field was renamed to `vcores`")]
+    pub fn task_slots(&self) -> i32 {
+        self.vcores
+    }
 }
 
 /// Summary of executed partition
@@ -460,7 +519,10 @@ impl ExecutePartitionResult {
 /// Definition of a task to be executed on an executor.
 #[derive(Clone, Debug)]
 pub struct TaskDefinition {
-    /// Unique task identifier.
+    /// Append-order slot of this task in `RunningStage.task_infos`. Not a
+    /// partition index — one task processes a slice of partitions given
+    /// by `global_output_partition_ids`. `(job_id, stage_id, task_id)` is
+    /// globally unique.
     pub task_id: usize,
     /// Current attempt number for this task.
     pub task_attempt_num: usize,
@@ -470,8 +532,16 @@ pub struct TaskDefinition {
     pub stage_id: usize,
     /// Current attempt number for the stage.
     pub stage_attempt_num: usize,
-    /// Partition to process.
-    pub partition_id: usize,
+    /// Global partition ids this task's restricted plan covers, in slice
+    /// order. `plan.output_partitioning().partition_count() == global_output_partition_ids.len()`
+    /// for pass-through shapes; writers use the slice to attach global
+    /// identity to shuffle files (with special-casing for plan-level
+    /// partitioning resets like SPM and RepartitionExec::Hash).
+    pub global_output_partition_ids: Vec<usize>,
+    /// Vcores this task consumed from the executor's budget at bind time.
+    /// Used to scale the task's memory pool so a task claiming N vcores
+    /// gets N/total_vcores of the executor's memory budget.
+    pub vcores_consumed: u32,
     /// Physical execution plan for this task.
     pub plan: Arc<dyn ExecutionPlan>,
     /// Timestamp when the task was launched.
diff --git a/ballista/core/src/serde/scheduler/to_proto.rs b/ballista/core/src/serde/scheduler/to_proto.rs
index 2f804a8ce0..2e78f9df50 100644
--- a/ballista/core/src/serde/scheduler/to_proto.rs
+++ b/ballista/core/src/serde/scheduler/to_proto.rs
@@ -124,95 +124,87 @@ pub fn hash_partitioning_to_proto(
     }
 }
 
-impl TryInto<protobuf::OperatorMetric> for &MetricValue {
+impl TryInto<operator_metric::Metric> for &MetricValue {
     type Error = BallistaError;
 
-    fn try_into(self) -> Result<protobuf::OperatorMetric, Self::Error> {
+    fn try_into(self) -> Result<operator_metric::Metric, Self::Error> {
         match self {
-            MetricValue::OutputRows(count) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::OutputRows(count.value() as u64)),
-            }),
-            MetricValue::ElapsedCompute(time) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::ElapseTime(time.value() as u64)),
-            }),
-            MetricValue::SpillCount(count) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::SpillCount(count.value() as u64)),
-            }),
-            MetricValue::SpilledBytes(count) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::SpilledBytes(count.value() as u64)),
-            }),
-            MetricValue::SpilledRows(count) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::SpilledRows(count.value() as u64)),
-            }),
-            MetricValue::CurrentMemoryUsage(gauge) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::CurrentMemoryUsage(
-                    gauge.value() as u64
-                )),
-            }),
-            MetricValue::Count { name, count } => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::Count(NamedCount {
+            MetricValue::OutputRows(count) => {
+                Ok(operator_metric::Metric::OutputRows(count.value() as u64))
+            }
+            MetricValue::ElapsedCompute(time) => {
+                Ok(operator_metric::Metric::ElapseTime(time.value() as u64))
+            }
+            MetricValue::SpillCount(count) => {
+                Ok(operator_metric::Metric::SpillCount(count.value() as u64))
+            }
+            MetricValue::SpilledBytes(count) => {
+                Ok(operator_metric::Metric::SpilledBytes(count.value() as u64))
+            }
+            MetricValue::SpilledRows(count) => {
+                Ok(operator_metric::Metric::SpilledRows(count.value() as u64))
+            }
+            MetricValue::CurrentMemoryUsage(gauge) => Ok(
+                operator_metric::Metric::CurrentMemoryUsage(gauge.value() as u64),
+            ),
+            MetricValue::Count { name, count } => {
+                Ok(operator_metric::Metric::Count(NamedCount {
                     name: name.to_string(),
                     value: count.value() as u64,
-                })),
-            }),
-            MetricValue::Gauge { name, gauge } => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::Gauge(NamedGauge {
+                }))
+            }
+            MetricValue::Gauge { name, gauge } => {
+                Ok(operator_metric::Metric::Gauge(NamedGauge {
                     name: name.to_string(),
                     value: gauge.value() as u64,
-                })),
-            }),
-            MetricValue::Time { name, time } => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::Time(NamedTime {
+                }))
+            }
+            MetricValue::Time { name, time } => {
+                Ok(operator_metric::Metric::Time(NamedTime {
                     name: name.to_string(),
                     value: time.value() as u64,
-                })),
-            }),
-            MetricValue::StartTimestamp(timestamp) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::StartTimestamp(
+                }))
+            }
+            MetricValue::StartTimestamp(timestamp) => {
+                Ok(operator_metric::Metric::StartTimestamp(
                     timestamp
                         .value()
                         .and_then(|m| m.timestamp_nanos_opt())
                         .unwrap_or(0),
-                )),
-            }),
-            MetricValue::EndTimestamp(timestamp) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::EndTimestamp(
+                ))
+            }
+            MetricValue::EndTimestamp(timestamp) => {
+                Ok(operator_metric::Metric::EndTimestamp(
                     timestamp
                         .value()
                         .and_then(|m| m.timestamp_nanos_opt())
                         .unwrap_or(0),
-                )),
-            }),
-            MetricValue::OutputBytes(count) => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::OutputBytes(count.value() as u64)),
-            }),
+                ))
+            }
+            MetricValue::OutputBytes(count) => {
+                Ok(operator_metric::Metric::OutputBytes(count.value() as u64))
+            }
             MetricValue::PruningMetrics {
                 name,
                 pruning_metrics,
-            } => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::PruningMetrics(
-                    NamedPruningMetrics {
-                        name: name.to_string(),
-                        pruned: pruning_metrics.pruned() as u64,
-                        matched: pruning_metrics.matched() as u64,
-                    },
-                )),
-            }),
+            } => Ok(operator_metric::Metric::PruningMetrics(
+                NamedPruningMetrics {
+                    name: name.to_string(),
+                    pruned: pruning_metrics.pruned() as u64,
+                    matched: pruning_metrics.matched() as u64,
+                },
+            )),
             MetricValue::Ratio {
                 name,
                 ratio_metrics,
-            } => Ok(protobuf::OperatorMetric {
-                metric: Some(operator_metric::Metric::Ratio(NamedRatio {
-                    name: name.to_string(),
-                    part: ratio_metrics.part() as u64,
-                    total: ratio_metrics.total() as u64,
-                })),
-            }),
-            MetricValue::OutputBatches(count) => Ok(protobuf::OperatorMetric {
-                metric: Some(
-                    operator_metric::Metric::OutputBatches(count.value() as u64),
-                ),
-            }),
+            } => Ok(operator_metric::Metric::Ratio(NamedRatio {
+                name: name.to_string(),
+                part: ratio_metrics.part() as u64,
+                total: ratio_metrics.total() as u64,
+            })),
+            MetricValue::OutputBatches(count) => {
+                Ok(operator_metric::Metric::OutputBatches(count.value() as u64))
+            }
             // at the moment there there is no way to serialize custom metrics
             // thus at the moment we can't support it
             MetricValue::Custom { .. } => Err(BallistaError::General(String::from(
@@ -228,7 +220,13 @@ impl TryInto<protobuf::OperatorMetricsSet> for MetricsSet {
     fn try_into(self) -> Result<protobuf::OperatorMetricsSet, Self::Error> {
         let metrics = self
             .iter()
-            .map(|m| m.value().try_into())
+            .map(|m| {
+                let metric: operator_metric::Metric = m.value().try_into()?;
+                Ok(protobuf::OperatorMetric {
+                    metric: Some(metric),
+                    partition: m.partition().map(|p| p as u32),
+                })
+            })
             .collect::<Result<Vec<_>, BallistaError>>()?;
         Ok(protobuf::OperatorMetricsSet { metrics })
     }
@@ -252,12 +250,10 @@ impl Into<protobuf::ExecutorMetadata> for ExecutorMetadata {
 impl Into<protobuf::ExecutorSpecification> for ExecutorSpecification {
     fn into(self) -> protobuf::ExecutorSpecification {
         protobuf::ExecutorSpecification {
-            resources: vec![protobuf::executor_resource::Resource::TaskSlots(
-                self.task_slots,
-            )]
-            .into_iter()
-            .map(|r| protobuf::ExecutorResource { resource: Some(r) })
-            .collect(),
+            resources: vec![protobuf::executor_resource::Resource::Vcores(self.vcores)]
+                .into_iter()
+                .map(|r| protobuf::ExecutorResource { resource: Some(r) })
+                .collect(),
         }
     }
 }
@@ -292,11 +288,9 @@ impl Into<protobuf::ExecutorData> for ExecutorData {
         protobuf::ExecutorData {
             executor_id: self.executor_id,
             resources: vec![ExecutorResourcePair {
-                total: protobuf::executor_resource::Resource::TaskSlots(
-                    self.total_task_slots,
-                ),
-                available: protobuf::executor_resource::Resource::TaskSlots(
-                    self.available_task_slots,
+                total: protobuf::executor_resource::Resource::Vcores(self.total_vcores),
+                available: protobuf::executor_resource::Resource::Vcores(
+                    self.available_vcores,
                 ),
             }]
             .into_iter()
diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs
index 8428884680..34b5b05f10 100644
--- a/ballista/core/src/utils.rs
+++ b/ballista/core/src/utils.rs
@@ -24,6 +24,7 @@ use datafusion::arrow::ipc::CompressionType;
 use datafusion::arrow::ipc::writer::IpcWriteOptions;
 use datafusion::arrow::ipc::writer::StreamWriter;
 use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::error::DataFusionError;
 use datafusion::execution::context::{SessionConfig, SessionState};
 use datafusion::execution::runtime_env::RuntimeEnvBuilder;
 use datafusion::execution::session_state::SessionStateBuilder;
@@ -72,6 +73,10 @@ pub struct GrpcClientConfig {
     pub io_retries_times: u8,
     /// Wait time in milliseconds between IO retries.
     pub io_retry_wait_time_ms: u64,
+    /// HTTP/2 initial connection-level flow-control window in bytes. 0 = tonic default.
+    pub initial_connection_window_size: u32,
+    /// HTTP/2 initial stream-level flow-control window in bytes. 0 = tonic default.
+    pub initial_stream_window_size: u32,
 }
 
 impl From<&BallistaConfig> for GrpcClientConfig {
@@ -87,6 +92,9 @@ impl From<&BallistaConfig> for GrpcClientConfig {
             max_message_size: config.grpc_client_max_message_size(),
             io_retries_times: config.io_retries_times() as u8,
             io_retry_wait_time_ms: config.io_retry_wait_time_ms() as u64,
+            initial_connection_window_size: config
+                .grpc_client_initial_connection_window_size(),
+            initial_stream_window_size: config.grpc_client_initial_stream_window_size(),
         }
     }
 }
@@ -102,6 +110,8 @@ impl Default for GrpcClientConfig {
             max_message_size: 16 * 1024 * 1024,
             io_retries_times: 3,
             io_retry_wait_time_ms: 3000,
+            initial_connection_window_size: 67108864,
+            initial_stream_window_size: 16777216,
         }
     }
 }
@@ -168,6 +178,17 @@ pub fn default_config_producer() -> SessionConfig {
     SessionConfig::new_with_ballista()
 }
 
+/// Creates [IpcWriteOptions] using the compression codec configured in the BallistaConfig
+pub fn create_write_options(
+    compression_type: Option<CompressionType>,
+) -> std::result::Result<IpcWriteOptions, DataFusionError> {
+    IpcWriteOptions::default()
+        .try_with_compression(compression_type)
+        .map_err(|err| {
+            DataFusionError::Internal(format!("Failed to set compression codec: {err}"))
+        })
+}
+
 /// Stream data to disk in Arrow IPC format.
 ///
 /// Batches are read from the async stream and forwarded through a bounded
@@ -178,6 +199,7 @@ pub async fn write_stream_to_disk(
     path: &Path,
     disk_write_metric: &metrics::Time,
     channel_capacity: usize,
+    compression_type: Option<CompressionType>,
 ) -> Result<PartitionStats> {
     let schema = stream.schema();
     let path_owned = path.to_owned();
@@ -191,8 +213,7 @@ pub async fn write_stream_to_disk(
             BallistaError::IoError(e)
         })?);
 
-        let options = IpcWriteOptions::default()
-            .try_with_compression(Some(CompressionType::LZ4_FRAME))?;
+        let options = create_write_options(compression_type)?;
 
         let mut writer =
             StreamWriter::try_new_with_options(file, schema.as_ref(), options)?;
@@ -294,7 +315,7 @@ where
 {
     let endpoint = tonic::transport::Endpoint::new(dst)?;
     if let Some(config) = config {
-        Ok(endpoint
+        let mut endpoint = endpoint
             .connect_timeout(Duration::from_secs(config.connect_timeout_seconds))
             .timeout(Duration::from_secs(config.timeout_seconds))
             .tcp_nodelay(true)
@@ -303,7 +324,17 @@ where
                 config.http2_keepalive_interval_seconds,
             ))
             .keep_alive_timeout(Duration::from_secs(20))
-            .keep_alive_while_idle(true))
+            .keep_alive_while_idle(true);
+        if config.initial_connection_window_size > 0 {
+            endpoint = endpoint.initial_connection_window_size(Some(
+                config.initial_connection_window_size,
+            ));
+        }
+        if config.initial_stream_window_size > 0 {
+            endpoint = endpoint
+                .initial_stream_window_size(Some(config.initial_stream_window_size));
+        }
+        Ok(endpoint)
     } else {
         Ok(endpoint)
     }
@@ -396,6 +427,8 @@ mod tests {
             max_message_size: 16 * 1024 * 1024,
             io_retries_times: 3,
             io_retry_wait_time_ms: 3000,
+            initial_connection_window_size: 67108864,
+            initial_stream_window_size: 16777216,
         };
         let result = create_grpc_client_endpoint("http://localhost:50051", Some(&config));
         assert!(result.is_ok());
diff --git a/ballista/executor/Cargo.toml b/ballista/executor/Cargo.toml
index c80e68b9cc..44740aaa69 100644
--- a/ballista/executor/Cargo.toml
+++ b/ballista/executor/Cargo.toml
@@ -19,7 +19,7 @@
 name = "ballista-executor"
 description = "Ballista Distributed Compute - Executor"
 license = "Apache-2.0"
-version = "53.0.0"
+version = "54.0.0"
 homepage = "https://datafusion.apache.org/ballista/"
 repository = "https://github.com/apache/datafusion-ballista"
 readme = "README.md"
@@ -34,7 +34,7 @@ required-features = ["build-binary"]
 
 [features]
 arrow-ipc-optimizations = []
-build-binary = ["clap", "tracing-subscriber", "tracing-appender", "tracing", "ballista-core/build-binary", "mimalloc"]
+build-binary = ["clap", "tracing-subscriber", "tracing-appender", "tracing", "ballista-core/build-binary", "mimalloc", "dep:axum"]
 default = ["arrow-ipc-optimizations", "build-binary"]
 spark-compat = ["ballista-core/spark-compat"]
 
@@ -42,7 +42,8 @@ spark-compat = ["ballista-core/spark-compat"]
 arrow = { workspace = true }
 arrow-flight = { workspace = true }
 async-trait = { workspace = true }
-ballista-core = { path = "../core", version = "53.0.0" }
+axum = { version = "0.8.9", optional = true }
+ballista-core = { path = "../core", version = "54.0.0" }
 bytesize = "2"
 clap = { workspace = true, optional = true }
 dashmap = { workspace = true }
@@ -50,6 +51,7 @@ datafusion = { workspace = true }
 datafusion-proto = { workspace = true }
 futures = { workspace = true }
 log = { workspace = true }
+lru = "0.18"
 memory-stats = { version = "1.2.0", features = ["always_use_statm"] }
 mimalloc = { workspace = true, optional = true }
 parking_lot = { workspace = true }
diff --git a/ballista/executor/src/bin/main.rs b/ballista/executor/src/bin/main.rs
index cbeb93b89e..14f77349e4 100644
--- a/ballista/executor/src/bin/main.rs
+++ b/ballista/executor/src/bin/main.rs
@@ -18,6 +18,7 @@
 //! Ballista Rust executor binary.
 
 use ballista_core::config::LogRotationPolicy;
+use ballista_core::error::BallistaError;
 use ballista_core::object_store::{
     runtime_env_with_s3_support, session_config_with_s3_support,
 };
@@ -25,8 +26,10 @@ use ballista_executor::config::Config;
 use ballista_executor::executor_process::{
     ExecutorProcessConfig, start_executor_process,
 };
+use ballista_executor::health::spawn_health_server;
 use clap::Parser;
 use std::env;
+use std::net::SocketAddr;
 use std::sync::Arc;
 use tracing_subscriber::EnvFilter;
 
@@ -38,6 +41,8 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 async fn main() -> ballista_core::error::Result<()> {
     // parse command-line arguments
     let opt = Config::parse();
+    let bind_host = opt.bind_host.clone();
+    let bind_health_port = opt.bind_health_port;
 
     let mut config: ExecutorProcessConfig = opt.try_into()?;
     config.override_config_producer = Some(Arc::new(session_config_with_s3_support));
@@ -76,5 +81,25 @@ async fn main() -> ballista_core::error::Result<()> {
         tracing.init();
     }
 
-    start_executor_process(Arc::new(config)).await
+    // Kubernetes-style health probes are a concern of the standalone binary,
+    // not of the library, so wire them here on top of the config's shared
+    // ExecutorHealth handle. `/healthz` is process-liveness, `/readyz`
+    // reflects heartbeat state (see `ballista_executor::health`).
+    let health = config.health.clone();
+    let health_addr: SocketAddr = format!("{bind_host}:{bind_health_port}")
+        .parse()
+        .map_err(|e| {
+            BallistaError::General(format!("invalid --bind-health-port address: {e}"))
+        })?;
+    let (health_shutdown_tx, health_shutdown_rx) = tokio::sync::oneshot::channel();
+    let health_handle = spawn_health_server(health_addr, health, health_shutdown_rx);
+
+    let result = start_executor_process(Arc::new(config)).await;
+
+    // Ask the health server to shut down so the process can exit cleanly.
+    let _ = health_shutdown_tx.send(());
+    if let Err(e) = health_handle.await {
+        log::warn!("health server task join error: {e}");
+    }
+    result
 }
diff --git a/ballista/executor/src/client_pool.rs b/ballista/executor/src/client_pool.rs
index ed1260c0a0..cb87a012b3 100644
--- a/ballista/executor/src/client_pool.rs
+++ b/ballista/executor/src/client_pool.rs
@@ -185,6 +185,8 @@ impl BallistaClientPool for DefaultBallistaClientPool {
                     customize_endpoint,
                     config.io_retries_times,
                     config.io_retry_wait_time_ms,
+                    config.initial_connection_window_size,
+                    config.initial_stream_window_size,
                 )
                 .await?
             }
diff --git a/ballista/executor/src/collect.rs b/ballista/executor/src/collect.rs
index 40fae6cc58..31d2bc28ca 100644
--- a/ballista/executor/src/collect.rs
+++ b/ballista/executor/src/collect.rs
@@ -18,9 +18,9 @@
 //! The CollectExec operator retrieves results from the cluster and returns them as a single
 //! vector of [`RecordBatch`](datafusion::arrow::record_batch::RecordBatch).
 
+use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
-use std::{any::Any, pin::Pin};
 
 use datafusion::arrow::{datatypes::SchemaRef, record_batch::RecordBatch};
 use datafusion::error::DataFusionError;
@@ -79,10 +79,6 @@ impl ExecutionPlan for CollectExec {
         "CollectExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.plan.schema()
     }
@@ -125,7 +121,7 @@ impl ExecutionPlan for CollectExec {
         }))
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         self.plan.partition_statistics(partition)
     }
 }
diff --git a/ballista/executor/src/config.rs b/ballista/executor/src/config.rs
index 8ddeb4ab3c..4a7b638d34 100644
--- a/ballista/executor/src/config.rs
+++ b/ballista/executor/src/config.rs
@@ -71,6 +71,13 @@ pub struct Config {
     /// Port for the executor's gRPC service (used for task management).
     #[arg(long, default_value_t = 50052, help = "Grpc service bind port.")]
     pub bind_grpc_port: u16,
+    /// Port for the executor's HTTP health server (/healthz and /readyz).
+    #[arg(
+        long,
+        default_value_t = 50053,
+        help = "HTTP port for the executor's Kubernetes health probes (/healthz and /readyz)."
+    )]
+    pub bind_health_port: u16,
     /// Timeout in seconds for establishing connection to scheduler (0 = fail immediately).
     #[arg(
         long,
@@ -81,14 +88,28 @@ pub struct Config {
     /// Directory for storing temporary shuffle data and IPC files.
     #[arg(long, help = "Directory for temporary IPC files")]
     pub work_dir: Option<String>,
-    /// Maximum number of concurrent tasks this executor can run (0 = use all CPU cores).
+    /// Number of virtual cores (vcores) this executor advertises to the scheduler
+    /// (0 = use all physical CPU cores). One task per executor drives this many
+    /// DataFusion partitions in parallel through one plan-Arc. Analogous to YARN
+    /// vcores (`yarn.nodemanager.resource.cpu-vcores`) or Spark's
+    /// `spark.executor.cores`.
     #[arg(
         short = 'c',
         long,
         default_value_t = 0,
-        help = "Max concurrent tasks (defaults to all available cores if left as zero)."
+        help = "Virtual cores advertised to the scheduler (defaults to physical CPU count if zero)."
     )]
-    pub concurrent_tasks: usize,
+    pub vcores: usize,
+    /// Deprecated alias for `--vcores`. Retained for one release to keep
+    /// in-place cluster upgrades working; will be removed in the next
+    /// major release.
+    #[arg(
+        long = "concurrent-tasks",
+        hide = true,
+        conflicts_with = "vcores",
+        help = "[deprecated] renamed to --vcores"
+    )]
+    pub concurrent_tasks: Option<usize>,
     /// Task scheduling policy: pull-staged (executor polls) or push-staged (scheduler pushes).
     #[arg(short = 's', long, default_value_t = ballista_core::config::TaskSchedulingPolicy::default(), help = "The task scheduling policy used by scheduler. Configuration must match with scheduler configured policy.")]
     pub task_scheduling_policy: ballista_core::config::TaskSchedulingPolicy,
@@ -163,19 +184,29 @@ pub struct Config {
     )]
     pub metric_collection_policy: ExecutorMetricCollectionPolicy,
     /// Optional total memory budget for the executor. Accepts human-readable
-    /// values like "8GB", "512MiB", or a plain byte count. When set, every
-    /// task gets a FairSpillPool of size `memory_pool_size / concurrent_tasks`.
+    /// values like "8GB", "512MiB", or a plain byte count. When set, each
+    /// per-vcore FairSpillPool gets `memory_pool_size / vcores`.
     #[arg(
         long,
         value_parser = parse_memory_pool_size,
-        help = "Optional total executor memory budget (e.g. \"8GB\", \"512MiB\"). Each concurrent task receives an equal share."
+        help = "Optional total executor memory budget (e.g. \"8GB\", \"512MiB\"). Split evenly across vcores."
     )]
     pub memory_pool_size: Option<u64>,
-    /// Number of seconds established client connection should be cached if not used (0 means no cache)
+    /// Maximum number of sessions whose shared runtime state (object-store
+    /// clients, Parquet footer cache) is retained on the executor (LRU). `0`
+    /// disables caching.
     #[arg(
         long,
-        default_value_t = 0,
-        help = "Number of seconds established client connection should be cached if not used (0 means no cache, connection will be disposed)."
+        default_value_t = 16,
+        help = "Max number of sessions whose shared runtime state (object-store clients, Parquet footer cache) is retained on the executor (LRU). 0 disables caching."
+    )]
+    pub session_runtime_cache_capacity: usize,
+    /// Number of seconds an established client connection is cached while idle
+    /// (0 disables the cache, so every shuffle fetch opens a new connection)
+    #[arg(
+        long,
+        default_value_t = 30,
+        help = "Number of seconds established client connection should be cached if not used (0 means no cache, connection will be disposed; a shuffle-heavy query can then exhaust the host's ephemeral ports)."
     )]
     pub client_ttl: u64,
 }
@@ -184,6 +215,15 @@ impl TryFrom<Config> for ExecutorProcessConfig {
     type Error = BallistaError;
 
     fn try_from(opt: Config) -> Result<Self, Self::Error> {
+        let vcores = match opt.concurrent_tasks {
+            Some(value) => {
+                eprintln!(
+                    "warning: --concurrent-tasks is deprecated; use --vcores instead"
+                );
+                value
+            }
+            None => opt.vcores,
+        };
         Ok(ExecutorProcessConfig {
             special_mod_log_level: opt.log_level_setting,
             external_host: opt.external_host,
@@ -193,7 +233,7 @@ impl TryFrom<Config> for ExecutorProcessConfig {
             scheduler_host: opt.scheduler_host,
             scheduler_port: opt.scheduler_port,
             scheduler_connect_timeout_seconds: opt.scheduler_connect_timeout_seconds,
-            concurrent_tasks: opt.concurrent_tasks,
+            vcores,
             task_scheduling_policy: opt.task_scheduling_policy,
             work_dir: opt.work_dir,
             log_dir: opt.log_dir,
@@ -207,6 +247,7 @@ impl TryFrom<Config> for ExecutorProcessConfig {
             executor_heartbeat_interval_seconds: opt.executor_heartbeat_interval_seconds,
             metric_collection_policy: opt.metric_collection_policy,
             memory_pool_size: opt.memory_pool_size,
+            session_runtime_cache_capacity: opt.session_runtime_cache_capacity,
             override_execution_engine: None,
             override_function_registry: None,
             override_config_producer: None,
@@ -216,6 +257,7 @@ impl TryFrom<Config> for ExecutorProcessConfig {
             override_arrow_flight_service: None,
             override_create_grpc_client_endpoint: None,
             client_ttl: opt.client_ttl,
+            health: crate::health::ExecutorHealth::new(),
         })
     }
 }
@@ -246,4 +288,20 @@ mod tests {
         assert!(parse_memory_pool_size("banana").is_err());
         assert!(parse_memory_pool_size("").is_err());
     }
+
+    /// The client pool is what keeps shuffle fetches from opening a fresh TCP
+    /// connection each time; with it off, one shuffle-heavy query can exhaust
+    /// the host's ephemeral ports and take the executor down with it. Pin the
+    /// default on so it cannot regress to the disabled state unnoticed.
+    #[cfg(feature = "build-binary")]
+    #[test]
+    fn client_pool_is_enabled_by_default() {
+        use clap::Parser;
+        let opt = super::Config::parse_from(["ballista-executor"]);
+        assert!(
+            opt.client_ttl > 0,
+            "client_ttl must default to a pooling value, got {}",
+            opt.client_ttl
+        );
+    }
 }
diff --git a/ballista/executor/src/execution_engine.rs b/ballista/executor/src/execution_engine.rs
index 6ce96fc8be..5633a6ddeb 100644
--- a/ballista/executor/src/execution_engine.rs
+++ b/ballista/executor/src/execution_engine.rs
@@ -25,13 +25,20 @@ use ballista_core::client_pool::BallistaClientPool;
 use ballista_core::execution_plans::sort_shuffle::SortShuffleWriterExec;
 use ballista_core::execution_plans::{ShuffleReaderExec, ShuffleWriterExec};
 use ballista_core::serde::protobuf::ShuffleWritePartition;
+use ballista_core::serde::scheduler::PartitionStats;
 use ballista_core::{JobId, utils};
+use datafusion::arrow::array::{
+    Array, StringArray, StructArray, UInt32Array, UInt64Array,
+};
 use datafusion::common::tree_node::{Transformed, TreeNode};
 use datafusion::error::{DataFusionError, Result};
 use datafusion::execution::context::TaskContext;
 use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::display::DisplayableExecutionPlan;
 use datafusion::physical_plan::metrics::MetricsSet;
 use datafusion::prelude::SessionConfig;
+use futures::stream::TryStreamExt;
+use log::info;
 use std::fmt::{Debug, Display};
 use std::sync::Arc;
 
@@ -46,11 +53,13 @@ pub trait ExecutionEngine: Sync + Send {
     ///
     /// The returned executor will be responsible for executing the given
     /// plan partition and writing shuffle output to the specified work directory.
+    #[allow(clippy::too_many_arguments)]
     fn create_query_stage_exec(
         &self,
         job_id: JobId,
         stage_id: usize,
-        partition_id: usize,
+        task_id: usize,
+        global_output_partition_ids: Vec<usize>,
         plan: Arc<dyn ExecutionPlan>,
         work_dir: &str,
         config: &SessionConfig,
@@ -65,13 +74,13 @@ pub trait ExecutionEngine: Sync + Send {
 /// Arrow IPC format. Subsequent stages read these results via ShuffleReaderExec.
 #[async_trait::async_trait]
 pub trait QueryStageExecutor: Sync + Send + Debug + Display {
-    /// Executes a single partition of this query stage.
+    /// Executes this query stage's assigned partition slice.
     ///
     /// Returns metadata about the shuffle partitions written to disk,
     /// including file paths and statistics.
     async fn execute_query_stage(
         &self,
-        input_partition: usize,
+        task_id: usize,
         context: Arc<TaskContext>,
     ) -> Result<Vec<ShuffleWritePartition>>;
 
@@ -106,14 +115,15 @@ impl ExecutionEngine for DefaultExecutionEngine {
         &self,
         job_id: JobId,
         stage_id: usize,
-        _partition_id: usize,
+        task_id: usize,
+        global_output_partition_ids: Vec<usize>,
         plan: Arc<dyn ExecutionPlan>,
         work_dir: &str,
         _config: &SessionConfig,
     ) -> Result<Arc<dyn QueryStageExecutor>> {
         let plan = plan
             .transform(|p| {
-                if let Some(reader) = p.as_any().downcast_ref::<ShuffleReaderExec>() {
+                if let Some(reader) = p.downcast_ref::<ShuffleReaderExec>() {
                     match &self.client_pool {
                         Some(client_pool) => Ok(Transformed::yes(Arc::new(
                             reader
@@ -125,6 +135,13 @@ impl ExecutionEngine for DefaultExecutionEngine {
                         ))),
                     }
                 } else {
+                    // Scan restriction is scheduler-side (see
+                    // ballista/scheduler/src/state/task_builder.rs). The plan
+                    // arriving here is shrink-restricted to slice.len()
+                    // partitions; the writer walks its child plan to attach
+                    // global identity, using `global_output_partition_ids` for the
+                    // pass-through case and detecting plan-level partitioning
+                    // resets (SPM, RepartitionExec::Hash) for the rest.
                     Ok(Transformed::no(p))
                 }
             })?
@@ -132,22 +149,22 @@ impl ExecutionEngine for DefaultExecutionEngine {
 
         // the query plan created by the scheduler always starts with a shuffle writer
         // (either ShuffleWriterExec or SortShuffleWriterExec)
-        if let Some(shuffle_writer) = plan.as_any().downcast_ref::<ShuffleWriterExec>() {
-            // recreate the shuffle writer with the correct working directory
+        if let Some(shuffle_writer) = plan.downcast_ref::<ShuffleWriterExec>() {
             let exec = ShuffleWriterExec::try_new(
                 job_id,
                 stage_id,
                 plan.children()[0].clone(),
                 work_dir.to_string(),
                 shuffle_writer.shuffle_output_partitioning().cloned(),
-            )?;
+            )?
+            .with_task_id(task_id)
+            .with_global_output_partition_ids(global_output_partition_ids);
             Ok(Arc::new(DefaultQueryStageExec::new(
                 ShuffleWriterVariant::Hash(exec),
             )))
         } else if let Some(sort_shuffle_writer) =
-            plan.as_any().downcast_ref::<SortShuffleWriterExec>()
+            plan.downcast_ref::<SortShuffleWriterExec>()
         {
-            // recreate the sort shuffle writer with the correct working directory
             let exec = SortShuffleWriterExec::try_new(
                 job_id,
                 stage_id,
@@ -155,7 +172,9 @@ impl ExecutionEngine for DefaultExecutionEngine {
                 work_dir.to_string(),
                 sort_shuffle_writer.shuffle_output_partitioning().clone(),
                 sort_shuffle_writer.config().clone(),
-            )?;
+            )?
+            .with_task_id(task_id)
+            .with_global_output_partition_ids(global_output_partition_ids);
             Ok(Arc::new(DefaultQueryStageExec::new(
                 ShuffleWriterVariant::Sort(exec),
             )))
@@ -233,23 +252,32 @@ impl Display for DefaultQueryStageExec {
 impl QueryStageExecutor for DefaultQueryStageExec {
     async fn execute_query_stage(
         &self,
-        input_partition: usize,
+        task_id: usize,
         context: Arc<TaskContext>,
     ) -> Result<Vec<ShuffleWritePartition>> {
-        match &self.shuffle_writer {
-            ShuffleWriterVariant::Hash(writer) => {
-                writer
-                    .clone()
-                    .execute_shuffle_write(input_partition, context)
-                    .await
-            }
-            ShuffleWriterVariant::Sort(writer) => {
-                writer
-                    .clone()
-                    .execute_shuffle_write(input_partition, context)
-                    .await
-            }
-        }
+        let (plan_arc, is_sort_shuffle): (Arc<dyn ExecutionPlan>, bool) =
+            match &self.shuffle_writer {
+                ShuffleWriterVariant::Hash(writer) => (Arc::new(writer.clone()), false),
+                ShuffleWriterVariant::Sort(writer) => (Arc::new(writer.clone()), true),
+            };
+        info!(
+            "executor plan pre-run (task_id={task_id}):\n{}",
+            DisplayableExecutionPlan::new(plan_arc.as_ref()).indent(true)
+        );
+
+        // Both variants share the same coordinator+oneshot handoff shape via
+        // `execute(N)` — drive K parallel calls so every oneshot receiver is
+        // taken concurrently and each output partition's summaries flow out
+        // as soon as its files are closed.
+        let result =
+            drive_shuffle_writer_stage(plan_arc.clone(), context, is_sort_shuffle).await;
+
+        info!(
+            "executor plan post-run (task_id={task_id}, ok={}):\n{}",
+            result.is_ok(),
+            DisplayableExecutionPlan::with_metrics(plan_arc.as_ref()).indent(true)
+        );
+        result
     }
 
     fn collect_plan_metrics(&self) -> Vec<MetricsSet> {
@@ -259,3 +287,156 @@ impl QueryStageExecutor for DefaultQueryStageExec {
         }
     }
 }
+
+/// Spawn K parallel `plan.execute(N, ctx)` calls against a shuffle writer,
+/// collect metadata batches from each, and turn them back into
+/// `Vec<ShuffleWritePartition>`. All K streams must be driven concurrently
+/// so the writer's internal coordinator sees every oneshot receiver taken.
+///
+/// `is_sort_shuffle` is stamped onto every summary produced from the batches
+/// — the metadata schema doesn't carry the flag (it's a handoff-only shape),
+/// but the reader side needs it in `PartitionLocation` to pick the right
+/// on-disk layout. The caller knows the variant from `ShuffleWriterVariant`.
+async fn drive_shuffle_writer_stage(
+    plan: Arc<dyn ExecutionPlan>,
+    context: Arc<TaskContext>,
+    is_sort_shuffle: bool,
+) -> Result<Vec<ShuffleWritePartition>> {
+    let k = plan.properties().output_partitioning().partition_count();
+
+    let mut stream_futures = Vec::with_capacity(k);
+    for n in 0..k {
+        let plan = plan.clone();
+        let ctx = context.clone();
+        stream_futures.push(tokio::spawn(async move {
+            let mut stream = plan.execute(n, ctx)?;
+            let mut batches = Vec::new();
+            while let Some(batch) = stream.try_next().await? {
+                batches.push(batch);
+            }
+            metadata_batches_to_summaries(batches, is_sort_shuffle)
+        }));
+    }
+
+    let mut summaries = Vec::with_capacity(k);
+    for handle in stream_futures {
+        let per_partition = handle.await.map_err(|e| {
+            DataFusionError::Execution(format!("shuffle writer drain panicked: {e}"))
+        })??;
+        summaries.extend(per_partition);
+    }
+    // Drop summaries for output slots that produced no data. The coordinator
+    // uses zero-content entries as sentinels so `execute(N)` streams don't
+    // stall on an unfilled oneshot; those must not become PartitionLocations
+    // the scheduler tries to fetch.
+    summaries.retain(|s| s.num_bytes > 0);
+    Ok(summaries)
+}
+
+/// Convert the writer's metadata batches (one per output partition, each
+/// with a single row) back into `ShuffleWritePartition` summaries.
+fn metadata_batches_to_summaries(
+    batches: Vec<datafusion::arrow::record_batch::RecordBatch>,
+    is_sort_shuffle: bool,
+) -> Result<Vec<ShuffleWritePartition>> {
+    let stats_fields = PartitionStats::default().arrow_struct_fields();
+    let num_rows_idx = stats_fields
+        .iter()
+        .position(|f| f.name() == "num_rows")
+        .expect("num_rows field present in PartitionStats");
+    let num_batches_idx = stats_fields
+        .iter()
+        .position(|f| f.name() == "num_batches")
+        .expect("num_batches field present in PartitionStats");
+    let num_bytes_idx = stats_fields
+        .iter()
+        .position(|f| f.name() == "num_bytes")
+        .expect("num_bytes field present in PartitionStats");
+
+    let mut out = Vec::new();
+    for batch in batches {
+        let partition_col = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<UInt32Array>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata batch: partition column not UInt32".into(),
+                )
+            })?;
+        let _path_col = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata batch: path column not Utf8".into(),
+                )
+            })?;
+        let file_id_col = batch
+            .column(2)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata batch: file_id column not UInt64".into(),
+                )
+            })?;
+        let stats_col = batch
+            .column(3)
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata batch: stats column not Struct".into(),
+                )
+            })?;
+        let num_rows_arr = stats_col
+            .column(num_rows_idx)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata stats.num_rows not UInt64".into(),
+                )
+            })?;
+        let num_batches_arr = stats_col
+            .column(num_batches_idx)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata stats.num_batches not UInt64".into(),
+                )
+            })?;
+        let num_bytes_arr = stats_col
+            .column(num_bytes_idx)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(
+                    "shuffle metadata stats.num_bytes not UInt64".into(),
+                )
+            })?;
+
+        for row in 0..batch.num_rows() {
+            let file_id = if file_id_col.is_null(row) {
+                None
+            } else {
+                Some(file_id_col.value(row))
+            };
+            out.push(ShuffleWritePartition {
+                partition_id: partition_col.value(row) as u64,
+                num_batches: num_batches_arr.value(row),
+                num_rows: num_rows_arr.value(row),
+                num_bytes: num_bytes_arr.value(row),
+                file_id,
+                is_sort_shuffle,
+            });
+        }
+    }
+    Ok(out)
+}
+
+// TODO: port these tests to scheduler/src/state/task_builder.rs (they used
+// to cover the executor-side restrict function that has moved scheduler-side).
diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs
index f64fae3014..c4fac99d2a 100644
--- a/ballista/executor/src/execution_loop.rs
+++ b/ballista/executor/src/execution_loop.rs
@@ -23,7 +23,7 @@
 
 use crate::cpu_bound_executor::DedicatedExecutor;
 use crate::executor::Executor;
-use crate::executor_process::remove_job_dir;
+use crate::executor_process::remove_job_data;
 use crate::{TaskExecutionTimes, as_task_status};
 use ballista_core::JobId;
 use ballista_core::error::BallistaError;
@@ -33,7 +33,7 @@ use ballista_core::serde::protobuf::{
     PollWorkParams, PollWorkResult, TaskDefinition, TaskStatus,
     scheduler_grpc_client::SchedulerGrpcClient,
 };
-use ballista_core::serde::scheduler::{ExecutorSpecification, PartitionId};
+use ballista_core::serde::scheduler::{ExecutorSpecification, TaskKey};
 use datafusion::execution::context::TaskContext;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_proto::logical_plan::AsLogicalPlan;
@@ -44,23 +44,46 @@ use std::any::Any;
 use std::convert::TryInto;
 use std::error::Error;
 use std::sync::mpsc::{Receiver, Sender, TryRecvError};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use std::{sync::Arc, time::Duration};
 use tokio::sync::{OwnedSemaphorePermit, Semaphore};
 use tonic::codegen::{Body, Bytes, StdError};
 
 /// Main execution loop that polls the scheduler for available tasks.
 ///
-/// This function runs indefinitely, periodically asking the scheduler for
-/// work. When tasks are received, they are executed on a dedicated thread
-/// pool and results are reported back to the scheduler.
+/// Runs indefinitely, periodically asking the scheduler for work. When tasks
+/// are received they are executed concurrently and results are reported back
+/// to the scheduler.
 ///
-/// The loop respects the executor's concurrent task limit via a semaphore,
-/// ensuring no more than the configured number of tasks run simultaneously.
+/// Concurrency is bounded by a semaphore. Pass `free_vcores` to supply your
+/// own semaphore — useful for sharing a single concurrency limit across
+/// multiple poll loops or for observing executor load from outside.
+/// Pass `None` to have the loop create a semaphore sized to the executor's
+/// configured vcore count.
+///
+/// **Shared semaphores**: when one semaphore is shared across loops that
+/// connect to different schedulers, each scheduler independently sees the
+/// current free capacity and may dispatch up to that many tasks. The semaphore
+/// still caps total concurrent execution — tasks that cannot run immediately
+/// wait for capacity — but both schedulers may over-commit relative to what
+/// the semaphore can actually admit at once. This is intentional: the
+/// semaphore acts as an execution throttle, not a reservation system.
+///
+/// **Semaphore sizing**: if the provided semaphore allows more concurrent
+/// tasks than the executor's thread pool has threads, excess admitted tasks
+/// will queue behind running ones. The caller is responsible for sizing the
+/// semaphore appropriately for their thread pool.
+///
+/// # Panics
+///
+/// Panics on startup if `free_vcores` is a semaphore with zero permits,
+/// which would cause the loop to deadlock immediately.
 pub async fn poll_loop<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan, C>(
     mut scheduler: SchedulerGrpcClient<C>,
     executor: Arc<Executor>,
     codec: BallistaCodec<T, U>,
+    free_vcores: Option<Arc<Semaphore>>,
+    health: crate::health::ExecutorHealth,
 ) -> Result<(), BallistaError>
 where
     C: tonic::client::GrpcService<tonic::body::Body>,
@@ -75,20 +98,28 @@ where
         .unwrap()
         .clone()
         .into();
-    let available_task_slots =
-        Arc::new(Semaphore::new(executor_specification.task_slots as usize));
+    let free_vcores = free_vcores.unwrap_or_else(|| {
+        Arc::new(Semaphore::new(executor_specification.vcores as usize))
+    });
+    assert!(
+        free_vcores.available_permits() > 0,
+        "free_vcores semaphore must have at least one permit; passing a closed or zero-permit semaphore would deadlock the poll loop"
+    );
 
     let (task_status_sender, mut task_status_receiver) =
         std::sync::mpsc::channel::<TaskStatus>();
     info!("Starting poll work loop with scheduler");
 
     let dedicated_executor =
-        DedicatedExecutor::new("task_runner", executor_specification.task_slots as usize);
+        DedicatedExecutor::new("task_runner", executor_specification.vcores as usize);
 
     loop {
-        // Wait for task slots to be available before asking for new work
-        let permit = available_task_slots.acquire().await.unwrap();
-        // Make the slot available again
+        // Wait for a vcore permit before asking for new work.
+        let permit = free_vcores
+            .acquire()
+            .await
+            .map_err(|_| BallistaError::Internal("vcore semaphore closed".to_string()))?;
+        // Make the vcore available again for the actual bind below.
         drop(permit);
 
         // Keeps track of whether we received task in last iteration
@@ -102,13 +133,14 @@ where
             scheduler
                 .poll_work(PollWorkParams {
                     metadata: Some(executor.metadata.clone()),
-                    num_free_slots: available_task_slots.available_permits() as u32,
+                    num_free_vcores: free_vcores.available_permits() as u32,
                     task_status,
                 })
                 .await;
 
         match poll_work_result {
             Ok(result) => {
+                health.mark_heartbeat_ok();
                 let PollWorkResult {
                     tasks,
                     jobs_to_clean,
@@ -119,12 +151,15 @@ where
                 for cleanup in jobs_to_clean {
                     let job_id = cleanup.job_id.clone().into();
                     let work_dir = executor.work_dir.clone();
+                    let remove_stage_ids = cleanup.remove_stage_ids.clone();
 
                     // In poll-based cleanup, removing job data is fire-and-forget.
                     // Failures here do not affect task execution and are only logged.
                     tokio::spawn(async move {
-                        if let Err(e) = remove_job_dir(&work_dir, &job_id).await {
-                            error!("failed to remove job dir {job_id}: {e}");
+                        if let Err(e) =
+                            remove_job_data(&work_dir, &job_id, &remove_stage_ids).await
+                        {
+                            error!("failed to remove job data {job_id}: {e}");
                         }
                     });
                 }
@@ -132,9 +167,11 @@ where
                 for task in tasks {
                     let task_status_sender = task_status_sender.clone();
 
-                    // Acquire a permit/slot for the task
+                    // Acquire a vcore permit for the task.
                     let permit =
-                        available_task_slots.clone().acquire_owned().await.unwrap();
+                        free_vcores.clone().acquire_owned().await.map_err(|_| {
+                            BallistaError::Internal("vcore semaphore closed".to_string())
+                        })?;
 
                     let start_exec_time = SystemTime::now()
                         .duration_since(UNIX_EPOCH)
@@ -158,14 +195,14 @@ where
                             // as scheduler expects notification.
                             //
 
-                            let partition_id = PartitionId {
+                            let task_key = TaskKey {
                                 job_id: task.job_id.clone().into(),
                                 stage_id: task.stage_id as usize,
-                                partition_id: task.partition_id as usize,
+                                task_id: task.task_id as usize,
                             };
 
                             warn!(
-                                "Executor failed to run task: {partition_id:?}, error: {e:?}"
+                                "Executor failed to run task: {task_key:?}, error: {e:?}"
                             );
 
                             let end_exec_time = SystemTime::now()
@@ -184,9 +221,8 @@ where
                             if let Err(error) = task_status_sender.send(as_task_status(
                                 Err(e),
                                 executor.metadata.id.clone(),
-                                task.task_id as usize,
                                 task.task_attempt_num as usize,
-                                partition_id,
+                                task_key,
                                 None,
                                 task_execution_times,
                             )) {
@@ -197,6 +233,7 @@ where
                 }
             }
             Err(error) => {
+                health.mark_heartbeat_failed();
                 warn!(
                     "Executor poll work loop failed. If this continues to happen the Scheduler might be marked as dead. Error: {error}"
                 );
@@ -236,13 +273,12 @@ async fn run_received_task<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
     let stage_id = task.stage_id;
     let stage_attempt_num = task.stage_attempt_num;
     let task_launch_time = task.launch_time;
-    let partition_id = task.partition_id;
     let start_exec_time = SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .unwrap()
         .as_millis() as u64;
     let task_identity = format!(
-        "TID {task_id} {job_id}/{stage_id}.{stage_attempt_num}/{partition_id}.{task_attempt_num}"
+        "TID {job_id}/{stage_id}.{stage_attempt_num}/{task_id}.{task_attempt_num}"
     );
     info!("Received task: [{task_identity}]");
 
@@ -256,8 +292,14 @@ async fn run_received_task<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
     let task_scalar_functions = executor.function_registry.scalar_functions.clone();
     let task_aggregate_functions = executor.function_registry.aggregate_functions.clone();
     let task_window_functions = executor.function_registry.window_functions.clone();
+    let task_higher_order_functions =
+        executor.function_registry.higher_order_functions.clone();
 
-    let runtime = executor.produce_runtime(&session_config)?;
+    let runtime = executor.produce_runtime_for_session(
+        &task.session_id,
+        &session_config,
+        task.vcores_consumed,
+    )?;
 
     let session_id = task.session_id.clone();
     let task_context = Arc::new(TaskContext::new(
@@ -265,6 +307,7 @@ async fn run_received_task<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
         session_id,
         session_config,
         task_scalar_functions,
+        task_higher_order_functions,
         task_aggregate_functions,
         task_window_functions,
         runtime.clone(),
@@ -275,25 +318,32 @@ async fn run_received_task<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
             proto.try_into_physical_plan(&task_context, codec.physical_extension_codec())
         })?;
 
+    let global_output_partition_ids: Vec<usize> = task
+        .global_output_partition_ids
+        .iter()
+        .map(|p| *p as usize)
+        .collect();
+
     let query_stage_exec = executor.execution_engine.create_query_stage_exec(
         job_id.clone(),
         stage_id as usize,
-        partition_id as usize,
+        task_id as usize,
+        global_output_partition_ids,
         plan,
         &executor.work_dir,
         task_context.session_config(),
     )?;
     dedicated_executor.spawn(async move {
         use std::panic::AssertUnwindSafe;
-        let part = PartitionId {
+        let key = TaskKey {
             job_id: job_id.clone(),
             stage_id: stage_id as usize,
-            partition_id: partition_id as usize,
+            task_id: task_id as usize,
         };
 
+        let task_start = Instant::now();
         let execution_result = match AssertUnwindSafe(executor.execute_query_stage(
-            task_id as usize,
-            part.clone(),
+            key.clone(),
             query_stage_exec.clone(),
             task_context,
         ))
@@ -308,7 +358,10 @@ async fn run_received_task<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
             }
         };
 
-        info!("Finished task : [{task_identity}]");
+        info!(
+            "Finished task : [{task_identity}] in {:?}",
+            task_start.elapsed()
+        );
         debug!("Task statistics: [{task_identity}] {execution_result:?}");
 
         let plan_metrics = query_stage_exec.collect_plan_metrics();
@@ -332,9 +385,8 @@ async fn run_received_task<T: 'static + AsLogicalPlan, U: 'static + AsExecutionP
         let _ = task_status_sender.send(as_task_status(
             execution_result,
             executor.metadata.id.clone(),
-            task_id as usize,
             stage_attempt_num as usize,
-            part,
+            key,
             operator_metrics,
             task_execution_times,
         ));
diff --git a/ballista/executor/src/executor.rs b/ballista/executor/src/executor.rs
index 3d577cf821..3a1c103be8 100644
--- a/ballista/executor/src/executor.rs
+++ b/ballista/executor/src/executor.rs
@@ -23,6 +23,7 @@ use crate::execution_engine::QueryStageExecutor;
 use crate::execution_loop::any_to_string;
 use crate::metrics::ExecutorMetricsCollector;
 use crate::metrics::LoggingMetricsCollector;
+use crate::runtime_cache::SessionRuntimeCache;
 use ballista_core::ConfigProducer;
 use ballista_core::JobId;
 use ballista_core::RuntimeProducer;
@@ -30,7 +31,7 @@ use ballista_core::error::BallistaError;
 use ballista_core::registry::BallistaFunctionRegistry;
 use ballista_core::serde::protobuf;
 use ballista_core::serde::protobuf::ExecutorRegistration;
-use ballista_core::serde::scheduler::PartitionId;
+use ballista_core::serde::scheduler::TaskKey;
 use dashmap::DashMap;
 use datafusion::execution::context::TaskContext;
 use datafusion::execution::runtime_env::RuntimeEnv;
@@ -65,7 +66,7 @@ impl Future for TasksDrainedFuture {
     }
 }
 
-type AbortHandles = Arc<DashMap<(usize, PartitionId), AbortHandle>>;
+type AbortHandles = Arc<DashMap<TaskKey, AbortHandle>>;
 
 /// Ballista executor
 #[derive(Clone)]
@@ -88,8 +89,8 @@ pub struct Executor {
     /// Collector for runtime execution metrics
     pub metrics_collector: Arc<dyn ExecutorMetricsCollector>,
 
-    /// Concurrent tasks can run in executor
-    pub concurrent_tasks: usize,
+    /// Virtual cores assigned to this executor. See CLI docs on `--vcores`.
+    pub vcores: usize,
 
     /// Handles to abort executing tasks
     abort_handles: AbortHandles,
@@ -97,6 +98,11 @@ pub struct Executor {
     /// Execution engine that the executor will delegate to
     /// for executing query stages
     pub(crate) execution_engine: Arc<dyn ExecutionEngine>,
+
+    /// Optional session-keyed cache of shared base runtime envs. When set,
+    /// `produce_runtime_for_session` reuses read-side state across a session's
+    /// tasks; when `None`, each task builds a runtime from `runtime_producer`.
+    session_runtime_cache: Option<Arc<dyn SessionRuntimeCache>>,
 }
 
 impl Executor {
@@ -107,7 +113,7 @@ impl Executor {
         work_dir: &str,
         runtime_producer: RuntimeProducer,
         config_producer: ConfigProducer,
-        concurrent_tasks: usize,
+        vcores: usize,
     ) -> Self {
         Self::new(
             metadata,
@@ -116,7 +122,7 @@ impl Executor {
             config_producer,
             Arc::new(BallistaFunctionRegistry::default()),
             Arc::new(LoggingMetricsCollector::default()),
-            concurrent_tasks,
+            vcores,
             Arc::new(DefaultExecutionEngine::new()),
         )
     }
@@ -131,7 +137,7 @@ impl Executor {
         config_producer: ConfigProducer,
         function_registry: Arc<BallistaFunctionRegistry>,
         metrics_collector: Arc<dyn ExecutorMetricsCollector>,
-        concurrent_tasks: usize,
+        vcores: usize,
         execution_engine: Arc<dyn ExecutionEngine>,
     ) -> Self {
         Self {
@@ -141,9 +147,10 @@ impl Executor {
             runtime_producer,
             config_producer,
             metrics_collector,
-            concurrent_tasks,
+            vcores,
             abort_handles: Default::default(),
             execution_engine,
+            session_runtime_cache: None,
         }
     }
     /// Creates new Executor with default `ExecutionEngine`.
@@ -155,7 +162,7 @@ impl Executor {
         config_producer: ConfigProducer,
         function_registry: Arc<BallistaFunctionRegistry>,
         metrics_collector: Arc<dyn ExecutorMetricsCollector>,
-        concurrent_tasks: usize,
+        vcores: usize,
     ) -> Self {
         Self {
             metadata,
@@ -164,9 +171,10 @@ impl Executor {
             runtime_producer,
             config_producer,
             metrics_collector,
-            concurrent_tasks,
+            vcores,
             abort_handles: Default::default(),
             execution_engine: Arc::new(DefaultExecutionEngine::new()),
+            session_runtime_cache: None,
         }
     }
 }
@@ -180,6 +188,33 @@ impl Executor {
         (self.runtime_producer)(config)
     }
 
+    /// Attaches (or clears) the session-keyed runtime cache. Cloning the
+    /// `Executor` shares the same cache via `Arc`.
+    pub fn with_session_runtime_cache(
+        mut self,
+        cache: Option<Arc<dyn SessionRuntimeCache>>,
+    ) -> Self {
+        self.session_runtime_cache = cache;
+        self
+    }
+
+    /// Produces the runtime for a task, reusing the session's shared read-side
+    /// state when a session cache is attached; otherwise builds a runtime per
+    /// task via the `runtime_producer`. `vcores_consumed` is the number of
+    /// vcores this task claimed at bind time; the memory pool policy uses it
+    /// to size the per-task pool proportionally.
+    pub fn produce_runtime_for_session(
+        &self,
+        session_id: &str,
+        config: &SessionConfig,
+        vcores_consumed: u32,
+    ) -> datafusion::error::Result<Arc<RuntimeEnv>> {
+        match &self.session_runtime_cache {
+            Some(cache) => cache.produce_runtime(session_id, config, vcores_consumed),
+            None => (self.runtime_producer)(config),
+        }
+    }
+
     /// Creates a default [`SessionConfig`] using the configured config producer.
     pub fn produce_config(&self) -> SessionConfig {
         (self.config_producer)()
@@ -190,17 +225,15 @@ impl Executor {
     /// and statistics.
     pub async fn execute_query_stage(
         &self,
-        task_id: usize,
-        partition: PartitionId,
+        key: TaskKey,
         query_stage_exec: Arc<dyn QueryStageExecutor>,
         task_ctx: Arc<TaskContext>,
     ) -> Result<Vec<protobuf::ShuffleWritePartition>, BallistaError> {
         let (task, abort_handle) = futures::future::abortable(
-            query_stage_exec.execute_query_stage(partition.partition_id, task_ctx),
+            query_stage_exec.execute_query_stage(key.task_id, task_ctx),
         );
 
-        self.abort_handles
-            .insert((task_id, partition.clone()), abort_handle);
+        self.abort_handles.insert(key.clone(), abort_handle);
 
         let partitions = match std::panic::AssertUnwindSafe(task).catch_unwind().await {
             Ok(Ok(result)) => {
@@ -217,12 +250,12 @@ impl Executor {
             }
         };
 
-        self.abort_handles.remove(&(task_id, partition.clone()));
+        self.abort_handles.remove(&key);
 
         self.metrics_collector.record_stage(
-            &partition.job_id,
-            partition.stage_id,
-            partition.partition_id,
+            &key.job_id,
+            key.stage_id,
+            key.task_id,
             query_stage_exec,
         );
 
@@ -234,19 +267,15 @@ impl Executor {
     /// Returns `Ok(true)` if the task was found and cancelled, `Ok(false)` if not found.
     pub async fn cancel_task(
         &self,
-        task_id: usize,
         job_id: JobId,
         stage_id: usize,
-        partition_id: usize,
+        task_id: usize,
     ) -> Result<bool, BallistaError> {
-        if let Some((_, handle)) = self.abort_handles.remove(&(
+        if let Some((_, handle)) = self.abort_handles.remove(&TaskKey {
+            job_id,
+            stage_id,
             task_id,
-            PartitionId {
-                job_id,
-                stage_id,
-                partition_id,
-            },
-        )) {
+        }) {
             handle.abort();
             Ok(true)
         } else {
@@ -269,23 +298,27 @@ impl Executor {
 mod test {
     use crate::execution_engine::{DefaultQueryStageExec, ShuffleWriterVariant};
     use crate::executor::Executor;
+    use crate::runtime_cache::{
+        DefaultSessionRuntimeCache, MemoryPoolPolicy, SessionRuntimeCache,
+    };
     use ballista_core::RuntimeProducer;
     use ballista_core::execution_plans::ShuffleWriterExec;
     use ballista_core::serde::protobuf::ExecutorRegistration;
-    use ballista_core::serde::scheduler::PartitionId;
+    use ballista_core::serde::scheduler::TaskKey;
     use ballista_core::utils::default_config_producer;
     use datafusion::arrow::datatypes::{Schema, SchemaRef};
     use datafusion::arrow::record_batch::RecordBatch;
     use datafusion::error::{DataFusionError, Result};
     use datafusion::execution::context::TaskContext;
+    use datafusion::execution::runtime_env::RuntimeEnv;
 
     use datafusion::physical_plan::{
         DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
         RecordBatchStream, SendableRecordBatchStream,
     };
+    use datafusion::prelude::SessionConfig;
     use datafusion::prelude::SessionContext;
     use futures::Stream;
-    use std::any::Any;
     use std::pin::Pin;
     use std::sync::Arc;
     use std::task::{Context, Poll};
@@ -354,10 +387,6 @@ mod test {
             "NeverendingOperator"
         }
 
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-
         fn schema(&self) -> SchemaRef {
             Arc::new(Schema::empty())
         }
@@ -404,11 +433,7 @@ mod test {
 
         let executor_registration = ExecutorRegistration {
             id: "executor".to_string(),
-            port: 0,
-            grpc_port: 0,
-            specification: None,
-            host: None,
-            os_info: None,
+            ..Default::default()
         };
         let config_producer = Arc::new(default_config_producer);
         let ctx = SessionContext::new();
@@ -429,13 +454,13 @@ mod test {
         // Spawn our non-terminating task on a separate fiber.
         let executor_clone = executor.clone();
         tokio::task::spawn(async move {
-            let part = PartitionId {
+            let key = TaskKey {
                 job_id: "job-id".into(),
                 stage_id: 1,
-                partition_id: 0,
+                task_id: 0,
             };
             let task_result = executor_clone
-                .execute_query_stage(1, part, Arc::new(query_stage_exec), ctx.task_ctx())
+                .execute_query_stage(key, Arc::new(query_stage_exec), ctx.task_ctx())
                 .await;
             sender.send(task_result).expect("sending result");
         });
@@ -444,7 +469,7 @@ mod test {
         // poll until that happens.
         for _ in 0..20 {
             if executor
-                .cancel_task(1, "job-id".into(), 1, 0)
+                .cancel_task("job-id".into(), 1, 0)
                 .await
                 .expect("cancelling task")
             {
@@ -464,4 +489,64 @@ mod test {
         let inner_result = result.unwrap().unwrap();
         assert!(inner_result.is_err());
     }
+
+    #[test]
+    fn produce_runtime_for_session_shares_read_side_state() {
+        let executor_registration = ExecutorRegistration {
+            id: "executor".to_string(),
+            ..Default::default()
+        };
+        let config_producer = Arc::new(default_config_producer);
+
+        // A base producer that builds a fresh env each call, plus an identity
+        // pool policy, so shared read-side state is observable via ptr equality.
+        let base_producer: RuntimeProducer =
+            Arc::new(|_| Ok(Arc::new(RuntimeEnv::default())));
+        let identity: MemoryPoolPolicy = Arc::new(|base, _, _| Ok(base));
+        let cache: Arc<dyn SessionRuntimeCache> = Arc::new(
+            DefaultSessionRuntimeCache::new(base_producer.clone(), identity, 4),
+        );
+
+        let executor = Executor::new_basic(
+            executor_registration,
+            "/tmp",
+            base_producer,
+            config_producer,
+            2,
+        )
+        .with_session_runtime_cache(Some(cache));
+
+        let cfg = SessionConfig::new();
+        let e1 = executor.produce_runtime_for_session("s1", &cfg, 1).unwrap();
+        let e2 = executor.produce_runtime_for_session("s1", &cfg, 1).unwrap();
+        let e3 = executor.produce_runtime_for_session("s2", &cfg, 1).unwrap();
+
+        assert!(Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager));
+        assert!(!Arc::ptr_eq(&e1.cache_manager, &e3.cache_manager));
+    }
+
+    #[test]
+    fn produce_runtime_for_session_falls_back_without_cache() {
+        let executor_registration = ExecutorRegistration {
+            id: "executor".to_string(),
+            ..Default::default()
+        };
+        let config_producer = Arc::new(default_config_producer);
+        let base_producer: RuntimeProducer =
+            Arc::new(|_| Ok(Arc::new(RuntimeEnv::default())));
+
+        // No cache attached: each call builds a fresh env.
+        let executor = Executor::new_basic(
+            executor_registration,
+            "/tmp",
+            base_producer,
+            config_producer,
+            2,
+        );
+
+        let cfg = SessionConfig::new();
+        let e1 = executor.produce_runtime_for_session("s1", &cfg, 1).unwrap();
+        let e2 = executor.produce_runtime_for_session("s1", &cfg, 1).unwrap();
+        assert!(!Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager));
+    }
 }
diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs
index 6050649ab9..d8f30163b5 100644
--- a/ballista/executor/src/executor_process.rs
+++ b/ballista/executor/src/executor_process.rs
@@ -42,7 +42,7 @@ use tokio::{fs, time};
 use uuid::Uuid;
 
 use datafusion::execution::memory_pool::{FairSpillPool, MemoryPool};
-use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
 use datafusion::prelude::SessionConfig;
 
 use ballista_core::config::{LogRotationPolicy, TaskSchedulingPolicy};
@@ -61,7 +61,9 @@ use ballista_core::utils::{
     GrpcClientConfig, GrpcServerConfig, create_grpc_client_endpoint, create_grpc_server,
     default_config_producer, get_time_before,
 };
-use ballista_core::{BALLISTA_VERSION, ConfigProducer, JobId, RuntimeProducer};
+use ballista_core::{
+    BALLISTA_PROTOCOL_VERSION, BALLISTA_VERSION, ConfigProducer, JobId, RuntimeProducer,
+};
 
 use crate::client_pool::DefaultBallistaClientPool;
 use crate::execution_engine::{DefaultExecutionEngine, ExecutionEngine};
@@ -70,36 +72,52 @@ use crate::executor_server::TERMINATING;
 use crate::flight_service::BallistaFlightService;
 use crate::metrics::ExecutorMetricCollectionPolicy;
 use crate::metrics::LoggingMetricsCollector;
+use crate::runtime_cache::{
+    DefaultSessionRuntimeCache, MemoryPoolPolicy, SessionRuntimeCache,
+};
 use crate::shutdown::Shutdown;
 use crate::shutdown::ShutdownNotifier;
 use crate::{ArrowFlightServerProvider, terminate};
 use crate::{execution_loop, executor_server};
 
-/// Wrap a [`RuntimeProducer`] so that every produced
-/// [`RuntimeEnv`](datafusion::execution::runtime_env::RuntimeEnv) carries a
-/// fresh [`FairSpillPool`] of size `total_bytes / concurrent_tasks`.
+/// Builds a per-task memory-pool policy: each task's runtime is rebuilt from the
+/// shared base env with a fresh [`FairSpillPool`] of size
+/// `total_bytes / vcores`. The base env's disk manager, cache manager,
+/// and object-store registry are preserved via
+/// [`RuntimeEnvBuilder::from_runtime_env`].
 ///
 /// Returns an error if the per-task share would be zero (i.e. `total_bytes <
-/// concurrent_tasks`). The inner env's disk manager, cache manager, and
-/// object store registry are preserved via [`RuntimeEnvBuilder::from_runtime_env`].
-fn wrap_runtime_producer_with_memory_pool(
-    inner: RuntimeProducer,
+/// vcores`).
+fn memory_pool_policy(
     total_bytes: u64,
-    concurrent_tasks: usize,
-) -> Result<RuntimeProducer, BallistaError> {
-    let per_task = (total_bytes / concurrent_tasks as u64) as usize;
-    if per_task == 0 {
+    vcores: usize,
+) -> Result<MemoryPoolPolicy, BallistaError> {
+    let per_vcore = (total_bytes / vcores as u64) as usize;
+    if per_vcore == 0 {
         return Err(BallistaError::Configuration(format!(
-            "memory_pool_size ({total_bytes} bytes) is smaller than concurrent_tasks ({concurrent_tasks})"
+            "memory_pool_size ({total_bytes} bytes) is smaller than vcores ({vcores})"
         )));
     }
-    Ok(Arc::new(move |session_config: &SessionConfig| {
-        let inner_env = inner(session_config)?;
-        let pool: Arc<dyn MemoryPool> = Arc::new(FairSpillPool::new(per_task));
-        RuntimeEnvBuilder::from_runtime_env(&inner_env)
-            .with_memory_pool(pool)
-            .build_arc()
-    }))
+    Ok(Arc::new(
+        move |base: Arc<RuntimeEnv>, _config: &SessionConfig, vcores_consumed: u32| {
+            // Multi-partition tasks that claim N vcores get N × per-vcore
+            // share of the executor's memory pool. A collapse task with
+            // vcores_consumed=1 (see scheduler `bind_one`) gets the
+            // single-vcore share it uses; a 4-partition task gets 4×,
+            // matching the parallelism DataFusion will actually drive.
+            let size = per_vcore.saturating_mul(vcores_consumed.max(1) as usize);
+            let pool: Arc<dyn MemoryPool> = Arc::new(FairSpillPool::new(size));
+            RuntimeEnvBuilder::from_runtime_env(&base)
+                .with_memory_pool(pool)
+                .build_arc()
+        },
+    ))
+}
+
+/// A no-op policy: the task uses the shared base env unchanged (DataFusion's
+/// default unbounded pool). Used when `--memory-pool-size` is unset.
+fn identity_pool_policy() -> MemoryPoolPolicy {
+    Arc::new(|base, _config, _vcores| Ok(base))
 }
 
 /// Configuration for the executor process.
@@ -122,8 +140,8 @@ pub struct ExecutorProcessConfig {
     pub scheduler_port: u16,
     /// Timeout in seconds for establishing scheduler connection.
     pub scheduler_connect_timeout_seconds: u16,
-    /// Maximum number of concurrent tasks this executor can run.
-    pub concurrent_tasks: usize,
+    /// Virtual cores advertised by this executor to the scheduler.
+    pub vcores: usize,
     /// Task scheduling policy (pull-staged or push-staged).
     pub task_scheduling_policy: TaskSchedulingPolicy,
     /// Directory for storing log files.
@@ -152,9 +170,14 @@ pub struct ExecutorProcessConfig {
     pub metric_collection_policy: ExecutorMetricCollectionPolicy,
     /// Optional total memory pool size in bytes. When set, every task's
     /// runtime env receives a FairSpillPool of size
-    /// `memory_pool_size / concurrent_tasks`. When `None`, no pool is
+    /// `memory_pool_size / vcores`. When `None`, no pool is
     /// installed and DataFusion falls back to its unbounded default.
     pub memory_pool_size: Option<u64>,
+    /// Maximum number of sessions whose shared base runtime env is retained on
+    /// the executor (LRU). Sharing reuses object-store clients and the Parquet
+    /// footer cache across a session's tasks and queries. `0` disables caching
+    /// and builds a fresh runtime per task.
+    pub session_runtime_cache_capacity: usize,
     /// Optional execution engine to use to execute physical plans, will default to
     /// DataFusion if none is provided.
     pub override_execution_engine: Option<Arc<dyn ExecutionEngine>>,
@@ -172,8 +195,16 @@ pub struct ExecutorProcessConfig {
     pub override_arrow_flight_service: Option<Arc<ArrowFlightServerProvider>>,
     /// Override function for customizing gRPC client endpoints before they are used
     pub override_create_grpc_client_endpoint: Option<EndpointOverrideFn>,
-    /// Number of seconds established client connection should be cached (0 means no cache)
+    /// Number of seconds an established client connection is cached while idle.
+    /// `0` disables the pool, so every shuffle fetch opens and drops its own
+    /// connection and a shuffle-heavy query can exhaust the host's ephemeral
+    /// ports.
     pub client_ttl: u64,
+    /// Shared readiness state that the heartbeat loops flip on every RPC
+    /// outcome. Embedders leave this as `Default::default()` and never
+    /// observe it; the standalone binary passes a handle here and also spawns
+    /// an HTTP server on it (see `bin/main.rs`).
+    pub health: crate::health::ExecutorHealth,
 }
 
 impl ExecutorProcessConfig {
@@ -199,7 +230,7 @@ impl Default for ExecutorProcessConfig {
             scheduler_host: "localhost".into(),
             scheduler_port: 50050,
             scheduler_connect_timeout_seconds: 0,
-            concurrent_tasks: std::thread::available_parallelism().unwrap().get(),
+            vcores: std::thread::available_parallelism().unwrap().get(),
             task_scheduling_policy: Default::default(),
             log_dir: None,
             work_dir: None,
@@ -214,6 +245,7 @@ impl Default for ExecutorProcessConfig {
             executor_heartbeat_interval_seconds: 60,
             metric_collection_policy: ExecutorMetricCollectionPolicy::default(),
             memory_pool_size: None,
+            session_runtime_cache_capacity: 16,
             override_execution_engine: None,
             override_function_registry: None,
             override_runtime_producer: None,
@@ -222,7 +254,8 @@ impl Default for ExecutorProcessConfig {
             override_physical_codec: None,
             override_arrow_flight_service: None,
             override_create_grpc_client_endpoint: None,
-            client_ttl: 0,
+            client_ttl: 30,
+            health: crate::health::ExecutorHealth::new(),
         }
     }
 }
@@ -260,11 +293,11 @@ pub async fn start_executor_process(
         ));
     };
 
-    let concurrent_tasks = if opt.concurrent_tasks == 0 {
+    let vcores = if opt.vcores == 0 {
         // use all available cores if no concurrency level is specified
         std::thread::available_parallelism().unwrap().get()
     } else {
-        opt.concurrent_tasks
+        opt.vcores
     };
     let task_scheduling_policy = opt.task_scheduling_policy;
     // assign this executor an unique ID
@@ -274,13 +307,10 @@ pub async fn start_executor_process(
     );
     info!("Executor id: {executor_id}");
     info!("Executor working directory: {work_dir}");
-    info!(
-        "Executor number of concurrent tasks (available CPU cores): {concurrent_tasks}"
-    );
+    info!("Executor vcores (default: available CPU cores): {vcores}");
     info!("Executor scheduling policy: {task_scheduling_policy:?}");
 
-    let executor_meta =
-        structure_executor_metadata(&executor_id, &opt, concurrent_tasks as u32);
+    let executor_meta = structure_executor_metadata(&executor_id, &opt, vcores as u32);
 
     // put them to session config
     let metrics_collector = Arc::new(LoggingMetricsCollector::default());
@@ -290,7 +320,7 @@ pub async fn start_executor_process(
         .unwrap_or_else(|| Arc::new(default_config_producer));
 
     let wd = work_dir.clone();
-    let runtime_producer: RuntimeProducer =
+    let base_runtime_producer: RuntimeProducer =
         opt.override_runtime_producer.clone().unwrap_or_else(|| {
             Arc::new(move |_| {
                 let runtime_env = RuntimeEnvBuilder::new()
@@ -300,19 +330,27 @@ pub async fn start_executor_process(
             })
         });
 
-    let runtime_producer = if let Some(total) = opt.memory_pool_size {
-        let producer = wrap_runtime_producer_with_memory_pool(
-            runtime_producer,
-            total,
-            concurrent_tasks,
-        )?;
-        let per_task = total / concurrent_tasks as u64;
+    let pool_policy: MemoryPoolPolicy = if let Some(total) = opt.memory_pool_size {
+        let policy = memory_pool_policy(total, vcores)?;
+        let per_vcore = total / vcores as u64;
         info!(
-            "Memory pool: total {total} bytes split into {concurrent_tasks} tasks ({per_task} bytes each)"
+            "Memory pool: total {total} bytes split into {vcores} tasks ({per_vcore} bytes each)"
         );
-        producer
+        policy
     } else {
-        runtime_producer
+        identity_pool_policy()
+    };
+
+    // Combined producer preserving the current per-task behavior: build a fresh
+    // base env, then apply the pool policy. Used by `Executor::produce_runtime`
+    // and as the fallback when session caching is disabled.
+    // Session-level fallback (used only when no session cache is attached);
+    // per-task vcores aren't threaded here, so size the pool for the smallest
+    // task shape (1 vcore).
+    let runtime_producer: RuntimeProducer = {
+        let base = base_runtime_producer.clone();
+        let policy = pool_policy.clone();
+        Arc::new(move |config: &SessionConfig| policy(base(config)?, config, 1))
     };
 
     let logical = opt
@@ -330,26 +368,40 @@ pub async fn start_executor_process(
         datafusion_proto::protobuf::PhysicalPlanNode,
     > = BallistaCodec::new(logical, physical);
 
-    let executor = Arc::new(Executor::new(
-        executor_meta.clone(),
-        &work_dir,
-        runtime_producer,
-        config_producer,
-        opt.override_function_registry.clone().unwrap_or_default(),
-        metrics_collector,
-        concurrent_tasks,
-        opt.override_execution_engine.clone().unwrap_or_else(|| {
-            if opt.client_ttl > 0 {
-                let client_pool =
-                    Arc::new(DefaultBallistaClientPool::with_eviction_thread(
-                        Duration::from_secs(opt.client_ttl),
-                    ));
-                Arc::new(DefaultExecutionEngine::with_client_pool(client_pool))
-            } else {
-                Arc::new(DefaultExecutionEngine::new())
-            }
-        }),
-    ));
+    // Always attach the session cache: `pool_policy` wraps whatever base env
+    // the producer returns with a fresh per-task memory pool sized to the
+    // task's vcores, so an override producer (e.g. S3-aware) composes with
+    // the pool the same way the default one does.
+    let session_runtime_cache: Arc<dyn SessionRuntimeCache> =
+        Arc::new(DefaultSessionRuntimeCache::new(
+            base_runtime_producer.clone(),
+            pool_policy.clone(),
+            opt.session_runtime_cache_capacity,
+        ));
+
+    let executor = Arc::new(
+        Executor::new(
+            executor_meta.clone(),
+            &work_dir,
+            runtime_producer,
+            config_producer,
+            opt.override_function_registry.clone().unwrap_or_default(),
+            metrics_collector,
+            vcores,
+            opt.override_execution_engine.clone().unwrap_or_else(|| {
+                if opt.client_ttl > 0 {
+                    let client_pool =
+                        Arc::new(DefaultBallistaClientPool::with_eviction_thread(
+                            Duration::from_secs(opt.client_ttl),
+                        ));
+                    Arc::new(DefaultExecutionEngine::with_client_pool(client_pool))
+                } else {
+                    Arc::new(DefaultExecutionEngine::new())
+                }
+            }),
+        )
+        .with_session_runtime_cache(Some(session_runtime_cache)),
+    );
 
     let connect_timeout = opt.scheduler_connect_timeout_seconds as u64;
     let session_config = (executor.config_producer)();
@@ -481,6 +533,12 @@ pub async fn start_executor_process(
     // Channels used to receive stop requests from Executor grpc service.
     let (stop_send, mut stop_recv) = mpsc::channel::<bool>(10);
 
+    // Shared readiness state, flipped by the heartbeat/poll_work loop. When
+    // the caller is the standalone binary, an HTTP probe server observes
+    // this same handle (see `bin/main.rs`); library embedders leave it
+    // unobserved.
+    let health = opt.health.clone();
+
     // Starting main executor process based on the TaskSchedulingPolicy
     //
     // PushStaged => starting new executor_server that waits for tasks from the schedule
@@ -496,6 +554,7 @@ pub async fn start_executor_process(
                     default_codec,
                     stop_send,
                     &shutdown_notification,
+                    health,
                 )
                 .await?,
             );
@@ -505,6 +564,8 @@ pub async fn start_executor_process(
                 scheduler.clone(),
                 executor.clone(),
                 default_codec,
+                None, // free_vcores: use internal semaphore
+                health,
             )));
         }
     };
@@ -728,7 +789,7 @@ async fn clean_all_shuffle_data(work_dir: &str) -> ballista_core::error::Result<
         }
     }
 
-    info!("The work_dir {:?} will be deleted", &to_deleted);
+    info!("The work_dir {:?} will be deleted", to_deleted);
     for del in to_deleted {
         if let Err(e) = fs::remove_dir_all(&del).await {
             error!("Fail to remove the directory {del:?} due to {e}");
@@ -737,11 +798,16 @@ async fn clean_all_shuffle_data(work_dir: &str) -> ballista_core::error::Result<
     Ok(())
 }
 
-/// Remove a job directory under work_dir.
+/// Remove job data under work_dir.
 /// Used by both push-based (gRPC handler) and pull-based (poll loop) cleanup.
-pub(crate) async fn remove_job_dir(
+///
+/// `remove_stage_ids` empty ⇒ remove the whole `work_dir/{job_id}` dir (legacy
+/// behavior). Non-empty ⇒ remove only the listed `work_dir/{job_id}/{stage_id}`
+/// subdirs, retaining the rest of the job dir (e.g. the final-stage output).
+pub(crate) async fn remove_job_data(
     work_dir: &str,
     job_id: &JobId,
+    remove_stage_ids: &[u32],
 ) -> ballista_core::error::Result<()> {
     let work_path = PathBuf::from(&work_dir);
     let job_path = work_path.join(job_id.as_str());
@@ -761,11 +827,30 @@ pub(crate) async fn remove_job_dir(
         )));
     }
 
-    info!("Remove data for job {:?}", job_id);
+    // Empty stage list ⇒ remove the whole job dir (legacy behavior).
+    if remove_stage_ids.is_empty() {
+        info!("Remove data for job {:?}", job_id);
+        return tokio::fs::remove_dir_all(&job_path).await.map_err(|e| {
+            BallistaError::General(format!("Failed to remove {job_path:?} due to {e}"))
+        });
+    }
 
-    tokio::fs::remove_dir_all(&job_path).await.map_err(|e| {
-        BallistaError::General(format!("Failed to remove {job_path:?} due to {e}"))
-    })?;
+    // Otherwise remove only the given (intermediate) stage subdirs.
+    for stage_id in remove_stage_ids {
+        let stage_path = job_path.join(stage_id.to_string());
+        if !tokio::fs::try_exists(&stage_path).await.unwrap_or(false) {
+            continue;
+        }
+        if !is_subdirectory(stage_path.as_path(), job_path.as_path()) {
+            return Err(BallistaError::General(format!(
+                "Path {stage_path:?} is not a subdirectory of {job_path:?}"
+            )));
+        }
+        info!("Remove intermediate data for job {job_id:?} stage {stage_id}");
+        tokio::fs::remove_dir_all(&stage_path).await.map_err(|e| {
+            BallistaError::General(format!("Failed to remove {stage_path:?} due to {e}"))
+        })?;
+    }
 
     Ok(())
 }
@@ -834,7 +919,7 @@ pub async fn satisfy_dir_ttl(
 pub fn structure_executor_metadata(
     executor_id: &str,
     options: &Arc<ExecutorProcessConfig>,
-    concurrent_tasks: u32,
+    vcores: u32,
 ) -> ExecutorRegistration {
     let system_name =
         System::name().unwrap_or_else(|| String::from("Unknown system name"));
@@ -863,7 +948,7 @@ pub fn structure_executor_metadata(
         grpc_port: options.grpc_port as u32,
         specification: Some(ExecutorSpecification {
             resources: vec![ExecutorResource {
-                resource: Some(Resource::TaskSlots(concurrent_tasks)),
+                resource: Some(Resource::Vcores(vcores)),
             }],
         }),
         os_info: Some(ExecutorOperatingSystemSpecification {
@@ -877,6 +962,7 @@ pub fn structure_executor_metadata(
             total_available_disk_space,
             open_files_limit,
         }),
+        ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
     }
 }
 
@@ -886,6 +972,7 @@ mod tests {
     use std::path::{Path, PathBuf};
 
     use super::clean_shuffle_data_loop;
+    use super::remove_job_data;
     use ballista_core::JobId;
     use std::fs;
     use std::fs::File;
@@ -984,6 +1071,43 @@ mod tests {
             assert!(!is_subdirectory(&job_path, base_dir));
         }
     }
+    #[tokio::test]
+    async fn test_remove_intermediate_stage_data() {
+        let work_dir = TempDir::new().unwrap();
+        let work = work_dir.path();
+        let job_id: JobId = "job".into();
+
+        // Create job/1, job/2, job/3, each with a data file.
+        for stage in [1u32, 2, 3] {
+            let stage_dir = work.join("job").join(stage.to_string());
+            fs::create_dir_all(&stage_dir).unwrap();
+            File::create(stage_dir.join("data.arrow"))
+                .unwrap()
+                .write_all(b"x")
+                .unwrap();
+        }
+
+        // Remove intermediate stages 1 and 2; stage 3 (final) is retained.
+        remove_job_data(work.to_str().unwrap(), &job_id, &[1, 2])
+            .await
+            .unwrap();
+        assert!(!work.join("job").join("1").exists());
+        assert!(!work.join("job").join("2").exists());
+        assert!(work.join("job").join("3").exists());
+
+        // Removing a missing stage id is a no-op (idempotent).
+        remove_job_data(work.to_str().unwrap(), &job_id, &[99])
+            .await
+            .unwrap();
+        assert!(work.join("job").join("3").exists());
+
+        // Empty list removes the whole job dir.
+        remove_job_data(work.to_str().unwrap(), &job_id, &[])
+            .await
+            .unwrap();
+        assert!(!work.join("job").exists());
+    }
+
     fn prepare_testing_job_directory(base_dir: &Path, job_id: &JobId) -> PathBuf {
         let mut path = base_dir.to_path_buf();
         path.push(job_id.as_str());
@@ -1002,56 +1126,61 @@ mod memory_pool_tests {
     use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
     use std::sync::Arc;
 
-    fn baseline_producer() -> RuntimeProducer {
-        Arc::new(|_| Ok(Arc::new(RuntimeEnv::default())))
-    }
-
     #[test]
-    fn returns_error_when_total_smaller_than_concurrent_tasks() {
-        let inner = baseline_producer();
-        let result = wrap_runtime_producer_with_memory_pool(inner, 4, 8);
+    fn returns_error_when_total_smaller_than_vcores() {
+        let result = memory_pool_policy(4, 8);
         assert!(result.is_err());
         let msg = result.err().unwrap().to_string();
         assert!(msg.contains("memory_pool_size"));
-        assert!(msg.contains("concurrent_tasks"));
+        assert!(msg.contains("vcores"));
     }
 
     #[test]
-    fn produces_runtime_with_fair_spill_pool_of_per_task_size() {
+    fn produces_runtime_with_fair_spill_pool_scaled_by_vcores() {
         let total = 8u64 * 1024 * 1024 * 1024;
-        let concurrent = 8usize;
-        let expected_per_task = (total / concurrent as u64) as usize;
+        let vcores = 8usize;
+        let per_vcore = (total / vcores as u64) as usize;
 
-        let wrapped = wrap_runtime_producer_with_memory_pool(
-            baseline_producer(),
-            total,
-            concurrent,
-        )
-        .unwrap();
-        let env = wrapped(&SessionConfig::new()).unwrap();
+        let policy = memory_pool_policy(total, vcores).unwrap();
+        let base = Arc::new(RuntimeEnv::default());
+
+        // A 1-vcore task gets the per-vcore share; a 4-vcore task gets 4×.
+        let env_1 = policy(base.clone(), &SessionConfig::new(), 1).unwrap();
+        let env_4 = policy(base, &SessionConfig::new(), 4).unwrap();
 
-        match env.memory_pool.memory_limit() {
-            MemoryLimit::Finite(n) => assert_eq!(n, expected_per_task),
-            MemoryLimit::Infinite => panic!("expected Finite limit, got Infinite"),
-            MemoryLimit::Unknown => panic!("expected Finite limit, got Unknown"),
+        match env_1.memory_pool.memory_limit() {
+            MemoryLimit::Finite(n) => assert_eq!(n, per_vcore),
+            MemoryLimit::Infinite => panic!("expected Finite, got Infinite"),
+            MemoryLimit::Unknown => panic!("expected Finite, got Unknown"),
+        }
+        match env_4.memory_pool.memory_limit() {
+            MemoryLimit::Finite(n) => assert_eq!(n, per_vcore * 4),
+            MemoryLimit::Infinite => panic!("expected Finite, got Infinite"),
+            MemoryLimit::Unknown => panic!("expected Finite, got Unknown"),
         }
     }
 
     #[test]
-    fn preserves_inner_object_store_registry() {
+    fn preserves_base_object_store_registry() {
         let registry: Arc<dyn datafusion::execution::object_store::ObjectStoreRegistry> =
             Arc::new(DefaultObjectStoreRegistry::new());
-        let registry_for_inner = registry.clone();
-        let inner: RuntimeProducer = Arc::new(move |_| {
-            let env = RuntimeEnvBuilder::new()
-                .with_object_store_registry(registry_for_inner.clone())
-                .build()?;
-            Ok(Arc::new(env))
-        });
+        let base = Arc::new(
+            RuntimeEnvBuilder::new()
+                .with_object_store_registry(registry.clone())
+                .build()
+                .unwrap(),
+        );
 
-        let wrapped = wrap_runtime_producer_with_memory_pool(inner, 1024, 1).unwrap();
-        let env = wrapped(&SessionConfig::new()).unwrap();
+        let policy = memory_pool_policy(1024, 1).unwrap();
+        let env = policy(base, &SessionConfig::new(), 1).unwrap();
 
         assert!(Arc::ptr_eq(&env.object_store_registry, &registry));
     }
+
+    #[test]
+    fn identity_policy_returns_base_unchanged() {
+        let base = Arc::new(RuntimeEnv::default());
+        let env = identity_pool_policy()(base.clone(), &SessionConfig::new(), 1).unwrap();
+        assert!(Arc::ptr_eq(&env, &base));
+    }
 }
diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs
index 6ef9fa5e4e..105ae7900d 100644
--- a/ballista/executor/src/executor_server.rs
+++ b/ballista/executor/src/executor_server.rs
@@ -27,7 +27,7 @@ use std::collections::HashMap;
 use std::convert::TryInto;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 use sysinfo::{MemoryRefreshKind, System};
 use tokio::sync::mpsc;
 
@@ -47,8 +47,8 @@ use ballista_core::serde::protobuf::{
     executor_metric, executor_status,
     scheduler_grpc_client::SchedulerGrpcClient,
 };
-use ballista_core::serde::scheduler::PartitionId;
 use ballista_core::serde::scheduler::TaskDefinition;
+use ballista_core::serde::scheduler::TaskKey;
 
 use ballista_core::serde::scheduler::from_proto::{
     get_task_definition, get_task_definition_vec,
@@ -63,11 +63,19 @@ use tokio::task::JoinHandle;
 
 use crate::cpu_bound_executor::DedicatedExecutor;
 use crate::executor::Executor;
-use crate::executor_process::{ExecutorProcessConfig, remove_job_dir};
+use crate::executor_process::{ExecutorProcessConfig, remove_job_data};
+use crate::health::ExecutorHealth;
 use crate::metrics::ExecutorMetricCollectionPolicy;
 use crate::shutdown::ShutdownNotifier;
 use crate::{TaskExecutionTimes, as_task_status};
 
+/// Number of consecutive heartbeat failures after which the executor
+/// initiates its own shutdown, letting k8s (or the operator) restart the
+/// pod. The typical trigger is a `FailedPrecondition` from a newer
+/// scheduler on a bumped `BALLISTA_PROTOCOL_VERSION`; a restart will keep
+/// crash-looping until the executor image is bumped to match.
+const HEARTBEAT_FAILURE_TERMINATION_THRESHOLD: u32 = 5;
+
 type ServerHandle = JoinHandle<Result<(), BallistaError>>;
 type SchedulerClients = Arc<DashMap<String, SchedulerGrpcClient<Channel>>>;
 
@@ -101,8 +109,9 @@ pub async fn startup<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>(
     codec: BallistaCodec<T, U>,
     stop_send: mpsc::Sender<bool>,
     shutdown_noti: &ShutdownNotifier,
+    health: ExecutorHealth,
 ) -> Result<ServerHandle, BallistaError> {
-    let channel_buf_size = executor.concurrent_tasks * 50;
+    let channel_buf_size = executor.vcores * 50;
     let (tx_task, rx_task) = mpsc::channel::<CuratorTaskDefinition>(channel_buf_size);
     let (tx_task_status, rx_task_status) =
         mpsc::channel::<CuratorTaskStatus>(channel_buf_size);
@@ -120,6 +129,7 @@ pub async fn startup<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>(
         config.grpc_max_decoding_message_size as usize,
         config.override_create_grpc_client_endpoint.clone(),
         config.metric_collection_policy,
+        health,
     );
 
     // 1. Start executor grpc service
@@ -222,6 +232,8 @@ pub struct ExecutorServer<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPl
     /// Metric collection policy for this specifix executor
     metric_collection_policy: ExecutorMetricCollectionPolicy,
     override_create_grpc_client_endpoint: Option<EndpointOverrideFn>,
+    /// Shared readiness signal reflected by the /readyz probe.
+    health: ExecutorHealth,
 }
 
 #[derive(Clone)]
@@ -251,6 +263,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
         grpc_max_decoding_message_size: usize,
         override_create_grpc_client_endpoint: Option<EndpointOverrideFn>,
         metric_collection_policy: ExecutorMetricCollectionPolicy,
+        health: ExecutorHealth,
     ) -> Self {
         Self {
             _start_time: SystemTime::now()
@@ -266,6 +279,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
             grpc_max_decoding_message_size,
             override_create_grpc_client_endpoint,
             metric_collection_policy,
+            health,
         }
     }
 
@@ -305,7 +319,9 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
 
     /// 1. First Heartbeat to its registration scheduler, if successful then return; else go next.
     /// 2. Heartbeat to schedulers which has launching tasks to this executor until one succeeds
-    async fn heartbeat(&self) {
+    ///
+    /// Returns `true` iff any scheduler acknowledged the heartbeat.
+    async fn heartbeat(&self) -> bool {
         let status = if TERMINATING.load(Ordering::Acquire) {
             executor_status::Status::Terminating(String::default())
         } else {
@@ -326,7 +342,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
             .await
         {
             Ok(_) => {
-                return;
+                self.health.mark_heartbeat_ok();
+                return true;
             }
             Err(e) => {
                 warn!(
@@ -344,7 +361,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
                 .await
             {
                 Ok(_) => {
-                    break;
+                    self.health.mark_heartbeat_ok();
+                    return true;
                 }
                 Err(e) => {
                     warn!(
@@ -353,6 +371,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
                 }
             }
         }
+        self.health.mark_heartbeat_failed();
+        false
     }
     /// This method should not return Err. If task fails, a failure task status should be sent
     /// to the channel to notify the scheduler.
@@ -368,25 +388,30 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
         let job_id = task.job_id;
         let stage_id = task.stage_id;
         let stage_attempt_num = task.stage_attempt_num;
-        let partition_id = task.partition_id;
+        let global_output_partition_ids = task.global_output_partition_ids;
         let plan = task.plan;
 
-        let part = PartitionId {
+        let key = TaskKey {
             job_id: job_id.clone(),
             stage_id,
-            partition_id,
+            task_id,
         };
 
         let exec = self.executor.execution_engine.create_query_stage_exec(
             job_id.clone(),
             stage_id,
-            partition_id,
+            task_id,
+            global_output_partition_ids,
             plan,
             &self.executor.work_dir,
             &task.session_config,
         );
 
-        let runtime = self.executor.produce_runtime(&task.session_config);
+        let runtime = self.executor.produce_runtime_for_session(
+            &task.session_id,
+            &task.session_config,
+            task.vcores_consumed,
+        );
 
         match (exec, runtime) {
             (Ok(exec), Ok(runtime)) => {
@@ -398,6 +423,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
                         task.session_id,
                         task.session_config,
                         function_registry.scalar_functions.clone(),
+                        function_registry.higher_order_functions.clone(),
                         function_registry.aggregate_functions.clone(),
                         function_registry.window_functions.clone(),
                         runtime,
@@ -406,16 +432,15 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
 
                 info!("Execute task  : [{task_identity}]");
 
+                let task_start = Instant::now();
                 let execution_result = self
                     .executor
-                    .execute_query_stage(
-                        task_id,
-                        part.clone(),
-                        exec.clone(),
-                        task_context,
-                    )
+                    .execute_query_stage(key.clone(), exec.clone(), task_context)
                     .await;
-                info!("Finished task : [{task_identity}]");
+                info!(
+                    "Finished task : [{task_identity}] in {:?}",
+                    task_start.elapsed()
+                );
                 debug!(
                     "Task [{task_identity}], execution statistics: {execution_result:?}"
                 );
@@ -441,9 +466,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
                 let task_status = as_task_status(
                     execution_result,
                     executor_id.clone(),
-                    task_id,
                     stage_attempt_num,
-                    part,
+                    key.clone(),
                     operator_metrics,
                     task_execution_times,
                 );
@@ -466,9 +490,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
                 let task_status = as_task_status(
                     Err(e),
                     self.executor.metadata.id.clone(),
-                    task_id,
                     stage_attempt_num,
-                    part,
+                    key.clone(),
                     None,
                     TaskExecutionTimes {
                         launch_time: task.launch_time,
@@ -627,11 +650,27 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> Heartbeater<T, U>
         let executor_server = self.executor_server.clone();
         let mut heartbeat_shutdown = shutdown_noti.subscribe_for_shutdown();
         let heartbeat_complete = shutdown_noti.shutdown_complete_tx.clone();
+        let notify_shutdown = shutdown_noti.notify_shutdown.clone();
         tokio::spawn(async move {
             info!("Starting heartbeater to send heartbeat the scheduler periodically");
+            let mut consecutive_failures: u32 = 0;
             // As long as the shutdown notification has not been received
             while !heartbeat_shutdown.is_shutdown() {
-                executor_server.heartbeat().await;
+                if executor_server.heartbeat().await {
+                    consecutive_failures = 0;
+                } else {
+                    consecutive_failures += 1;
+                    if consecutive_failures >= HEARTBEAT_FAILURE_TERMINATION_THRESHOLD {
+                        error!(
+                            "Heartbeat failed {consecutive_failures} consecutive times; \
+                             initiating executor shutdown. Check for scheduler outage \
+                             or BALLISTA_PROTOCOL_VERSION mismatch."
+                        );
+                        let _ = notify_shutdown.send(());
+                        drop(heartbeat_complete);
+                        return;
+                    }
+                }
                 tokio::select! {
                     _ = tokio::time::sleep(Duration::from_secs(executor_heartbeat_interval_seconds)) => {},
                     _ = heartbeat_shutdown.recv() => {
@@ -759,10 +798,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskRunnerPool<T,
 
             // Use a dedicated executor for CPU bound tasks so that the main tokio
             // executor can still answer requests even when under load
-            let dedicated_executor = DedicatedExecutor::new(
-                "task_runner",
-                executor_server.executor.concurrent_tasks,
-            );
+            let dedicated_executor =
+                DedicatedExecutor::new("task_runner", executor_server.executor.vcores);
 
             // As long as the shutdown notification has not been received
             while !task_runner_shutdown.is_shutdown() {
@@ -776,15 +813,14 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskRunnerPool<T,
                 };
                 if let Some(curator_task) = maybe_task {
                     let task_identity = format!(
-                        "TID {} {}/{}.{}/{}.{}",
-                        &curator_task.task.task_id,
-                        &curator_task.task.job_id,
-                        &curator_task.task.stage_id,
-                        &curator_task.task.stage_attempt_num,
-                        &curator_task.task.partition_id,
-                        &curator_task.task.task_attempt_num,
+                        "TID {}/{}.{}/{}.{}",
+                        curator_task.task.job_id,
+                        curator_task.task.stage_id,
+                        curator_task.task.stage_attempt_num,
+                        curator_task.task.task_id,
+                        curator_task.task.task_attempt_num,
                     );
-                    debug!("Received task {:?}", &task_identity);
+                    debug!("Received task {:?}", task_identity);
 
                     let server = executor_server.clone();
                     dedicated_executor.spawn(async move {
@@ -824,6 +860,10 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorGrpc
                         self.executor.function_registry.scalar_functions.clone(),
                         self.executor.function_registry.aggregate_functions.clone(),
                         self.executor.function_registry.window_functions.clone(),
+                        self.executor
+                            .function_registry
+                            .higher_order_functions
+                            .clone(),
                         self.codec.clone(),
                     )
                     .map_err(|e| Status::invalid_argument(format!("{e}")))?,
@@ -853,6 +893,10 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorGrpc
                 self.executor.function_registry.scalar_functions.clone(),
                 self.executor.function_registry.aggregate_functions.clone(),
                 self.executor.function_registry.window_functions.clone(),
+                self.executor
+                    .function_registry
+                    .higher_order_functions
+                    .clone(),
                 self.codec.clone(),
             )
             .map_err(|e| Status::invalid_argument(format!("{e}")))?;
@@ -905,10 +949,9 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorGrpc
             if let Err(e) = self
                 .executor
                 .cancel_task(
-                    task.task_id as usize,
                     task.job_id.into(),
                     task.stage_id as usize,
-                    task.partition_id as usize,
+                    task.task_id as usize,
                 )
                 .await
             {
@@ -924,9 +967,10 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorGrpc
         &self,
         request: Request<RemoveJobDataParams>,
     ) -> Result<Response<RemoveJobDataResult>, Status> {
-        let job_id = request.into_inner().job_id.into();
+        let params = request.into_inner();
+        let job_id = params.job_id.into();
 
-        remove_job_dir(&self.executor.work_dir, &job_id)
+        remove_job_data(&self.executor.work_dir, &job_id, &params.remove_stage_ids)
             .await
             .map_err(|e| Status::invalid_argument(e.to_string()))?;
 
diff --git a/ballista/executor/src/health.rs b/ballista/executor/src/health.rs
new file mode 100644
index 0000000000..74ab1b4a1f
--- /dev/null
+++ b/ballista/executor/src/health.rs
@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared readiness state for Kubernetes-style health probes.
+//!
+//! The state itself (`ExecutorHealth`) is always compiled: the heartbeat
+//! loops in `execution_loop` and `executor_server` flip it on every RPC
+//! outcome regardless of whether anything is observing it. That keeps the
+//! library transport-neutral — callers that embed the executor as a library
+//! never have to link in axum, bind a port, or field an unwanted HTTP
+//! surface.
+//!
+//! The HTTP surface (`/healthz` and `/readyz` served over axum) is behind
+//! `#[cfg(feature = "build-binary")]` and lives here for symmetry, but is
+//! spawned from `bin/main.rs`, not from `start_executor_process`.
+//!
+//! `/healthz` is deliberately independent of scheduler connectivity: if the
+//! scheduler is flapping or on the wrong protocol version, restarting this
+//! pod will not help — that would just cascade into a fleet-wide crash
+//! loop. `/readyz` reflects the last heartbeat: startup begins not-ready, a
+//! successful heartbeat flips it ready, a failure flips it back. Failing
+//! `/readyz` removes the executor from any `Service` endpoint but does not
+//! restart the pod.
+
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+/// Shared executor health state, cheap to clone (internally arc'd).
+#[derive(Clone, Default)]
+pub struct ExecutorHealth {
+    heartbeat_ok: Arc<AtomicBool>,
+}
+
+impl ExecutorHealth {
+    /// Creates a new health handle. Starts in the not-ready state; the
+    /// executor's first successful heartbeat flips it to ready.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Marks the last heartbeat as successful. Called from the heartbeat
+    /// loop after the scheduler acknowledges the heartbeat.
+    pub fn mark_heartbeat_ok(&self) {
+        self.heartbeat_ok.store(true, Ordering::Release);
+    }
+
+    /// Marks the last heartbeat as failed. Called from the heartbeat loop
+    /// when the scheduler returns an error, including
+    /// `FailedPrecondition` for a protocol-version mismatch.
+    pub fn mark_heartbeat_failed(&self) {
+        self.heartbeat_ok.store(false, Ordering::Release);
+    }
+
+    /// True once the executor's most recent heartbeat to the scheduler
+    /// succeeded. Embedders that host `ExecutorHealth` in their own process
+    /// can fold this into their app-wide readiness check the same way the
+    /// scheduler exposes `SchedulerServer::is_ready`.
+    pub fn is_ready(&self) -> bool {
+        self.heartbeat_ok.load(Ordering::Acquire)
+    }
+}
+
+#[cfg(feature = "build-binary")]
+mod server {
+    use super::ExecutorHealth;
+    use axum::{
+        Router,
+        extract::State,
+        http::StatusCode,
+        response::{IntoResponse, Response},
+        routing::get,
+    };
+    use ballista_core::error::BallistaError;
+    use log::{info, warn};
+    use std::net::SocketAddr;
+    use tokio::task::JoinHandle;
+
+    /// Spawns an HTTP server serving `/healthz` and `/readyz`. The task
+    /// shuts down when the receiver at `shutdown` fires.
+    pub fn spawn_health_server(
+        addr: SocketAddr,
+        health: ExecutorHealth,
+        shutdown: tokio::sync::oneshot::Receiver<()>,
+    ) -> JoinHandle<Result<(), BallistaError>> {
+        tokio::spawn(async move {
+            let router = Router::new()
+                .route("/healthz", get(healthz))
+                .route("/readyz", get(readyz))
+                .with_state(health);
+
+            let listener = tokio::net::TcpListener::bind(addr).await.map_err(|e| {
+                BallistaError::General(format!(
+                    "failed to bind health server on {addr}: {e}"
+                ))
+            })?;
+
+            info!("Executor health server listening on {addr}");
+
+            axum::serve(listener, router.into_make_service())
+                .with_graceful_shutdown(async move {
+                    let _ = shutdown.await;
+                })
+                .await
+                .map_err(|e| {
+                    warn!("Executor health server exited with error: {e}");
+                    BallistaError::General(format!("health server error: {e}"))
+                })
+        })
+    }
+
+    async fn healthz() -> Response {
+        (StatusCode::OK, "ok\n").into_response()
+    }
+
+    async fn readyz(State(health): State<ExecutorHealth>) -> Response {
+        if health.is_ready() {
+            (StatusCode::OK, "ready\n").into_response()
+        } else {
+            (
+                StatusCode::SERVICE_UNAVAILABLE,
+                "not ready: no successful heartbeat yet\n",
+            )
+                .into_response()
+        }
+    }
+}
+
+#[cfg(feature = "build-binary")]
+pub use server::spawn_health_server;
diff --git a/ballista/executor/src/lib.rs b/ballista/executor/src/lib.rs
index f0d0f995ec..7eaf453706 100644
--- a/ballista/executor/src/lib.rs
+++ b/ballista/executor/src/lib.rs
@@ -37,8 +37,12 @@ pub mod executor_process;
 pub mod executor_server;
 /// Arrow Flight service for streaming shuffle data between executors.
 pub mod flight_service;
+/// HTTP server for Kubernetes-style /healthz and /readyz probes.
+pub mod health;
 /// Metrics collection for executor runtime statistics.
 pub mod metrics;
+/// Session-scoped cache of shared executor runtime environments.
+pub mod runtime_cache;
 /// Graceful shutdown coordination for executor components.
 pub mod shutdown;
 /// Signal handling for process termination.
@@ -62,7 +66,7 @@ use ballista_core::serde::protobuf::{
     FailedTask, OperatorMetricsSet, ShuffleWritePartition, SuccessfulTask, TaskStatus,
     task_status,
 };
-use ballista_core::serde::scheduler::PartitionId;
+use ballista_core::serde::scheduler::TaskKey;
 use ballista_core::utils::GrpcServerConfig;
 
 /// [ArrowFlightServerProvider] provides a function which creates a new Arrow Flight server.
@@ -102,26 +106,24 @@ pub struct TaskExecutionTimes {
 pub fn as_task_status(
     execution_result: ballista_core::error::Result<Vec<ShuffleWritePartition>>,
     executor_id: String,
-    task_id: usize,
     stage_attempt_num: usize,
-    partition_id: PartitionId,
+    key: TaskKey,
     operator_metrics: Option<Vec<OperatorMetricsSet>>,
     execution_times: TaskExecutionTimes,
 ) -> TaskStatus {
     let metrics = operator_metrics.unwrap_or_default();
+    let task_id = key.task_id;
     match execution_result {
         Ok(partitions) => {
             debug!(
-                "Task {:?} finished with operator_metrics array size {}",
-                task_id,
+                "Task {task_id} finished with operator_metrics array size {}",
                 metrics.len()
             );
             TaskStatus {
                 task_id: task_id as u32,
-                job_id: partition_id.job_id.into(),
-                stage_id: partition_id.stage_id as u32,
+                job_id: key.job_id.clone().into(),
+                stage_id: key.stage_id as u32,
                 stage_attempt_num: stage_attempt_num as u32,
-                partition_id: partition_id.partition_id as u32,
                 launch_time: execution_times.launch_time,
                 start_exec_time: execution_times.start_exec_time,
                 end_exec_time: execution_times.end_exec_time,
@@ -134,14 +136,13 @@ pub fn as_task_status(
         }
         Err(e) => {
             let error_msg = e.to_string();
-            info!("Task {task_id:?} failed: {error_msg}");
+            info!("Task {task_id} failed: {error_msg}");
 
             TaskStatus {
                 task_id: task_id as u32,
-                job_id: partition_id.job_id.into(),
-                stage_id: partition_id.stage_id as u32,
+                job_id: key.job_id.clone().into(),
+                stage_id: key.stage_id as u32,
                 stage_attempt_num: stage_attempt_num as u32,
-                partition_id: partition_id.partition_id as u32,
                 launch_time: execution_times.launch_time,
                 start_exec_time: execution_times.start_exec_time,
                 end_exec_time: execution_times.end_exec_time,
diff --git a/ballista/executor/src/metrics/mod.rs b/ballista/executor/src/metrics/mod.rs
index 97fde7066b..7bcf45a6a8 100644
--- a/ballista/executor/src/metrics/mod.rs
+++ b/ballista/executor/src/metrics/mod.rs
@@ -31,7 +31,7 @@ pub trait ExecutorMetricsCollector: Send + Sync {
         &self,
         job_id: &JobId,
         stage_id: usize,
-        partition: usize,
+        task_id: usize,
         plan: Arc<dyn QueryStageExecutor>,
     );
 }
@@ -46,11 +46,11 @@ impl ExecutorMetricsCollector for LoggingMetricsCollector {
         &self,
         job_id: &JobId,
         stage_id: usize,
-        partition: usize,
+        task_id: usize,
         plan: Arc<dyn QueryStageExecutor>,
     ) {
         debug!(
-            "\n=== [{job_id}/{stage_id}/{partition}] Physical plan with metrics ===\n{plan}\n"
+            "\n=== [{job_id}/{stage_id}/{task_id}] Physical plan with metrics ===\n{plan}\n"
         );
     }
 }
diff --git a/ballista/executor/src/runtime_cache.rs b/ballista/executor/src/runtime_cache.rs
new file mode 100644
index 0000000000..617cfabb27
--- /dev/null
+++ b/ballista/executor/src/runtime_cache.rs
@@ -0,0 +1,231 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Session-scoped reuse of executor runtime state.
+
+use std::num::NonZeroUsize;
+use std::sync::Arc;
+
+use ballista_core::RuntimeProducer;
+use datafusion::execution::runtime_env::RuntimeEnv;
+use datafusion::prelude::SessionConfig;
+use lru::LruCache;
+use parking_lot::Mutex;
+
+/// Derives a task's runtime from a shared base [`RuntimeEnv`] by installing a
+/// fresh per-task memory pool sized to the task's vcore claim. The base's
+/// object-store registry and the cache manager's underlying file-metadata
+/// (footer) cache are preserved via
+/// [`RuntimeEnvBuilder::from_runtime_env`](datafusion::execution::runtime_env::RuntimeEnvBuilder::from_runtime_env)
+/// — the outer `CacheManager` is rebuilt around that same inner cache — so
+/// read-side state is shared while the memory pool stays per task.
+///
+/// `vcores_consumed` is the number of vcores this task claims from the
+/// executor's budget (`min(input_partition_ids.len(), budget)` for non-collapse
+/// stages, `1` for collapse). The policy scales the pool proportionally so a
+/// task claiming N/total vcores gets N/total of the executor's memory budget.
+pub type MemoryPoolPolicy = Arc<
+    dyn Fn(
+            Arc<RuntimeEnv>,
+            &SessionConfig,
+            u32,
+        ) -> datafusion::error::Result<Arc<RuntimeEnv>>
+        + Send
+        + Sync,
+>;
+
+/// Produces a task's [`RuntimeEnv`], optionally reusing read-side state across
+/// the tasks of a session.
+///
+/// The executor calls [`produce_runtime`](Self::produce_runtime) for every task;
+/// implementors decide whether and how to share base runtime state (object-store
+/// clients, file-metadata cache) between a session's tasks.
+/// [`DefaultSessionRuntimeCache`] is the built-in implementation — provide a
+/// custom one to change the caching/sharing strategy.
+pub trait SessionRuntimeCache: Send + Sync {
+    /// Returns the per-task runtime for `session_id`, sized to `vcores_consumed`
+    /// vcores of the executor's memory budget.
+    fn produce_runtime(
+        &self,
+        session_id: &str,
+        config: &SessionConfig,
+        vcores_consumed: u32,
+    ) -> datafusion::error::Result<Arc<RuntimeEnv>>;
+}
+
+/// A bounded, session-keyed cache of shared *base* [`RuntimeEnv`]s.
+///
+/// A base env carries the read-side state safe to share across all tasks of a
+/// session: the object-store registry and the cache manager (whose Parquet
+/// footer cache is thereby reused across the session's tasks and queries).
+/// `RuntimeEnvBuilder::from_runtime_env` also carries over the disk manager
+/// (rooted at the executor's `work_dir`), so a session's tasks share one
+/// `DiskManager` too; this is safe because spill temp files are uniquely
+/// named, matching the standard one-`RuntimeEnv`-per-`SessionContext` model.
+/// Each task's real runtime is produced by applying [`MemoryPoolPolicy`] to
+/// the shared base, which installs a fresh per-task memory pool — so memory
+/// isolation is unchanged.
+///
+/// The cache is bounded by an LRU of `capacity` sessions. A capacity of `0`
+/// disables caching entirely: every call builds a fresh base env, matching the
+/// behavior of building a runtime per task.
+pub struct DefaultSessionRuntimeCache {
+    base_producer: RuntimeProducer,
+    pool_policy: MemoryPoolPolicy,
+    cache: Option<Mutex<LruCache<String, Arc<RuntimeEnv>>>>,
+}
+
+impl DefaultSessionRuntimeCache {
+    /// Creates a new cache that produces per-task runtimes from `base_producer`
+    /// (invoked at most once per cached session) and `pool_policy` (invoked on
+    /// every call). `capacity` bounds the number of distinct sessions kept in
+    /// the LRU; `0` disables caching.
+    pub fn new(
+        base_producer: RuntimeProducer,
+        pool_policy: MemoryPoolPolicy,
+        capacity: usize,
+    ) -> Self {
+        let cache = NonZeroUsize::new(capacity).map(|cap| Mutex::new(LruCache::new(cap)));
+        Self {
+            base_producer,
+            pool_policy,
+            cache,
+        }
+    }
+}
+
+impl SessionRuntimeCache for DefaultSessionRuntimeCache {
+    /// Returns the per-task runtime for `session_id`, reusing a cached base env
+    /// when present and building + caching one on miss.
+    fn produce_runtime(
+        &self,
+        session_id: &str,
+        config: &SessionConfig,
+        vcores_consumed: u32,
+    ) -> datafusion::error::Result<Arc<RuntimeEnv>> {
+        let base = match &self.cache {
+            None => (self.base_producer)(config)?,
+            Some(cache) => {
+                if let Some(base) = cache.lock().get(session_id) {
+                    base.clone()
+                } else {
+                    // Build the base env without holding the lock, so a miss
+                    // never stalls other sessions' lookups. A rare concurrent
+                    // first-miss for the same session may build twice; that is
+                    // harmless (idempotent, cheap) — the last writer wins and
+                    // the extra env is dropped.
+                    let base = (self.base_producer)(config)?;
+                    cache.lock().put(session_id.to_string(), base.clone());
+                    base
+                }
+            }
+        };
+        (self.pool_policy)(base, config, vcores_consumed)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::execution::memory_pool::GreedyMemoryPool;
+    use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+
+    fn base_producer() -> RuntimeProducer {
+        Arc::new(|_| Ok(Arc::new(RuntimeEnv::default())))
+    }
+
+    /// Identity policy: the task shares the base env unchanged (no pool swap).
+    fn identity_policy() -> MemoryPoolPolicy {
+        Arc::new(|base, _, _| Ok(base))
+    }
+
+    /// Rebuilding policy: mimics production — a fresh per-task pool layered onto
+    /// the shared base, preserving the base's read-side state.
+    fn per_task_pool_policy() -> MemoryPoolPolicy {
+        Arc::new(|base, _, _| {
+            RuntimeEnvBuilder::from_runtime_env(&base)
+                .with_memory_pool(Arc::new(GreedyMemoryPool::new(1024)))
+                .build_arc()
+        })
+    }
+
+    #[test]
+    fn same_session_shares_cache_manager() {
+        let cache =
+            DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 4);
+        let cfg = SessionConfig::new();
+        let e1 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        let e2 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        assert!(Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager));
+    }
+
+    #[test]
+    fn different_sessions_get_different_base() {
+        let cache =
+            DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 4);
+        let cfg = SessionConfig::new();
+        let e1 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        let e2 = cache.produce_runtime("s2", &cfg, 1).unwrap();
+        assert!(!Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager));
+    }
+
+    #[test]
+    fn per_task_pool_shares_footer_cache_but_not_env() {
+        let cache =
+            DefaultSessionRuntimeCache::new(base_producer(), per_task_pool_policy(), 4);
+        let cfg = SessionConfig::new();
+        let e1 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        let e2 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        // Different runtime envs (fresh per-task pool)...
+        assert!(!Arc::ptr_eq(&e1, &e2));
+        // ...but the shared read-side state is reused: object-store registry
+        // passes through unchanged, and the inner footer (file-metadata) cache
+        // is the same instance even though the outer CacheManager wrapper is
+        // rebuilt. Do NOT compare the outer Arc<CacheManager>.
+        assert!(Arc::ptr_eq(
+            &e1.object_store_registry,
+            &e2.object_store_registry
+        ));
+        assert!(Arc::ptr_eq(
+            &e1.cache_manager.get_file_metadata_cache(),
+            &e2.cache_manager.get_file_metadata_cache(),
+        ));
+    }
+
+    #[test]
+    fn capacity_zero_disables_cache() {
+        let cache =
+            DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 0);
+        let cfg = SessionConfig::new();
+        let e1 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        let e2 = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        assert!(!Arc::ptr_eq(&e1.cache_manager, &e2.cache_manager));
+    }
+
+    #[test]
+    fn evicts_least_recently_used() {
+        let cache =
+            DefaultSessionRuntimeCache::new(base_producer(), identity_policy(), 2);
+        let cfg = SessionConfig::new();
+        let s1_a = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        cache.produce_runtime("s2", &cfg, 1).unwrap();
+        // Inserting s3 (capacity 2) evicts the least-recently-used, s1.
+        cache.produce_runtime("s3", &cfg, 1).unwrap();
+        let s1_b = cache.produce_runtime("s1", &cfg, 1).unwrap();
+        assert!(!Arc::ptr_eq(&s1_a.cache_manager, &s1_b.cache_manager));
+    }
+}
diff --git a/ballista/executor/src/standalone.rs b/ballista/executor/src/standalone.rs
index bd72dee4b4..fd9841a0e6 100644
--- a/ballista/executor/src/standalone.rs
+++ b/ballista/executor/src/standalone.rs
@@ -27,7 +27,7 @@ use ballista_core::extension::SessionConfigExt;
 use ballista_core::registry::BallistaFunctionRegistry;
 use ballista_core::utils::{GrpcServerConfig, default_config_producer};
 use ballista_core::{
-    BALLISTA_VERSION,
+    BALLISTA_PROTOCOL_VERSION, BALLISTA_VERSION,
     error::Result,
     serde::BallistaCodec,
     serde::protobuf::{ExecutorRegistration, scheduler_grpc_client::SchedulerGrpcClient},
@@ -50,7 +50,7 @@ use uuid::Uuid;
 /// components.
 pub async fn new_standalone_executor_from_state(
     scheduler: SchedulerGrpcClient<Channel>,
-    concurrent_tasks: usize,
+    vcores: usize,
     session_state: &SessionState,
 ) -> Result<()> {
     let logical = session_state.config().ballista_logical_extension_codec();
@@ -69,7 +69,7 @@ pub async fn new_standalone_executor_from_state(
 
     new_standalone_executor_from_builder(
         scheduler,
-        concurrent_tasks,
+        vcores,
         config_producer,
         runtime_producer,
         codec,
@@ -87,7 +87,7 @@ pub async fn new_standalone_executor_from_state(
 /// The executor binds to a random available port on localhost.
 pub async fn new_standalone_executor_from_builder(
     scheduler: SchedulerGrpcClient<Channel>,
-    concurrent_tasks: usize,
+    vcores: usize,
     config_producer: ConfigProducer,
     runtime_producer: RuntimeProducer,
     codec: BallistaCodec,
@@ -106,10 +106,11 @@ pub async fn new_standalone_executor_from_builder(
         grpc_port: 50020,
         specification: Some(
             ExecutorSpecification::default()
-                .with_task_slots(concurrent_tasks as u32)
+                .with_vcores(vcores as u32)
                 .into(),
         ),
         os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+        ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
     };
 
     let config = config_producer();
@@ -126,7 +127,7 @@ pub async fn new_standalone_executor_from_builder(
         config_producer,
         Arc::new(function_registry),
         Arc::new(LoggingMetricsCollector::default()),
-        concurrent_tasks,
+        vcores,
     ));
 
     let service = BallistaFlightService::new(work_dir);
@@ -142,7 +143,13 @@ pub async fn new_standalone_executor_from_builder(
             )),
     );
 
-    tokio::spawn(execution_loop::poll_loop(scheduler, executor, codec));
+    tokio::spawn(execution_loop::poll_loop(
+        scheduler,
+        executor,
+        codec,
+        None,
+        crate::health::ExecutorHealth::new(),
+    ));
     Ok(())
 }
 
@@ -150,7 +157,7 @@ pub async fn new_standalone_executor_from_builder(
 /// set as default.
 pub async fn new_standalone_executor(
     scheduler: SchedulerGrpcClient<Channel>,
-    concurrent_tasks: usize,
+    vcores: usize,
     codec: BallistaCodec,
 ) -> Result<()> {
     use ballista_core::extension::{
@@ -170,7 +177,7 @@ pub async fn new_standalone_executor(
 
     new_standalone_executor_from_builder(
         scheduler,
-        concurrent_tasks,
+        vcores,
         Arc::new(default_config_producer),
         runtime_producer,
         codec,
diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml
index f882d26f38..d2012b3469 100644
--- a/ballista/scheduler/Cargo.toml
+++ b/ballista/scheduler/Cargo.toml
@@ -19,7 +19,7 @@
 name = "ballista-scheduler"
 description = "Ballista Distributed Compute - Scheduler"
 license = "Apache-2.0"
-version = "53.0.0"
+version = "54.0.0"
 homepage = "https://datafusion.apache.org/ballista/"
 repository = "https://github.com/apache/datafusion-ballista"
 readme = "README.md"
@@ -50,7 +50,7 @@ arrow-flight = { workspace = true }
 async-trait = { workspace = true }
 axum = "0.8.9"
 tower-http = { version = "0.7", features = ["cors"] }
-ballista-core = { path = "../core", version = "53.0.0" }
+ballista-core = { path = "../core", version = "54.0.0" }
 clap = { workspace = true, optional = true }
 dashmap = { workspace = true }
 datafusion = { workspace = true }
@@ -79,7 +79,12 @@ tracing-appender = { workspace = true, optional = true }
 tracing-subscriber = { workspace = true, optional = true }
 uuid = { workspace = true }
 
+[[test]]
+name = "tpch_plan_stability"
+path = "tests/tpch_plan_stability/main.rs"
+
 [dev-dependencies]
+regex = "1"
 rstest = { workspace = true }
 
 [build-dependencies]
diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs
index ff886a69f8..a0e1a60d57 100644
--- a/ballista/scheduler/src/api/handlers.rs
+++ b/ballista/scheduler/src/api/handlers.rs
@@ -22,9 +22,12 @@ use axum::{
     extract::{Path, State},
     response::{IntoResponse, Response},
 };
+use ballista_core::serde::protobuf::failed_task::FailedReason::{
+    ExecutionError, ExecutorLost, FetchPartitionError, IoError, ResultLost, TaskKilled,
+};
 use ballista_core::serde::protobuf::job_status::Status;
 use ballista_core::serde::protobuf::{
-    ExecutorMetric, executor_metric::Metric, task_status,
+    ExecutorMetric, FailedTask, executor_metric::Metric, task_status,
 };
 use ballista_core::serde::scheduler::{
     ExecutorOperatingSystemSpecification, ExecutorSpecification,
@@ -141,8 +144,11 @@ pub struct TaskSummary {
     pub id: usize,
     /// Task status
     pub status: TaskStatus,
-    /// partition id
-    pub partition_id: u32,
+    /// Global partition ids covered by this task. For a single-partition
+    /// task this is a one-element list — JSON-compatible with the old
+    /// scalar `partition_id` for callers that read `partition_id[0]`, and
+    /// honestly plural for multi-partition tasks.
+    pub partition_id: Vec<u32>,
     /// Scheduler schedule time
     pub scheduled_time: u64,
     /// Scheduler launch time (ms since epoch)
@@ -165,14 +171,17 @@ pub struct TaskSummary {
 pub enum TaskStatus {
     Running,
     Successful,
-    Failed,
+    Failed { reason: String, error: String },
 }
 
 impl From<&task_status::Status> for TaskStatus {
     fn from(value: &task_status::Status) -> Self {
         match value {
             task_status::Status::Running(_) => TaskStatus::Running,
-            task_status::Status::Failed(_) => TaskStatus::Failed,
+            task_status::Status::Failed(failed) => TaskStatus::Failed {
+                reason: failed_reason(failed),
+                error: failed.error.clone(),
+            },
             task_status::Status::Successful(_) => TaskStatus::Successful,
         }
     }
@@ -558,35 +567,39 @@ pub async fn get_query_stages<
                         summary.tasks = running_stage
                             .task_infos
                             .iter()
-                            .enumerate()
-                            .map(|(partition_id, task_info)| {
-                                task_info.as_ref().map(|info| {
-                                    let (input_rows, output_rows) = running_stage
-                                        .stage_metrics
-                                        .as_deref()
-                                        .map(|metrics| {
-                                            get_partition_counts(metrics, partition_id)
-                                        })
-                                        .unwrap_or((0, 0));
-
-                                    let start_exec_time = info.start_exec_time as u64;
-                                    let end_exec_time = info.end_exec_time as u64;
-
-                                    let task_status: TaskStatus = (&info.task_status).into();
-
-                                    TaskSummary {
-                                        id: info.task_id,
-                                        partition_id: partition_id as u32,
-                                        scheduled_time: info.scheduled_time as u64,
-                                        launch_time: info.launch_time as u64,
-                                        start_exec_time,
-                                        end_exec_time,
-                                        exec_duration: end_exec_time.saturating_sub(start_exec_time),
-                                        finish_time: info.finish_time as u64,
-                                        input_rows,
-                                        output_rows,
-                                        status: task_status
-                                    }
+                            .map(|info| {
+                                let (input_rows, output_rows) = running_stage
+                                    .stage_metrics
+                                    .as_deref()
+                                    .map(|metrics| {
+                                        get_partition_counts(
+                                            metrics,
+                                            &info.global_input_partition_ids,
+                                        )
+                                    })
+                                    .unwrap_or((0, 0));
+
+                                let start_exec_time = info.start_exec_time as u64;
+                                let end_exec_time = info.end_exec_time as u64;
+
+                                let task_status: TaskStatus = (&info.task_status).into();
+
+                                Some(TaskSummary {
+                                    id: info.task_id,
+                                    partition_id: info
+                                        .global_input_partition_ids
+                                        .iter()
+                                        .map(|&p| p as u32)
+                                        .collect(),
+                                    scheduled_time: info.scheduled_time as u64,
+                                    launch_time: info.launch_time as u64,
+                                    start_exec_time,
+                                    end_exec_time,
+                                    exec_duration: end_exec_time.saturating_sub(start_exec_time),
+                                    finish_time: info.finish_time as u64,
+                                    input_rows,
+                                    output_rows,
+                                    status: task_status
                                 })
                             })
                             .collect();
@@ -611,11 +624,10 @@ pub async fn get_query_stages<
                         summary.tasks = completed_stage
                             .task_infos
                             .iter()
-                            .enumerate()
-                            .map(|(partition_id, task_info)| {
+                            .map(|task_info| {
                                 let (input_rows, output_rows) = get_partition_counts(
                                     &completed_stage.stage_metrics,
-                                    partition_id,
+                                    &task_info.global_input_partition_ids,
                                 );
 
                                 let start_exec_time = task_info.start_exec_time as u64;
@@ -623,7 +635,11 @@ pub async fn get_query_stages<
                                 let task_status = (&task_info.task_status).into();
                                 Some(TaskSummary {
                                     id: task_info.task_id,
-                                    partition_id: partition_id as u32,
+                                    partition_id: task_info
+                                        .global_input_partition_ids
+                                        .iter()
+                                        .map(|&p| p as u32)
+                                        .collect(),
                                     scheduled_time: task_info.scheduled_time as u64,
                                     launch_time: task_info.launch_time as u64,
                                     start_exec_time,
@@ -637,6 +653,51 @@ pub async fn get_query_stages<
                             })
                             .collect();
                     }
+                    ExecutionStage::Failed(failed_stage) => {
+                        let metrics = failed_stage.stage_metrics.as_deref().unwrap_or(&[]);
+                        summary.stage_plan = Some(match plan_format {
+                            PlanFormat::Default => displayable(failed_stage.plan.as_ref()).indent(false).to_string(),
+                            PlanFormat::Tree => displayable(failed_stage.plan.as_ref()).tree_render().to_string(),
+                            PlanFormat::Metrics => format_stage_metrics(failed_stage.plan.as_ref(), metrics),
+                        });
+                        summary.input_rows = get_combined_count(metrics, "input_rows");
+                        summary.output_rows = get_combined_count(metrics, "output_rows");
+                        summary.elapsed_compute =
+                            get_finished_stage_time(&failed_stage.task_infos);
+
+                        summary.tasks = failed_stage
+                            .task_infos
+                            .iter()
+                            .map(|info| {
+                                let (input_rows, output_rows) = get_partition_counts(
+                                    metrics,
+                                    &info.global_input_partition_ids,
+                                );
+
+                                let start_exec_time = info.start_exec_time as u64;
+                                let end_exec_time = info.end_exec_time as u64;
+                                let task_status: TaskStatus = (&info.task_status).into();
+
+                                Some(TaskSummary {
+                                    id: info.task_id,
+                                    partition_id: info
+                                        .global_input_partition_ids
+                                        .iter()
+                                        .map(|&p| p as u32)
+                                        .collect(),
+                                    scheduled_time: info.scheduled_time as u64,
+                                    launch_time: info.launch_time as u64,
+                                    start_exec_time,
+                                    end_exec_time,
+                                    exec_duration: end_exec_time.saturating_sub(start_exec_time),
+                                    finish_time: info.finish_time as u64,
+                                    input_rows,
+                                    output_rows,
+                                    status: task_status,
+                                })
+                            })
+                            .collect();
+                    }
                     _ => {}
                 }
                 summary.task_duration_percentiles = task_duration_percentiles(&summary.tasks);
@@ -741,13 +802,10 @@ fn format_job_status(status: &Option<Status>, elapsed_ms: u64) -> (String, Strin
     }
 }
 
-fn get_running_stage_time(
-    task_infos: &[Option<TaskInfo>],
-    current_time: u128,
-) -> Option<String> {
+fn get_running_stage_time(task_infos: &[TaskInfo], current_time: u128) -> Option<String> {
     let min_start = task_infos
         .iter()
-        .flat_map(|t| t.as_ref().map(|t| t.start_exec_time))
+        .map(|t| t.start_exec_time)
         .filter(|t| *t > 0)
         .min();
 
@@ -761,6 +819,19 @@ fn get_running_stage_time(
     }
 }
 
+fn failed_reason(failed: &FailedTask) -> String {
+    match &failed.failed_reason {
+        Some(ExecutionError(_)) => "ExecutionError",
+        Some(FetchPartitionError(_)) => "FetchPartitionError",
+        Some(IoError(_)) => "IoError",
+        Some(ExecutorLost(_)) => "ExecutorLost",
+        Some(ResultLost(_)) => "ResultLost",
+        Some(TaskKilled(_)) => "TaskKilled",
+        None => "Failed",
+    }
+    .to_string()
+}
+
 fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option<String> {
     let min_start = task_infos
         .iter()
@@ -784,20 +855,31 @@ fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option<String> {
     }
 }
 
-fn get_partition_counts(metrics: &[MetricsSet], partition_id: usize) -> (usize, usize) {
-    let input_rows = get_partition_count(metrics, partition_id, "input_rows");
-    let output_rows = get_partition_count(metrics, partition_id, "output_rows");
+/// Sum a task's `input_rows` / `output_rows` across the global partitions the
+/// task owns. Metrics are keyed by global partition id — for single-partition
+/// tasks `partitions` is a one-element slice; for multi-partition tasks it is
+/// the task's `global_input_partition_ids`.
+fn get_partition_counts(metrics: &[MetricsSet], partitions: &[usize]) -> (usize, usize) {
+    let input_rows = get_partition_count(metrics, partitions, "input_rows");
+    let output_rows = get_partition_count(metrics, partitions, "output_rows");
     (input_rows, output_rows)
 }
 
-fn get_partition_count(metrics: &[MetricsSet], partition_id: usize, name: &str) -> usize {
+fn get_partition_count(
+    metrics: &[MetricsSet],
+    partitions: &[usize],
+    name: &str,
+) -> usize {
     metrics
         .iter()
         .flat_map(|vec| {
             vec.iter().map(|metric| {
                 let metric_value = metric.value();
-                if metric.partition() == Some(partition_id) && metric_value.name() == name
-                {
+                let owned_by_task = metric
+                    .partition()
+                    .map(|p| partitions.contains(&p))
+                    .unwrap_or(false);
+                if owned_by_task && metric_value.name() == name {
                     metric_value.as_usize()
                 } else {
                     0
@@ -957,6 +1039,8 @@ mod tests {
             end_exec_time: end,
             finish_time: 0,
             task_status: task_status::Status::Running(Default::default()),
+            global_input_partition_ids: vec![],
+            vcores_consumed: 0,
         }
     }
 
@@ -1016,15 +1100,15 @@ mod tests {
     }
 
     #[test]
-    fn test_running_all_none_returns_none() {
-        let tasks: Vec<Option<TaskInfo>> = vec![None, None];
+    fn test_running_all_zero_start_returns_none() {
+        let tasks: Vec<TaskInfo> = vec![make_task_info(0, 0), make_task_info(0, 0)];
         assert_eq!(get_running_stage_time(&tasks, 1000), None);
     }
 
     #[test]
     fn test_running_future_start_returns_none() {
         // start_exec_time beyond current time → elapsed clamped to 0
-        let tasks = vec![Some(make_task_info(u128::MAX, 0))];
+        let tasks = vec![make_task_info(u128::MAX, 0)];
         assert_eq!(get_running_stage_time(&tasks, 1000), None);
     }
 
@@ -1032,7 +1116,7 @@ mod tests {
     fn test_running_past_start_returns_some() {
         let now = 4_000;
         let start = 1_000;
-        let tasks = vec![Some(make_task_info(start, 0))];
+        let tasks = vec![make_task_info(start, 0)];
         assert_eq!(
             get_running_stage_time(&tasks, now),
             Some("3.00s".to_string())
@@ -1040,15 +1124,15 @@ mod tests {
     }
 
     #[test]
-    fn test_running_mixed_some_none_uses_earliest_some() {
+    fn test_running_mixed_zero_start_uses_earliest_nonzero() {
         let now = 3_000;
         let earlier = 1_000;
         let later = 2_000;
         let tasks = vec![
-            None,
-            Some(make_task_info(later, 0)),
-            Some(make_task_info(earlier, 0)),
-            None,
+            make_task_info(0, 0),
+            make_task_info(later, 0),
+            make_task_info(earlier, 0),
+            make_task_info(0, 0),
         ];
         let result = get_running_stage_time(&tasks, now);
         assert_eq!(result, Some("2.00s".to_string()));
diff --git a/ballista/scheduler/src/api/health.rs b/ballista/scheduler/src/api/health.rs
new file mode 100644
index 0000000000..d0ee4a8524
--- /dev/null
+++ b/ballista/scheduler/src/api/health.rs
@@ -0,0 +1,163 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Kubernetes-style health and readiness probes.
+//!
+//! `/healthz` always returns 200 while the process is running — it reports
+//! liveness only, so a probe failure signals "the process is dead" (which is
+//! the sole condition under which k8s should kill and restart the pod).
+//!
+//! `/readyz` returns 200 only when the scheduler is ready to accept work,
+//! defined as having at least `min_ready_executors` live executors registered.
+//! A readiness failure removes the pod from Service endpoints but leaves the
+//! process alone, which is what we want during rolling upgrades: a fresh
+//! scheduler with `BALLISTA_PROTOCOL_VERSION` rejects executors from the
+//! previous release, so it will fail /readyz until new executors register.
+
+use axum::{
+    Router,
+    extract::State,
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    routing::get,
+};
+use datafusion_proto::logical_plan::AsLogicalPlan;
+use datafusion_proto::physical_plan::AsExecutionPlan;
+use std::sync::Arc;
+
+use crate::scheduler_server::SchedulerServer;
+
+/// Routes for k8s liveness/readiness probes. Always mounted, regardless of
+/// the `rest-api` feature flag.
+pub(crate) fn health_routes<
+    T: AsLogicalPlan + Clone + Send + Sync + 'static,
+    U: AsExecutionPlan + Send + Sync + 'static,
+>(
+    scheduler_server: Arc<SchedulerServer<T, U>>,
+) -> Router {
+    Router::new()
+        .route("/healthz", get(healthz))
+        .route("/readyz", get(readyz::<T, U>))
+        .with_state(scheduler_server)
+}
+
+async fn healthz() -> Response {
+    (StatusCode::OK, "ok").into_response()
+}
+
+async fn readyz<
+    T: AsLogicalPlan + Clone + Send + Sync + 'static,
+    U: AsExecutionPlan + Send + Sync + 'static,
+>(
+    State(scheduler): State<Arc<SchedulerServer<T, U>>>,
+) -> Response {
+    let alive = scheduler.state.executor_manager.get_alive_executors().len();
+    let required = scheduler.state.config.min_ready_executors;
+    if scheduler.is_ready() {
+        (
+            StatusCode::OK,
+            format!("ready: {alive} executors registered\n"),
+        )
+            .into_response()
+    } else {
+        (
+            StatusCode::SERVICE_UNAVAILABLE,
+            format!("not ready: {alive}/{required} executors registered\n"),
+        )
+            .into_response()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::SchedulerConfig;
+    use crate::metrics::default_metrics_collector;
+    use crate::scheduler_server::SchedulerServer;
+    use crate::test_utils::test_cluster_context;
+    use ballista_core::serde::BallistaCodec;
+    use ballista_core::serde::protobuf::{
+        ExecutorHeartbeat, ExecutorStatus, executor_status,
+    };
+    use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode};
+
+    async fn scheduler_with_min_ready(
+        min_ready: usize,
+    ) -> Arc<SchedulerServer<LogicalPlanNode, PhysicalPlanNode>> {
+        let config = SchedulerConfig {
+            min_ready_executors: min_ready,
+            ..Default::default()
+        };
+        let mut server = SchedulerServer::new(
+            "localhost:50050".to_owned(),
+            test_cluster_context(),
+            BallistaCodec::default(),
+            Arc::new(config),
+            default_metrics_collector().unwrap(),
+        );
+        server.init().await.expect("scheduler init");
+        Arc::new(server)
+    }
+
+    #[tokio::test]
+    async fn healthz_is_always_ok() {
+        let response = healthz().await;
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn readyz_fails_with_no_executors() {
+        let scheduler = scheduler_with_min_ready(1).await;
+        let response = readyz(State(scheduler)).await;
+        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+    }
+
+    #[tokio::test]
+    async fn readyz_ok_when_min_ready_is_zero() {
+        let scheduler = scheduler_with_min_ready(0).await;
+        let response = readyz(State(scheduler)).await;
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn readyz_ok_when_active_executor_registered() {
+        let scheduler = scheduler_with_min_ready(1).await;
+
+        // Publish an active-status heartbeat so the executor counts as alive.
+        scheduler
+            .state
+            .executor_manager
+            .save_executor_heartbeat(ExecutorHeartbeat {
+                executor_id: "abc".to_owned(),
+                timestamp: std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap()
+                    .as_secs(),
+                metrics: vec![],
+                status: Some(ExecutorStatus {
+                    status: Some(executor_status::Status::Active("".to_string())),
+                }),
+                peak_proc_physical_memory: 0,
+                peak_proc_virtual_memory: 0,
+            })
+            .await
+            .expect("save heartbeat");
+
+        let response = readyz(State(scheduler)).await;
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+}
diff --git a/ballista/scheduler/src/api/mod.rs b/ballista/scheduler/src/api/mod.rs
index c02b47fd4c..81d7177cfa 100644
--- a/ballista/scheduler/src/api/mod.rs
+++ b/ballista/scheduler/src/api/mod.rs
@@ -12,8 +12,10 @@
 
 #[cfg(feature = "rest-api")]
 mod handlers;
+mod health;
 #[cfg(feature = "rest-api")]
 mod routes;
+pub(super) use health::health_routes;
 #[cfg(feature = "rest-api")]
 pub use routes::get_routes;
 
diff --git a/ballista/scheduler/src/cluster/memory.rs b/ballista/scheduler/src/cluster/memory.rs
index d73206ffa1..51dfa37dce 100644
--- a/ballista/scheduler/src/cluster/memory.rs
+++ b/ballista/scheduler/src/cluster/memory.rs
@@ -22,7 +22,7 @@ use crate::cluster::{
 use crate::state::execution_graph::ExecutionGraphBox;
 use ballista_core::error::{BallistaError, Result};
 use ballista_core::serde::protobuf::{
-    AvailableTaskSlots, ExecutorHeartbeat, ExecutorStatus, FailedJob, QueuedJob,
+    AvailableVcores, ExecutorHeartbeat, ExecutorStatus, FailedJob, QueuedJob,
     executor_status,
 };
 use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata};
@@ -47,13 +47,13 @@ use super::{ClusterStateEvent, ClusterStateEventStream};
 
 /// In-memory implementation of cluster state.
 ///
-/// This stores all cluster state (executor registration, task slots, heartbeats)
+/// This stores all cluster state (executor registration, free vcores, heartbeats)
 /// in memory without persistence. Suitable for single-scheduler deployments
 /// or testing scenarios.
 #[derive(Default)]
 pub struct InMemoryClusterState {
-    /// Current available task slots for each executor.
-    task_slots: Mutex<HashMap<String, AvailableTaskSlots>>,
+    /// Current free vcores for each executor.
+    available_vcores: Mutex<HashMap<String, AvailableVcores>>,
     /// Current executors and their metadata.
     executors: DashMap<String, ExecutorMetadata>,
     /// Last heartbeat received for each executor.
@@ -70,12 +70,12 @@ impl ClusterState for InMemoryClusterState {
         active_jobs: Arc<HashMap<JobId, JobInfoCache>>,
         executors: Option<HashSet<String>>,
     ) -> Result<Vec<BoundTask>> {
-        let mut guard = self.task_slots.lock().await;
+        let mut guard = self.available_vcores.lock().await;
 
-        let available_slots: Vec<&mut AvailableTaskSlots> = guard
+        let budgets: Vec<&mut AvailableVcores> = guard
             .values_mut()
             .filter_map(|data| {
-                (data.slots > 0
+                (data.vcores > 0
                     && executors
                         .as_ref()
                         .map(|executors| executors.contains(&data.executor_id))
@@ -86,13 +86,13 @@ impl ClusterState for InMemoryClusterState {
 
         let bound_tasks = match distribution {
             TaskDistributionPolicy::Bias => {
-                bind_task_bias(available_slots, active_jobs, |_| false).await
+                bind_task_bias(budgets, active_jobs, |_| false).await
             }
             TaskDistributionPolicy::RoundRobin => {
-                bind_task_round_robin(available_slots, active_jobs, |_| false).await
+                bind_task_round_robin(budgets, active_jobs, |_| false).await
             }
             TaskDistributionPolicy::Custom(ref policy) => {
-                policy.bind_tasks(available_slots, active_jobs).await?
+                policy.bind_tasks(budgets, active_jobs).await?
             }
         };
 
@@ -101,16 +101,16 @@ impl ClusterState for InMemoryClusterState {
 
     async fn unbind_tasks(&self, executor_slots: Vec<ExecutorSlot>) -> Result<()> {
         let mut increments = HashMap::new();
-        for (executor_id, num_slots) in executor_slots {
+        for (executor_id, num_vcores) in executor_slots {
             let v = increments.entry(executor_id).or_insert_with(|| 0);
-            *v += num_slots;
+            *v += num_vcores;
         }
 
-        let mut guard = self.task_slots.lock().await;
+        let mut guard = self.available_vcores.lock().await;
 
-        for (executor_id, num_slots) in increments {
+        for (executor_id, num_vcores) in increments {
             if let Some(data) = guard.get_mut(&executor_id) {
-                data.slots += num_slots;
+                data.vcores += num_vcores;
             }
         }
 
@@ -138,13 +138,13 @@ impl ClusterState for InMemoryClusterState {
         })
         .await?;
 
-        let mut guard = self.task_slots.lock().await;
+        let mut guard = self.available_vcores.lock().await;
 
         guard.insert(
             executor_id.clone(),
-            AvailableTaskSlots {
+            AvailableVcores {
                 executor_id: executor_id.clone(),
-                slots: spec.available_task_slots,
+                vcores: spec.available_vcores,
             },
         );
 
@@ -224,7 +224,7 @@ impl ClusterState for InMemoryClusterState {
     async fn remove_executor(&self, executor_id: &str) -> Result<()> {
         log::debug!("removing executor: {}", executor_id);
         {
-            let mut guard = self.task_slots.lock().await;
+            let mut guard = self.available_vcores.lock().await;
 
             guard.remove(executor_id);
         }
@@ -658,7 +658,7 @@ mod test {
             host: "".to_string(),
             port: 50055,
             grpc_port: 50050,
-            specification: ExecutorSpecification::default().with_task_slots(2),
+            specification: ExecutorSpecification::default().with_vcores(2),
             os_info: ExecutorOperatingSystemSpecification::default(),
         };
 
diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs
index 6bd474f9a6..cea3858d1d 100644
--- a/ballista/scheduler/src/cluster/mod.rs
+++ b/ballista/scheduler/src/cluster/mod.rs
@@ -22,17 +22,19 @@ use crate::state::execution_graph::{
     ExecutionGraphBox, TaskDescription, create_task_info,
 };
 use crate::state::task_manager::JobInfoCache;
+use ballista_core::config::BallistaConfig;
 use ballista_core::error::Result;
+use ballista_core::execution_plans::ShuffleReaderExec;
 use ballista_core::serde::protobuf::{
-    AvailableTaskSlots, ExecutorHeartbeat, JobStatus, job_status,
+    AvailableVcores, ExecutorHeartbeat, JobStatus, job_status,
 };
-use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata, PartitionId};
+use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata, TaskKey};
 use ballista_core::utils::{default_config_producer, default_session_builder};
 use ballista_core::{ConfigProducer, JobId, JobStatusSubscriber};
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::{SessionConfig, SessionContext};
 use futures::Stream;
-use log::debug;
+use log::{debug, info};
 use std::collections::{HashMap, HashSet};
 use std::pin::Pin;
 use std::sync::Arc;
@@ -153,7 +155,7 @@ pub type ExecutorSlot = (String, u32);
 
 /// Trait for maintaining a globally consistent view of cluster resources.
 ///
-/// Implementations track executor registration, heartbeats, and available task slots.
+/// Implementations track executor registration, heartbeats, and free vcores.
 #[async_trait::async_trait]
 pub trait ClusterState: Send + Sync + 'static {
     /// Initializes the cluster state backend.
@@ -163,9 +165,9 @@ pub trait ClusterState: Send + Sync + 'static {
         Ok(())
     }
 
-    /// Binds ready-to-run tasks from active jobs to available executor slots.
+    /// Binds ready-to-run tasks from active jobs to executor vcores.
     ///
-    /// If `executors` is provided, only bind slots from the specified executor IDs.
+    /// If `executors` is provided, only bind on the specified executor IDs.
     async fn bind_schedulable_tasks(
         &self,
         distribution: TaskDistributionPolicy,
@@ -173,9 +175,9 @@ pub trait ClusterState: Send + Sync + 'static {
         executors: Option<HashSet<String>>,
     ) -> Result<Vec<BoundTask>>;
 
-    /// Unbinds executor slots when tasks finish or fail.
+    /// Releases reserved vcores when tasks finish or fail.
     ///
-    /// This operation is atomic: either all slots are released or none are.
+    /// This operation is atomic: either all vcores are released or none are.
     async fn unbind_tasks(&self, executor_slots: Vec<ExecutorSlot>) -> Result<()>;
 
     /// Registers a new executor in the cluster.
@@ -317,6 +319,9 @@ pub trait JobState: Send + Sync {
 
     /// Persists the current state of an owned job.
     ///
+    /// Implementations are responsible for applying any backend-specific retry policy.
+    /// Callers must not assume that this operation is safe to retry.
+    ///
     /// Returns an error if the job is not owned by the caller.
     async fn save_job(&self, job_id: &JobId, graph: &ExecutionGraphBox) -> Result<()>;
 
@@ -351,24 +356,178 @@ pub trait JobState: Send + Sync {
     fn produce_config(&self) -> SessionConfig;
 }
 
+/// Whether this stage's plan contains a collapse — any operator whose
+/// `output_partitioning().partition_count() == 1` (`CoalescePartitionsExec`,
+/// `SortPreservingMergeExec`, `AggregateExec(Final, gby=[])`,
+/// `SortExec(preserve_partitioning=false)`, …). Downstream of a collapse
+/// expects the *combined* input; splitting across tasks would give each
+/// task's local collapse only a slice, producing partial results.
+///
+/// Walks past *any* single-child operator: today's writers that bake
+/// partitioning into the shuffle write itself (`SortShuffleWriterExec::Hash`),
+/// and future partition operators that separate partitioning from writing
+/// (e.g. `UnorderedRangeRepartitionExec`). Stops at leaves, multi-child
+/// operators (fan-in / joins), and stage boundaries.
+///
+/// The stage-boundary stop is currently a `ShuffleReaderExec` downcast — the
+/// only kind of stage-boundary leaf that appears in a resolved stage plan.
+/// The *general* rule is "stop at any stage boundary"; if new stage-boundary
+/// operators appear, add them here (or, better, get `ExecutionPlan` upstream
+/// to expose an `is_stage_boundary()` property so we don't keep
+/// enumerating).
+fn stage_has_input_collapse(plan_root: &Arc<dyn ExecutionPlan>) -> bool {
+    fn walk(node: &Arc<dyn ExecutionPlan>) -> bool {
+        if node.downcast_ref::<ShuffleReaderExec>().is_some() {
+            return false;
+        }
+        if node.properties().output_partitioning().partition_count() == 1 {
+            return true;
+        }
+        match node.children().as_slice() {
+            [child] => walk(child),
+            _ => false,
+        }
+    }
+    let result = match plan_root.children().as_slice() {
+        [child] => walk(child),
+        _ => false,
+    };
+    info!(
+        "DIAG stage_has_input_collapse: root={} root_partitions={} → {result}",
+        plan_root.name(),
+        plan_root
+            .properties()
+            .output_partitioning()
+            .partition_count(),
+    );
+    result
+}
+
+/// Bind a task to an executor: draw a partition slice, append the resulting
+/// `TaskInfo` to the stage, and produce a `TaskDescription` for dispatch.
+///
+/// Vcore accounting under DataFusion's volcano/pull model — the plan is
+/// driven by one tokio task per root output partition, so a task consumes
+/// vcores equal to the number of threads it will actually keep busy:
+///
+/// - **Non-collapse stage**: the root's output partitioning matches the
+///   number of input partitions bundled into this task, so consumption =
+///   `slice.len()`. Leftover budget stays available for another bind on
+///   the same executor in the same scheduling round.
+///
+/// - **Collapse stage** (see [`stage_has_input_collapse`]): the plan's
+///   root has a single output partition, so only 1 thread is ever active
+///   driving the whole pipeline — reserve 1 vcore regardless of how many
+///   input partitions are packed into the task, and let other stages'
+///   tasks run in parallel on the remaining vcores. Correctness still
+///   requires packing the entire pending queue into one bind (a split
+///   collapse would produce partial results downstream can't merge).
+fn bind_one(
+    running_stage: &mut crate::state::execution_stage::RunningStage,
+    session_id: &str,
+    job_id: &JobId,
+    budget: &mut AvailableVcores,
+) -> Option<BoundTask> {
+    let is_collapse = stage_has_input_collapse(&running_stage.plan);
+    // Cap non-collapse slices at the configured `max_partitions_per_task`.
+    // Collapse stages must still pack their full pending queue into a single
+    // task for correctness — a split collapse would produce partial results
+    // downstream can't merge.
+    let cap = running_stage
+        .session_config
+        .options()
+        .extensions
+        .get::<BallistaConfig>()
+        .map(|bc| bc.max_partitions_per_task())
+        .filter(|&n| n > 0)
+        .unwrap_or(usize::MAX);
+    let max_partitions = if is_collapse {
+        usize::MAX
+    } else {
+        (budget.vcores as usize).min(cap)
+    };
+    let input_partition_ids = running_stage.pending.next_slice(max_partitions);
+    if input_partition_ids.is_empty() {
+        return None;
+    }
+    // Non-collapse: DataFusion's volcano/pull model drives one tokio task per
+    // root output partition, so N input partitions bundled into a task need
+    // N vcores of concurrent execution. Collapse: the plan's root has a
+    // single output partition, so only 1 thread is ever active for the
+    // whole pipeline no matter how many inputs the task packs — reserve
+    // one vcore and leave the rest for other stages' tasks on this executor.
+    let vcores_consumed = if is_collapse {
+        1
+    } else {
+        input_partition_ids.len() as u32
+    };
+    info!(
+        "DIAG bind_one: job={} stage={} exec={} vcores={} max={} slice_len={} consumed={} partitions={:?} collapse={}",
+        job_id,
+        running_stage.stage_id,
+        budget.executor_id,
+        budget.vcores,
+        if max_partitions == usize::MAX {
+            -1i64
+        } else {
+            max_partitions as i64
+        },
+        input_partition_ids.len(),
+        vcores_consumed,
+        input_partition_ids,
+        is_collapse,
+    );
+    let executor_id = budget.executor_id.clone();
+    // task_id is the append-order slot in `task_infos` — since we're
+    // about to push, that's `task_infos.len()`. `(job_id, stage_id,
+    // task_id)` is globally unique.
+    let task_id = running_stage.task_infos.len();
+    let task_attempt = input_partition_ids
+        .iter()
+        .map(|pid| running_stage.task_failure_numbers[*pid])
+        .max()
+        .unwrap_or(0);
+    let mut task_info = create_task_info(executor_id.clone(), task_id);
+    task_info.global_input_partition_ids = input_partition_ids.clone();
+    task_info.vcores_consumed = vcores_consumed;
+    running_stage.task_infos.push(task_info);
+    let key = TaskKey {
+        job_id: job_id.clone(),
+        stage_id: running_stage.stage_id,
+        task_id,
+    };
+    let task_desc = TaskDescription {
+        session_id: session_id.to_string(),
+        key,
+        stage_attempt_num: running_stage.stage_attempt_num,
+        task_attempt,
+        global_input_partition_ids: input_partition_ids,
+        vcores_consumed,
+        plan: running_stage.plan.clone(),
+        session_config: running_stage.session_config.clone(),
+    };
+    budget.vcores -= vcores_consumed;
+    Some((executor_id, task_desc))
+}
+
 pub(crate) async fn bind_task_bias(
-    mut slots: Vec<&mut AvailableTaskSlots>,
+    mut budgets: Vec<&mut AvailableVcores>,
     running_jobs: Arc<HashMap<JobId, JobInfoCache>>,
     if_skip: fn(Arc<dyn ExecutionPlan>) -> bool,
 ) -> Vec<BoundTask> {
     let mut schedulable_tasks: Vec<BoundTask> = vec![];
 
-    let total_slots = slots.iter().fold(0, |acc, s| acc + s.slots);
-    if total_slots == 0 {
-        debug!("Not enough available executor slots for task running!!!");
+    if budgets.iter().all(|b| b.vcores == 0) {
+        debug!("No executor vcores available for task binding");
         return schedulable_tasks;
     }
 
-    // Sort the slots by descending order
-    slots.sort_by(|a, b| Ord::cmp(&b.slots, &a.slots));
+    // Bias: give each stage the biggest exec available. Sort descending
+    // so bind_one keeps packing tasks onto the largest executor until its
+    // vcore budget is drained.
+    budgets.sort_by(|a, b| Ord::cmp(&b.vcores, &a.vcores));
 
-    let mut idx_slot = 0usize;
-    let mut slot = &mut slots[idx_slot];
+    let mut idx = 0usize;
     for (job_id, job_info) in running_jobs.iter() {
         if !matches!(job_info.status, Some(job_status::Status::Running(_))) {
             debug!("Job {job_id} is not in running status and will be skipped");
@@ -377,9 +536,7 @@ pub(crate) async fn bind_task_bias(
         let mut graph = job_info.execution_graph.write().await;
         let session_id = graph.session_id().to_string();
         let mut black_list = vec![];
-        while let Some((running_stage, task_id_gen)) =
-            graph.fetch_running_stage(&black_list)
-        {
+        while let Some(running_stage) = graph.fetch_running_stage(&black_list) {
             if if_skip(running_stage.plan.clone()) {
                 debug!(
                     "Will skip stage {}/{} for bias task binding",
@@ -388,46 +545,17 @@ pub(crate) async fn bind_task_bias(
                 black_list.push(running_stage.stage_id);
                 continue;
             }
-            // We are sure that it will at least bind one task by going through the following logic.
-            // It will not go into a dead loop.
-            let runnable_tasks = running_stage
-                .task_infos
-                .iter_mut()
-                .enumerate()
-                .filter(|(_partition, info)| info.is_none())
-                .take(total_slots as usize)
-                .collect::<Vec<_>>();
-            for (partition_id, task_info) in runnable_tasks {
-                // Assign [`slot`] with a slot available slot number larger than 0
-                while slot.slots == 0 {
-                    idx_slot += 1;
-                    if idx_slot >= slots.len() {
-                        return schedulable_tasks;
-                    }
-                    slot = &mut slots[idx_slot];
+            while idx < budgets.len() {
+                while idx < budgets.len() && budgets[idx].vcores == 0 {
+                    idx += 1;
+                }
+                if idx >= budgets.len() {
+                    return schedulable_tasks;
+                }
+                match bind_one(running_stage, &session_id, job_id, &mut *budgets[idx]) {
+                    Some(bound) => schedulable_tasks.push(bound),
+                    None => break, // stage's pending is drained
                 }
-                let executor_id = slot.executor_id.clone();
-                let task_id = *task_id_gen;
-                *task_id_gen += 1;
-                *task_info = Some(create_task_info(executor_id.clone(), task_id));
-
-                let partition = PartitionId {
-                    job_id: job_id.clone(),
-                    stage_id: running_stage.stage_id,
-                    partition_id,
-                };
-                let task_desc = TaskDescription {
-                    session_id: session_id.clone(),
-                    partition,
-                    stage_attempt_num: running_stage.stage_attempt_num,
-                    task_id,
-                    task_attempt: running_stage.task_failure_numbers[partition_id],
-                    plan: running_stage.plan.clone(),
-                    session_config: running_stage.session_config.clone(),
-                };
-                schedulable_tasks.push((executor_id, task_desc));
-
-                slot.slots -= 1;
             }
         }
     }
@@ -436,23 +564,23 @@ pub(crate) async fn bind_task_bias(
 }
 
 pub(crate) async fn bind_task_round_robin(
-    mut slots: Vec<&mut AvailableTaskSlots>,
+    mut budgets: Vec<&mut AvailableVcores>,
     running_jobs: Arc<HashMap<JobId, JobInfoCache>>,
     if_skip: fn(Arc<dyn ExecutionPlan>) -> bool,
 ) -> Vec<BoundTask> {
     let mut schedulable_tasks: Vec<BoundTask> = vec![];
 
-    let mut total_slots = slots.iter().fold(0, |acc, s| acc + s.slots);
-    if total_slots == 0 {
-        debug!("Not enough available executor slots for task running!!!");
+    if budgets.iter().all(|b| b.vcores == 0) {
+        debug!("No executor vcores available for task binding");
         return schedulable_tasks;
     }
-    debug!("Total slot number is {total_slots}");
 
-    // Sort the slots by descending order
-    slots.sort_by(|a, b| Ord::cmp(&b.slots, &a.slots));
+    // Round-robin across execs so multiple running stages each get one
+    // exec per rotation. Order by vcores desc so the largest exec goes
+    // first in the rotation.
+    budgets.sort_by(|a, b| Ord::cmp(&b.vcores, &a.vcores));
 
-    let mut idx_slot = 0usize;
+    let mut idx = 0usize;
     for (job_id, job_info) in running_jobs.iter() {
         if !matches!(job_info.status, Some(job_status::Status::Running(_))) {
             debug!("Job {job_id} is not in running status and will be skipped");
@@ -461,9 +589,7 @@ pub(crate) async fn bind_task_round_robin(
         let mut graph = job_info.execution_graph.write().await;
         let session_id = graph.session_id().to_string();
         let mut black_list = vec![];
-        while let Some((running_stage, task_id_gen)) =
-            graph.fetch_running_stage(&black_list)
-        {
+        while let Some(running_stage) = graph.fetch_running_stage(&black_list) {
             if if_skip(running_stage.plan.clone()) {
                 debug!(
                     "Will skip stage {}/{} for round robin task binding",
@@ -472,52 +598,21 @@ pub(crate) async fn bind_task_round_robin(
                 black_list.push(running_stage.stage_id);
                 continue;
             }
-            // We are sure that it will at least bind one task by going through the following logic.
-            // It will not go into a dead loop.
-            let runnable_tasks = running_stage
-                .task_infos
-                .iter_mut()
-                .enumerate()
-                .filter(|(_partition, info)| info.is_none())
-                .take(total_slots as usize)
-                .collect::<Vec<_>>();
-            for (partition_id, task_info) in runnable_tasks {
-                // Move to the index which has available slots
-                if idx_slot >= slots.len() {
-                    idx_slot = 0;
-                }
-                if slots[idx_slot].slots == 0 {
-                    idx_slot = 0;
+            loop {
+                let mut scanned = 0usize;
+                while budgets[idx].vcores == 0 {
+                    idx = (idx + 1) % budgets.len();
+                    scanned += 1;
+                    if scanned >= budgets.len() {
+                        return schedulable_tasks;
+                    }
                 }
-                // Since the slots is a vector with descending order, and the total available slots is larger than 0,
-                // we are sure the available slot number at idx_slot is larger than 1
-                let slot = &mut slots[idx_slot];
-                let executor_id = slot.executor_id.clone();
-                let task_id = *task_id_gen;
-                *task_id_gen += 1;
-                *task_info = Some(create_task_info(executor_id.clone(), task_id));
-
-                let partition = PartitionId {
-                    job_id: job_id.to_owned(),
-                    stage_id: running_stage.stage_id,
-                    partition_id,
-                };
-                let task_desc = TaskDescription {
-                    session_id: session_id.clone(),
-                    partition,
-                    stage_attempt_num: running_stage.stage_attempt_num,
-                    task_id,
-                    task_attempt: running_stage.task_failure_numbers[partition_id],
-                    plan: running_stage.plan.clone(),
-                    session_config: running_stage.session_config.clone(),
-                };
-                schedulable_tasks.push((executor_id, task_desc));
-
-                idx_slot += 1;
-                slot.slots -= 1;
-                total_slots -= 1;
-                if total_slots == 0 {
-                    return schedulable_tasks;
+                match bind_one(running_stage, &session_id, job_id, &mut *budgets[idx]) {
+                    Some(bound) => {
+                        schedulable_tasks.push(bound);
+                        idx = (idx + 1) % budgets.len();
+                    }
+                    None => break, // stage's pending is drained
                 }
             }
         }
@@ -541,7 +636,7 @@ pub trait DistributionPolicy: std::fmt::Debug + Send + Sync {
     ///
     /// # Parameters
     ///
-    /// * `slots` - vector of available executor slots, there may not be available slots
+    /// * `budgets` - per-executor free-vcore budgets (may be empty)
     /// * `running_jobs` - (JobId -> JobInfoCache) cache must contain only running jobs
     ///
     /// # Returns
@@ -550,7 +645,7 @@ pub trait DistributionPolicy: std::fmt::Debug + Send + Sync {
     ///
     async fn bind_tasks(
         &self,
-        mut slots: Vec<&mut AvailableTaskSlots>,
+        mut budgets: Vec<&mut AvailableVcores>,
         running_jobs: Arc<HashMap<JobId, JobInfoCache>>,
     ) -> datafusion::error::Result<Vec<BoundTask>>;
 
@@ -565,7 +660,7 @@ mod test {
 
     use ballista_core::JobId;
     use ballista_core::error::Result;
-    use ballista_core::serde::protobuf::AvailableTaskSlots;
+    use ballista_core::serde::protobuf::AvailableVcores;
     use ballista_core::serde::scheduler::{
         ExecutorMetadata, ExecutorOperatingSystemSpecification, ExecutorSpecification,
     };
@@ -575,48 +670,55 @@ mod test {
     use crate::state::task_manager::JobInfoCache;
     use crate::test_utils::{
         mock_completed_task, revive_graph_and_complete_next_stage,
-        test_aggregation_plan_with_job_id,
+        test_aggregation_plan_with_config,
     };
+    use ballista_core::config::BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK;
+    use ballista_core::extension::SessionConfigExt;
+    use datafusion::prelude::SessionConfig;
 
     #[tokio::test]
     async fn test_bind_task_bias() -> Result<()> {
         let num_partition = 8usize;
         let active_jobs = mock_active_jobs(num_partition).await?;
-        let mut available_slots = mock_available_slots();
-        let available_slots_ref: Vec<&mut AvailableTaskSlots> =
-            available_slots.iter_mut().collect();
+        let mut budgets = mock_budgets();
+        let budgets_ref: Vec<&mut AvailableVcores> = budgets.iter_mut().collect();
         let bound_tasks =
-            bind_task_bias(available_slots_ref, Arc::new(active_jobs), |_| false).await;
-        assert_eq!(9, bound_tasks.len());
+            bind_task_bias(budgets_ref, Arc::new(active_jobs), |_| false).await;
+        // 9 total pending partitions (job_a: 2, job_b: 7) — verify all were
+        // covered. Task count is emergent under multi-partition binding.
+        assert_eq!(9, total_partitions_covered(&bound_tasks));
 
         let result = get_result(bound_tasks);
 
+        // Multi-partition-task binding: each bind consumes slice.len() vcores,
+        // so an executor with leftover vcores keeps taking work under the bias
+        // policy (largest exec stays hot). Distribution depends on HashMap
+        // iteration order across jobs.
         let mut expected = Vec::new();
         {
+            // job_a iterated first: exec_3(7) takes job_a's 2 partitions
+            // (exec_3 now has 5 vcores left), then bias keeps exec_3 hot so
+            // it takes 5 of job_b's 7, and exec_2(5) takes the remaining 2.
             let mut expected0: HashMap<JobId, HashMap<String, usize>> = HashMap::new();
-
             let mut entry_a = HashMap::new();
             entry_a.insert("executor_3".to_string(), 2);
             let mut entry_b = HashMap::new();
             entry_b.insert("executor_3".to_string(), 5);
             entry_b.insert("executor_2".to_string(), 2);
-
             expected0.insert("job_a".into(), entry_a);
             expected0.insert("job_b".into(), entry_b);
-
             expected.push(expected0);
         }
         {
+            // job_b iterated first: exec_3(7) takes job_b's full 7 partitions,
+            // then exec_2(5) takes job_a's 2.
             let mut expected0: HashMap<JobId, HashMap<String, usize>> = HashMap::new();
-
             let mut entry_b = HashMap::new();
             entry_b.insert("executor_3".to_string(), 7);
             let mut entry_a = HashMap::new();
             entry_a.insert("executor_2".to_string(), 2);
-
             expected0.insert("job_a".into(), entry_a);
             expected0.insert("job_b".into(), entry_b);
-
             expected.push(expected0);
         }
 
@@ -632,47 +734,44 @@ mod test {
     async fn test_bind_task_round_robin() -> Result<()> {
         let num_partition = 8usize;
         let active_jobs = mock_active_jobs(num_partition).await?;
-        let mut available_slots = mock_available_slots();
-        let available_slots_ref: Vec<&mut AvailableTaskSlots> =
-            available_slots.iter_mut().collect();
+        let mut budgets = mock_budgets();
+        let budgets_ref: Vec<&mut AvailableVcores> = budgets.iter_mut().collect();
         let bound_tasks =
-            bind_task_round_robin(available_slots_ref, Arc::new(active_jobs), |_| false)
-                .await;
-        assert_eq!(9, bound_tasks.len());
+            bind_task_round_robin(budgets_ref, Arc::new(active_jobs), |_| false).await;
+        // 9 total pending partitions (job_a: 2, job_b: 7) — verify all were
+        // covered. Task count is emergent under multi-partition binding.
+        assert_eq!(9, total_partitions_covered(&bound_tasks));
 
         let result = get_result(bound_tasks);
 
+        // Multi-partition-task binding: same shape as bias variant under this
+        // mock — each executor consumes its full vcore budget in one task per
+        // (job, stage) and round-robin's idx rotation gives the same
+        // ordering. Two variants depending on job HashMap iteration order.
         let mut expected: Vec<HashMap<JobId, HashMap<String, usize>>> = Vec::new();
         {
+            // job_a iterated first: exec_3(7) takes job_a's 2 partitions,
+            // then exec_2(5) + exec_1(3) split job_b's 7 as 5/2.
             let mut expected0: HashMap<JobId, HashMap<String, usize>> = HashMap::new();
-
             let mut entry_a = HashMap::new();
-            entry_a.insert("executor_3".to_string(), 1);
-            entry_a.insert("executor_2".to_string(), 1);
+            entry_a.insert("executor_3".to_string(), 2);
             let mut entry_b = HashMap::new();
-            entry_b.insert("executor_1".to_string(), 3);
-            entry_b.insert("executor_3".to_string(), 2);
-            entry_b.insert("executor_2".to_string(), 2);
-
+            entry_b.insert("executor_2".to_string(), 5);
+            entry_b.insert("executor_1".to_string(), 2);
             expected0.insert("job_a".into(), entry_a);
             expected0.insert("job_b".into(), entry_b);
-
             expected.push(expected0);
         }
         {
+            // job_b iterated first: exec_3(7) takes job_b's full 7,
+            // then exec_2(5) takes job_a's 2.
             let mut expected0: HashMap<JobId, HashMap<String, usize>> = HashMap::new();
-
             let mut entry_b = HashMap::new();
-            entry_b.insert("executor_3".to_string(), 3);
-            entry_b.insert("executor_2".to_string(), 2);
-            entry_b.insert("executor_1".to_string(), 2);
+            entry_b.insert("executor_3".to_string(), 7);
             let mut entry_a = HashMap::new();
-            entry_a.insert("executor_2".to_string(), 1);
-            entry_a.insert("executor_1".to_string(), 1);
-
+            entry_a.insert("executor_2".to_string(), 2);
             expected0.insert("job_a".into(), entry_a);
             expected0.insert("job_b".into(), entry_b);
-
             expected.push(expected0);
         }
 
@@ -684,20 +783,35 @@ mod test {
         Ok(())
     }
 
+    /// Sum partitions covered per (job, executor). Under multi-partition
+    /// tasks one task covers a slice of partitions rather than exactly one,
+    /// so counting bound tasks would understate the distribution. Summing
+    /// `global_input_partition_ids.len()` preserves the "N partitions distributed as
+    /// X/Y/Z across executors" invariant the assertions actually care about.
     fn get_result(bound_tasks: Vec<BoundTask>) -> HashMap<JobId, HashMap<String, usize>> {
         let mut result = HashMap::new();
 
         for bound_task in bound_tasks {
             let entry = result
-                .entry(bound_task.1.partition.job_id)
+                .entry(bound_task.1.key.job_id)
                 .or_insert_with(HashMap::new);
             let n = entry.entry(bound_task.0).or_insert_with(|| 0);
-            *n += 1;
+            *n += bound_task.1.global_input_partition_ids.len();
         }
 
         result
     }
 
+    /// Total partitions covered across every bound task — the multi-partition
+    /// analogue of "how many tasks did we bind" for tests that used to assert
+    /// on `bound_tasks.len()`.
+    fn total_partitions_covered(bound_tasks: &[BoundTask]) -> usize {
+        bound_tasks
+            .iter()
+            .map(|(_, task)| task.global_input_partition_ids.len())
+            .sum()
+    }
+
     async fn mock_active_jobs(
         num_partition: usize,
     ) -> Result<HashMap<JobId, JobInfoCache>> {
@@ -723,14 +837,28 @@ mod test {
         num_target_partitions: usize,
         num_pending_task: usize,
     ) -> Result<StaticExecutionGraph> {
-        let mut graph =
-            test_aggregation_plan_with_job_id(num_target_partitions, job_id).await;
+        // These tests validate the *multi-partition* binding path: expected
+        // task distributions include slice sizes up to 7. That requires an
+        // unbounded `max_partitions_per_task`. Since `bc7a4eda` flipped the
+        // default to 1 (single-partition-per-task, matching master's
+        // pre-branch behaviour), we override it back here so the mock
+        // exercises the branch feature these tests are *for*.
+        let session_config = Arc::new(
+            SessionConfig::new_with_ballista()
+                .set_str(BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, "0"),
+        );
+        let mut graph = test_aggregation_plan_with_config(
+            num_target_partitions,
+            job_id,
+            session_config,
+        )
+        .await;
         let executor = ExecutorMetadata {
             id: "executor_0".to_string(),
             host: "localhost".to_string(),
             port: 50051,
             grpc_port: 50052,
-            specification: ExecutorSpecification::default().with_task_slots(32),
+            specification: ExecutorSpecification::default().with_vcores(32),
             os_info: ExecutorOperatingSystemSpecification::default(),
         };
 
@@ -747,19 +875,19 @@ mod test {
         Ok(graph)
     }
 
-    fn mock_available_slots() -> Vec<AvailableTaskSlots> {
+    fn mock_budgets() -> Vec<AvailableVcores> {
         vec![
-            AvailableTaskSlots {
+            AvailableVcores {
                 executor_id: "executor_1".to_string(),
-                slots: 3,
+                vcores: 3,
             },
-            AvailableTaskSlots {
+            AvailableVcores {
                 executor_id: "executor_2".to_string(),
-                slots: 5,
+                vcores: 5,
             },
-            AvailableTaskSlots {
+            AvailableVcores {
                 executor_id: "executor_3".to_string(),
-                slots: 7,
+                vcores: 7,
             },
         ]
     }
diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs
index 3c6cdb0074..940888714b 100644
--- a/ballista/scheduler/src/config.rs
+++ b/ballista/scheduler/src/config.rs
@@ -173,6 +173,13 @@ pub struct Config {
         help = "Interval, in seconds, to check expired or dead executors."
     )]
     pub expire_dead_executor_interval_seconds: u64,
+    /// Minimum number of registered executors before /readyz returns 200
+    #[arg(
+        long,
+        default_value_t = 1,
+        help = "Minimum number of registered executors before the scheduler's /readyz probe returns 200. Set to 0 to always report ready."
+    )]
+    pub min_ready_executors: usize,
     /// Number of failures attempts before task is considered failed
     #[arg(
         long,
@@ -265,6 +272,8 @@ pub struct SchedulerConfig {
     pub override_create_grpc_client_endpoint: Option<EndpointOverrideFn>,
     /// Whether to use TLS when connecting to executors (for flight proxy)
     pub use_tls: bool,
+    /// Minimum number of registered executors before /readyz returns 200
+    pub min_ready_executors: usize,
     /// Number of failures attempts before task is considered failed
     pub task_max_failures: usize,
     /// Number of failures attempts before stage is considered failed
@@ -306,6 +315,7 @@ impl Default for SchedulerConfig {
             override_physical_codec: None,
             override_create_grpc_client_endpoint: None,
             use_tls: false,
+            min_ready_executors: 1,
             task_max_failures: 4,
             stage_max_failures: 4,
             #[cfg(feature = "rest-api")]
@@ -456,7 +466,7 @@ impl SchedulerConfig {
 #[derive(Clone, Copy, Debug, serde::Deserialize, Default)]
 #[cfg_attr(feature = "build-binary", derive(clap::ValueEnum))]
 pub enum TaskDistribution {
-    /// Eagerly assign tasks to executor slots. This will assign as many task slots per executor
+    /// Eagerly assign tasks to executor slots. This will assign as many vcores per executor
     /// as are currently available
     #[default]
     Bias,
@@ -486,7 +496,7 @@ impl std::str::FromStr for TaskDistribution {
 /// Policy for distributing tasks to available executor slots.
 #[derive(Debug, Clone, Default)]
 pub enum TaskDistributionPolicy {
-    /// Eagerly assign tasks to executor slots. This will assign as many task slots per executor
+    /// Eagerly assign tasks to executor slots. This will assign as many vcores per executor
     /// as are currently available
     #[default]
     Bias,
@@ -538,6 +548,7 @@ impl TryFrom<Config> for SchedulerConfig {
             override_session_builder: None,
             override_create_grpc_client_endpoint: None,
             use_tls: false,
+            min_ready_executors: opt.min_ready_executors,
             task_max_failures: opt.task_max_failures,
             stage_max_failures: opt.stage_max_failures,
             #[cfg(feature = "rest-api")]
diff --git a/ballista/scheduler/src/metrics/mod.rs b/ballista/scheduler/src/metrics/mod.rs
index 18e06c21cb..8f7520d7db 100644
--- a/ballista/scheduler/src/metrics/mod.rs
+++ b/ballista/scheduler/src/metrics/mod.rs
@@ -92,3 +92,25 @@ pub fn default_metrics_collector() -> Result<Arc<dyn SchedulerMetricsCollector>>
 pub fn default_metrics_collector() -> Result<Arc<dyn SchedulerMetricsCollector>> {
     Ok(Arc::new(NoopMetricsCollector::default()))
 }
+
+/// Increments the counter of executor RPCs (register/heartbeat/poll_work)
+/// rejected because their `ballista_protocol_version` did not match the
+/// scheduler's `BALLISTA_PROTOCOL_VERSION`. No-op when the `prometheus`
+/// feature is disabled.
+#[cfg(feature = "prometheus")]
+pub fn record_protocol_mismatch() {
+    use ::prometheus::{IntCounter, register_int_counter};
+    use once_cell::sync::Lazy;
+    static COUNTER: Lazy<IntCounter> = Lazy::new(|| {
+        register_int_counter!(
+            "ballista_scheduler_rejected_by_protocol_version_total",
+            "Executor RPCs rejected because of BALLISTA_PROTOCOL_VERSION mismatch",
+        )
+        .expect("register ballista_scheduler_rejected_by_protocol_version_total")
+    });
+    COUNTER.inc();
+}
+
+/// See prometheus-featured variant.
+#[cfg(not(feature = "prometheus"))]
+pub fn record_protocol_mismatch() {}
diff --git a/ballista/scheduler/src/physical_optimizer/join_selection.rs b/ballista/scheduler/src/physical_optimizer/join_selection.rs
index b50b0f3c21..20d4649283 100644
--- a/ballista/scheduler/src/physical_optimizer/join_selection.rs
+++ b/ballista/scheduler/src/physical_optimizer/join_selection.rs
@@ -111,6 +111,31 @@ fn supports_collect_by_thresholds(
     }
 }
 
+/// Whether a `CollectLeft` (broadcast) hash join is correct for `join_type` in
+/// Ballista's distributed execution.
+///
+/// `CollectLeft` replicates the build (left) side to every probe task, and each
+/// task processes only its own slice of the probe (right) side. Output rows that
+/// are determined by a single probe-side row are therefore produced exactly
+/// once. Join types that emit rows on behalf of the build side — unmatched outer
+/// rows, or semi/anti/mark rows that depend on a build row's global match status
+/// — would instead be emitted independently by every task, producing duplicate
+/// or spurious rows. Only join types whose output is driven entirely by the
+/// probe side are safe to broadcast.
+///
+/// `join_type` must be the join type *after* any build-side swap, since the
+/// build side is always the left input of the resulting `HashJoinExec`.
+pub(crate) fn collect_left_broadcast_safe(join_type: JoinType) -> bool {
+    matches!(
+        join_type,
+        JoinType::Inner
+            | JoinType::Right
+            | JoinType::RightSemi
+            | JoinType::RightAnti
+            | JoinType::RightMark
+    )
+}
+
 /// Predicate that checks whether the given join type supports input swapping.
 #[deprecated(since = "45.0.0", note = "use JoinType::supports_swap instead")]
 #[allow(dead_code)]
@@ -323,66 +348,65 @@ fn statistical_join_selection_subrule(
     collect_threshold_byte_size: usize,
     collect_threshold_num_rows: usize,
 ) -> Result<Transformed<Arc<dyn ExecutionPlan>>> {
-    let transformed =
-        if let Some(hash_join) = plan.as_any().downcast_ref::<HashJoinExec>() {
-            match hash_join.partition_mode() {
-                PartitionMode::Auto => try_collect_left(
-                    hash_join,
-                    false,
-                    collect_threshold_byte_size,
-                    collect_threshold_num_rows,
-                )?
+    let transformed = if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() {
+        match hash_join.partition_mode() {
+            PartitionMode::Auto => try_collect_left(
+                hash_join,
+                false,
+                collect_threshold_byte_size,
+                collect_threshold_num_rows,
+            )?
+            .map_or_else(
+                || partitioned_hash_join(hash_join).map(Some),
+                |v| Ok(Some(v)),
+            )?,
+            PartitionMode::CollectLeft => try_collect_left(hash_join, true, 0, 0)?
                 .map_or_else(
                     || partitioned_hash_join(hash_join).map(Some),
                     |v| Ok(Some(v)),
                 )?,
-                PartitionMode::CollectLeft => try_collect_left(hash_join, true, 0, 0)?
-                    .map_or_else(
-                        || partitioned_hash_join(hash_join).map(Some),
-                        |v| Ok(Some(v)),
-                    )?,
-                PartitionMode::Partitioned => {
-                    let left = hash_join.left();
-                    let right = hash_join.right();
-                    if hash_join.join_type().supports_swap()
-                        && should_swap_join_order(&**left, &**right)?
-                    {
-                        hash_join
-                            .swap_inputs(PartitionMode::Partitioned)
-                            .map(Some)?
-                    } else {
-                        None
-                    }
+            PartitionMode::Partitioned => {
+                let left = hash_join.left();
+                let right = hash_join.right();
+                if hash_join.join_type().supports_swap()
+                    && should_swap_join_order(&**left, &**right)?
+                {
+                    hash_join
+                        .swap_inputs(PartitionMode::Partitioned)
+                        .map(Some)?
+                } else {
+                    None
                 }
             }
-        } else if let Some(cross_join) = plan.as_any().downcast_ref::<CrossJoinExec>() {
-            let left = cross_join.left();
-            let right = cross_join.right();
-            if right.properties().output_partitioning().partition_count() > 1 {
-                None
-            } else if should_swap_join_order(&**left, &**right)? {
-                cross_join.swap_inputs().map(Some)?
-            } else {
-                None
-            }
-        } else if let Some(nl_join) = plan.as_any().downcast_ref::<NestedLoopJoinExec>() {
-            let left = nl_join.left();
-            let right = nl_join.right();
-            // next few lines are different from original datafusion rule
-            // partition count of right side has to be equal one to be
-            // able to swap inputs
-            if right.properties().output_partitioning().partition_count() > 1 {
-                None
-            } else if nl_join.join_type().supports_swap()
-                && should_swap_join_order(&**left, &**right)?
-            {
-                nl_join.swap_inputs().map(Some)?
-            } else {
-                None
-            }
+        }
+    } else if let Some(cross_join) = plan.downcast_ref::<CrossJoinExec>() {
+        let left = cross_join.left();
+        let right = cross_join.right();
+        if right.properties().output_partitioning().partition_count() > 1 {
+            None
+        } else if should_swap_join_order(&**left, &**right)? {
+            cross_join.swap_inputs().map(Some)?
         } else {
             None
-        };
+        }
+    } else if let Some(nl_join) = plan.downcast_ref::<NestedLoopJoinExec>() {
+        let left = nl_join.left();
+        let right = nl_join.right();
+        // next few lines are different from original datafusion rule
+        // partition count of right side has to be equal one to be
+        // able to swap inputs
+        if right.properties().output_partitioning().partition_count() > 1 {
+            None
+        } else if nl_join.join_type().supports_swap()
+            && should_swap_join_order(&**left, &**right)?
+        {
+            nl_join.swap_inputs().map(Some)?
+        } else {
+            None
+        }
+    } else {
+        None
+    };
 
     Ok(if let Some(transformed) = transformed {
         Transformed::yes(transformed)
@@ -416,7 +440,7 @@ fn hash_join_convert_symmetric_subrule(
     config_options: &ConfigOptions,
 ) -> Result<Arc<dyn ExecutionPlan>> {
     // Check if the current plan node is a HashJoinExec.
-    if let Some(hash_join) = input.as_any().downcast_ref::<HashJoinExec>() {
+    if let Some(hash_join) = input.downcast_ref::<HashJoinExec>() {
         let left_unbounded = hash_join.left.boundedness().is_unbounded();
         let left_incremental = matches!(
             hash_join.left.pipeline_behavior(),
@@ -556,7 +580,7 @@ pub fn hash_join_swap_subrule(
     mut input: Arc<dyn ExecutionPlan>,
     _config_options: &ConfigOptions,
 ) -> Result<Arc<dyn ExecutionPlan>> {
-    if let Some(hash_join) = input.as_any().downcast_ref::<HashJoinExec>()
+    if let Some(hash_join) = input.downcast_ref::<HashJoinExec>()
         && hash_join.left.boundedness().is_unbounded()
         && !hash_join.right.boundedness().is_unbounded()
         && matches!(
@@ -618,6 +642,7 @@ fn apply_subrules(
 mod test {
     use std::sync::Arc;
 
+    use super::collect_left_broadcast_safe;
     use datafusion::{
         arrow::datatypes::{DataType, Field, Schema},
         common::{ColumnStatistics, JoinSide, JoinType, Statistics, stats::Precision},
@@ -630,6 +655,41 @@ mod test {
             test::exec::StatisticsExec,
         },
     };
+
+    // A CollectLeft broadcast replicates the build (left) side to every probe
+    // task. Only join types whose output is driven entirely by the probe (right)
+    // side may be broadcast; any type that emits rows on behalf of the build side
+    // would duplicate them across tasks. This locks down the full set so a new or
+    // reclassified `JoinType` can't silently become broadcastable.
+    #[test]
+    fn collect_left_broadcast_safe_matches_probe_driven_join_types() {
+        let safe = [
+            JoinType::Inner,
+            JoinType::Right,
+            JoinType::RightSemi,
+            JoinType::RightAnti,
+            JoinType::RightMark,
+        ];
+        let unsafe_types = [
+            JoinType::Left,
+            JoinType::Full,
+            JoinType::LeftSemi,
+            JoinType::LeftAnti,
+            JoinType::LeftMark,
+        ];
+        for join_type in safe {
+            assert!(
+                collect_left_broadcast_safe(join_type),
+                "{join_type:?} should be safe to broadcast"
+            );
+        }
+        for join_type in unsafe_types {
+            assert!(
+                !collect_left_broadcast_safe(join_type),
+                "{join_type:?} must not be broadcast"
+            );
+        }
+    }
     //
     // join selection should not change order of joins for
     // nested loop join as we're not able to insert new CoalescePartitions
@@ -830,7 +890,7 @@ mod test {
         // `swap_inputs` for Inner wraps the join in a ProjectionExec to
         // restore the output column order. Walk the tree to find the join.
         fn find_hash_join(plan: &Arc<dyn ExecutionPlan>) -> Option<&HashJoinExec> {
-            if let Some(hj) = plan.as_any().downcast_ref::<HashJoinExec>() {
+            if let Some(hj) = plan.downcast_ref::<HashJoinExec>() {
                 return Some(hj);
             }
             for child in plan.children() {
diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs
index ed396719f4..0128540c5b 100644
--- a/ballista/scheduler/src/planner.rs
+++ b/ballista/scheduler/src/planner.rs
@@ -35,11 +35,10 @@ use ballista_core::{
 use datafusion::arrow::datatypes::DataType;
 use datafusion::common::tree_node::{Transformed, TreeNode};
 use datafusion::config::ConfigOptions;
-use datafusion::physical_expr::PhysicalExpr;
 use datafusion::physical_optimizer::PhysicalOptimizerRule;
 use datafusion::physical_optimizer::enforce_sorting::EnforceSorting;
 use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
-use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};
+use datafusion::physical_plan::empty::EmptyExec;
 use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};
 use datafusion::physical_plan::projection::ProjectionExec;
 use datafusion::physical_plan::repartition::RepartitionExec;
@@ -49,7 +48,9 @@ use datafusion::physical_plan::{
 };
 use log::debug;
 
-use crate::physical_optimizer::join_selection::should_swap_join_order;
+use crate::physical_optimizer::join_selection::{
+    collect_left_broadcast_safe, should_swap_join_order,
+};
 
 type PartialQueryStageResult = (Arc<dyn ExecutionPlan>, Vec<Arc<dyn ShuffleWriter>>);
 
@@ -111,10 +112,8 @@ impl DistributedPlanner for DefaultDistributedPlanner {
         config: &ConfigOptions,
     ) -> Result<Vec<Arc<dyn ShuffleWriter>>> {
         debug!("Planning query stages for job: [{job_id}]");
-        // Workaround until DF 54 migration : <https://github.com/apache/datafusion/pull/21885>
-        let serde_safe_plan = make_filter_projection_serde_safe(execution_plan)?;
         let (new_plan, mut stages) =
-            self.plan_query_stages_internal(job_id, serde_safe_plan, config)?;
+            self.plan_query_stages_internal(job_id, execution_plan, config)?;
         stages.push(create_shuffle_writer_with_config(
             job_id,
             self.next_stage_id(),
@@ -146,15 +145,13 @@ impl DefaultDistributedPlanner {
 
         // Broadcast-join lowering: HashJoinExec(CollectLeft) gets its own
         // controlled recursion so the build side is written as a broadcast stage.
-        if let Some(hash_join) = execution_plan.as_any().downcast_ref::<HashJoinExec>()
+        if let Some(hash_join) = execution_plan.downcast_ref::<HashJoinExec>()
             && *hash_join.partition_mode() == PartitionMode::CollectLeft
         {
             // Build subtree: peel CoalescePartitionsExec if present, then
             // recurse to lower its internal stages.
             let mut build = hash_join.left().clone();
-            if let Some(coalesce) =
-                build.as_any().downcast_ref::<CoalescePartitionsExec>()
-            {
+            if let Some(coalesce) = build.downcast_ref::<CoalescePartitionsExec>() {
                 build = coalesce.children()[0].clone();
             }
             let (build, mut stages) =
@@ -198,10 +195,7 @@ impl DefaultDistributedPlanner {
             stages.append(&mut child_stages);
         }
 
-        if let Some(_coalesce) = execution_plan
-            .as_any()
-            .downcast_ref::<CoalescePartitionsExec>()
-        {
+        if let Some(_coalesce) = execution_plan.downcast_ref::<CoalescePartitionsExec>() {
             let input = children[0].clone();
             let input = self.optimizer_enforce_sorting.optimize(input, config)?;
             let shuffle_writer = create_shuffle_writer_with_config(
@@ -218,10 +212,9 @@ impl DefaultDistributedPlanner {
                 with_new_children_if_necessary(execution_plan, vec![unresolved_shuffle])?,
                 stages,
             ))
-        } else if let Some(_sort_preserving_merge) = execution_plan
-            .as_any()
-            .downcast_ref::<SortPreservingMergeExec>(
-        ) {
+        } else if let Some(_sort_preserving_merge) =
+            execution_plan.downcast_ref::<SortPreservingMergeExec>()
+        {
             let shuffle_writer = create_shuffle_writer_with_config(
                 job_id,
                 self.next_stage_id(),
@@ -235,9 +228,7 @@ impl DefaultDistributedPlanner {
                 with_new_children_if_necessary(execution_plan, vec![unresolved_shuffle])?,
                 stages,
             ))
-        } else if let Some(repart) =
-            execution_plan.as_any().downcast_ref::<RepartitionExec>()
-        {
+        } else if let Some(repart) = execution_plan.downcast_ref::<RepartitionExec>() {
             match repart.properties().output_partitioning() {
                 Partitioning::Hash(_, _) => {
                     let input = children[0].clone();
@@ -275,10 +266,22 @@ impl DefaultDistributedPlanner {
         self.next_stage_id
     }
 
-    /// If `plan` is a `HashJoinExec(Partitioned)` whose smaller side fits
-    /// under the broadcast threshold, returns a rewritten
-    /// `HashJoinExec(CollectLeft)` (with a swap if the small side was on
-    /// the right) wrapped so the build subtree is a single-partition input.
+    /// Reconciles a join's partition mode with the Ballista broadcast
+    /// threshold (`ballista.optimizer.broadcast_join_threshold_bytes`).
+    ///
+    /// - A `HashJoinExec(Partitioned)` whose smaller side fits under the
+    ///   threshold is rewritten as a `HashJoinExec(CollectLeft)` (with a swap
+    ///   if the small side was on the right) wrapped so the build subtree is a
+    ///   single-partition input.
+    /// - A `HashJoinExec(CollectLeft)` that DataFusion's `JoinSelection` chose
+    ///   using its own session threshold is demoted back to `Partitioned` when
+    ///   its join type is not broadcast-safe, or when the build side is not
+    ///   under the Ballista threshold (including a threshold of `0`, which
+    ///   disables broadcast joins). This makes the Ballista key authoritative
+    ///   even when it is overridden at runtime below the DataFusion session
+    ///   value. Null-aware anti joins are never demoted (they require
+    ///   `CollectLeft`).
+    ///
     /// Otherwise returns the input unchanged.
     fn maybe_promote_to_broadcast(
         plan: Arc<dyn ExecutionPlan>,
@@ -291,11 +294,54 @@ impl DefaultDistributedPlanner {
             .unwrap_or_else(|| {
                 BallistaConfig::default().broadcast_join_threshold_bytes()
             });
+
+        // DataFusion's own `JoinSelection` may have already stamped
+        // `CollectLeft` on this join (it does so for any build side under
+        // `datafusion.optimizer.hash_join_single_partition_threshold`, without
+        // restricting by join type). A `CollectLeft` join replicates the build
+        // side to every probe task.
+        if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>()
+            && *hash_join.partition_mode() == PartitionMode::CollectLeft
+        {
+            // Broadcasting is only correct for probe-driven join types. If the
+            // join type is not broadcast-safe, demote it back to a partitioned
+            // (shuffle) join. Correctness guard, independent of the threshold.
+            if !collect_left_broadcast_safe(*hash_join.join_type()) {
+                debug!(
+                    "broadcast check: demoting DataFusion-promoted CollectLeft join with unsafe join_type={:?} to Partitioned",
+                    hash_join.join_type(),
+                );
+                return Self::demote_collect_left_to_partitioned(hash_join, config);
+            }
+            // Null-aware anti joins must stay `CollectLeft`: they track
+            // probe-side state that a partitioned join cannot reconstruct, so
+            // never demote them regardless of the threshold.
+            if hash_join.null_aware {
+                return Ok(plan);
+            }
+            // Safe join type: honor the Ballista broadcast threshold. DataFusion
+            // decided `CollectLeft` using its own session threshold, which can
+            // exceed a user's runtime `broadcast_join_threshold_bytes` override.
+            // If broadcasts are disabled (0) or the build (left) side is not
+            // under the Ballista threshold, demote so the Ballista key is
+            // authoritative in the static planner path too.
+            if threshold_bytes == 0 || !under(&**hash_join.left(), threshold_bytes) {
+                debug!(
+                    "broadcast check: demoting CollectLeft join to Partitioned; build side not under Ballista threshold={threshold_bytes} (or broadcasts disabled)",
+                );
+                return Self::demote_collect_left_to_partitioned(hash_join, config);
+            }
+            return Ok(plan);
+        }
+
         if threshold_bytes == 0 {
             debug!("broadcast check: threshold is 0, broadcast disabled");
             return Ok(plan);
         }
-        let Some(hash_join) = plan.as_any().downcast_ref::<HashJoinExec>() else {
+
+        // The candidate must already be a `HashJoinExec(Partitioned)`. A
+        // `SortMergeJoinExec` is never a broadcast candidate.
+        let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() else {
             return Ok(plan);
         };
         debug!(
@@ -375,6 +421,22 @@ impl DefaultDistributedPlanner {
             right_under
         };
 
+        // A CollectLeft join broadcasts the build (left) side to every probe
+        // task, which is only correct for join types that never emit rows on
+        // behalf of the build side. The build side is the left input of the
+        // resulting join, so check the join type after the swap is applied.
+        let promoted_join_type = if swap {
+            hash_join.join_type().swap()
+        } else {
+            *hash_join.join_type()
+        };
+        if !collect_left_broadcast_safe(promoted_join_type) {
+            debug!(
+                "broadcast check: join type {promoted_join_type:?} is not broadcast-safe, skipping promotion"
+            );
+            return Ok(plan);
+        }
+
         debug!(
             "broadcast check: promoting to CollectLeft (left_under={left_under}, right_under={right_under}, swap={swap})"
         );
@@ -397,10 +459,24 @@ impl DefaultDistributedPlanner {
             )
         };
 
-        let promoted_join = promoted
-            .as_any()
+        // `swap_inputs` may wrap the join in a `ProjectionExec` to restore the
+        // original column order. Locate the `HashJoinExec` (bare or under that
+        // projection) so the build side can be coalesced, then re-wrap if needed.
+        let (join_node, projection): (
+            Arc<dyn ExecutionPlan>,
+            Option<Arc<dyn ExecutionPlan>>,
+        ) = if promoted.downcast_ref::<HashJoinExec>().is_some() {
+            (promoted.clone(), None)
+        } else if let Some(proj) = promoted.downcast_ref::<ProjectionExec>() {
+            (proj.input().clone(), Some(promoted.clone()))
+        } else {
+            debug!("broadcast check: unexpected promoted plan shape, skipping");
+            return Ok(plan);
+        };
+
+        let promoted_join = join_node
             .downcast_ref::<HashJoinExec>()
-            .expect("promoted plan must still be a HashJoinExec");
+            .expect("promoted join node must be a HashJoinExec");
         let new_left: Arc<dyn ExecutionPlan> = if promoted_join
             .left()
             .properties()
@@ -413,45 +489,47 @@ impl DefaultDistributedPlanner {
             promoted_join.left().clone()
         };
         let new_right = promoted_join.right().clone();
-        Ok(with_new_children_if_necessary(
-            promoted,
-            vec![new_left, new_right],
-        )?)
+        let rebuilt_join =
+            with_new_children_if_necessary(join_node, vec![new_left, new_right])?;
+
+        // Re-wrap in the projection if `swap_inputs` added one. The recursive
+        // lowering descends into the projection and broadcasts the build side of
+        // the inner `HashJoinExec(CollectLeft)`.
+        match projection {
+            Some(proj) => Ok(with_new_children_if_necessary(proj, vec![rebuilt_join])?),
+            None => Ok(rebuilt_join),
+        }
     }
-}
 
-/// Workaround until DF 54 migration: <https://github.com/apache/datafusion/pull/21885>
-/// datafusion-proto 53.1.0 encodes both `Some([])` (zero cols) and `None` (all cols)
-/// as an empty list, decoding back to `None` and shifting column indices (#1838).
-/// Rewrite `FilterExec(Some([]))` → empty `ProjectionExec` wrapped `FilterExec(None)`
-fn make_filter_projection_serde_safe(
-    plan: Arc<dyn ExecutionPlan>,
-) -> datafusion::error::Result<Arc<dyn ExecutionPlan>> {
-    Ok(plan
-        .transform_up(|node| {
-            if let Some(filter) = node.as_any().downcast_ref::<FilterExec>() {
-                let empty_projection = filter
-                    .projection()
-                    .as_ref()
-                    .map(|p| p.is_empty())
-                    .unwrap_or(false);
-                if empty_projection {
-                    let orig_filter_exec = FilterExecBuilder::from(filter)
-                        .apply_projection(None)?
-                        .build()?;
-                    let zero_expr: Vec<(Arc<dyn PhysicalExpr>, String)> = vec![];
-                    let proj = ProjectionExec::try_new(
-                        zero_expr,
-                        Arc::new(orig_filter_exec) as Arc<dyn ExecutionPlan>,
-                    )?;
-                    return Ok(
-                        Transformed::yes(Arc::new(proj) as Arc<dyn ExecutionPlan>),
-                    );
-                }
-            }
-            Ok(Transformed::no(node))
-        })?
-        .data)
+    /// Rewrites a `HashJoinExec(CollectLeft)` as an equivalent
+    /// `HashJoinExec(Partitioned)`, hash-partitioning both inputs on the join
+    /// keys so the join is executed as a shuffle join rather than by
+    /// broadcasting the build side. Used to undo an unsafe `CollectLeft`
+    /// promotion that DataFusion's `JoinSelection` applied before the
+    /// distributed planner ran.
+    fn demote_collect_left_to_partitioned(
+        hash_join: &HashJoinExec,
+        config: &ConfigOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let partitions = config.execution.target_partitions;
+        let left_keys: Vec<_> =
+            hash_join.on().iter().map(|(l, _)| Arc::clone(l)).collect();
+        let right_keys: Vec<_> =
+            hash_join.on().iter().map(|(_, r)| Arc::clone(r)).collect();
+        let new_left: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(
+            hash_join.left().clone(),
+            Partitioning::Hash(left_keys, partitions),
+        )?);
+        let new_right: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(
+            hash_join.right().clone(),
+            Partitioning::Hash(right_keys, partitions),
+        )?);
+        Ok(hash_join
+            .builder()
+            .with_partition_mode(PartitionMode::Partitioned)
+            .with_new_children(vec![new_left, new_right])?
+            .build_exec()?)
+    }
 }
 
 fn create_unresolved_shuffle(
@@ -470,9 +548,7 @@ fn create_unresolved_shuffle(
 pub fn find_unresolved_shuffles(
     plan: &Arc<dyn ExecutionPlan>,
 ) -> Result<Vec<UnresolvedShuffleExec>> {
-    if let Some(unresolved_shuffle) =
-        plan.as_any().downcast_ref::<UnresolvedShuffleExec>()
-    {
+    if let Some(unresolved_shuffle) = plan.downcast_ref::<UnresolvedShuffleExec>() {
         Ok(vec![unresolved_shuffle.clone()])
     } else {
         Ok(plan
@@ -495,9 +571,7 @@ pub fn remove_unresolved_shuffles(
 ) -> Result<Arc<dyn ExecutionPlan>> {
     let mut new_children: Vec<Arc<dyn ExecutionPlan>> = vec![];
     for child in stage.children() {
-        if let Some(unresolved_shuffle) =
-            child.as_any().downcast_ref::<UnresolvedShuffleExec>()
-        {
+        if let Some(unresolved_shuffle) = child.downcast_ref::<UnresolvedShuffleExec>() {
             let p = partition_locations
                 .get(&unresolved_shuffle.stage_id)
                 .ok_or_else(|| {
@@ -558,7 +632,7 @@ pub fn rollback_resolved_shuffles(
 ) -> Result<Arc<dyn ExecutionPlan>> {
     let mut new_children: Vec<Arc<dyn ExecutionPlan>> = vec![];
     for child in stage.children() {
-        if let Some(shuffle_reader) = child.as_any().downcast_ref::<ShuffleReaderExec>() {
+        if let Some(shuffle_reader) = child.downcast_ref::<ShuffleReaderExec>() {
             let stage_id = shuffle_reader.stage_id;
             let unresolved = if shuffle_reader.broadcast {
                 Arc::new(UnresolvedShuffleExec::new_broadcast(
@@ -581,6 +655,42 @@ pub fn rollback_resolved_shuffles(
     Ok(with_new_children_if_necessary(stage, new_children)?)
 }
 
+/// Rewrites every multi-partition [`EmptyExec`] into a round-robin [`RepartitionExec`]
+/// over a single-partition [`EmptyExec`], which is an equivalent plan whose partition
+/// count survives serialization.
+///
+/// `datafusion-proto` encodes an `EmptyExec` as its schema alone, so a multi-partition
+/// `EmptyExec` decodes on the executor with a single partition
+/// (<https://github.com/apache/datafusion/issues/23642>). A stage is sized from the
+/// scheduler-side partition count, so every task above partition 0 would then fail with
+/// `EmptyExec invalid partition N (expected less than 1)`. A `RepartitionExec` carries
+/// its partitioning on the wire, so the count arrives intact.
+///
+/// This runs at the point the stage plan is built for the wire, after all physical
+/// optimizer rules, so no later rule can collapse the rewrite. It can be removed once
+/// the partition count survives `EmptyExec` serialization upstream.
+fn make_empty_exec_serde_safe(
+    plan: Arc<dyn ExecutionPlan>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    Ok(plan
+        .transform_up(|node| {
+            let Some(empty) = node.downcast_ref::<EmptyExec>() else {
+                return Ok(Transformed::no(node));
+            };
+            let partition_count =
+                empty.properties().output_partitioning().partition_count();
+            if partition_count <= 1 {
+                return Ok(Transformed::no(node));
+            }
+            let single: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(empty.schema()));
+            Ok(Transformed::yes(Arc::new(RepartitionExec::try_new(
+                single,
+                Partitioning::RoundRobinBatch(partition_count),
+            )?)))
+        })?
+        .data)
+}
+
 pub(crate) fn create_shuffle_writer_with_config(
     job_id: &JobId,
     stage_id: usize,
@@ -588,6 +698,8 @@ pub(crate) fn create_shuffle_writer_with_config(
     partitioning: Option<Partitioning>,
     config: &ConfigOptions,
 ) -> Result<Arc<dyn ShuffleWriter>> {
+    let plan = make_empty_exec_serde_safe(plan)?;
+
     // Check if sort-based shuffle is enabled
     let ballista_config = config
         .extensions
@@ -600,7 +712,6 @@ pub(crate) fn create_shuffle_writer_with_config(
         if let Some(Partitioning::Hash(exprs, partition_count)) = partitioning {
             let sort_config = SortShuffleConfig::new(
                 true,
-                datafusion::arrow::ipc::CompressionType::LZ4_FRAME,
                 ballista_config.shuffle_sort_based_batch_size(),
             )
             .with_memory_limit_per_task_bytes(
@@ -630,21 +741,20 @@ pub(crate) fn create_shuffle_writer_with_config(
 
 #[cfg(test)]
 mod test {
+    use crate::assert_plan;
     use crate::planner::{DefaultDistributedPlanner, DistributedPlanner};
     use crate::test_utils::datafusion_test_context;
     use ballista_core::error::BallistaError;
     use ballista_core::execution_plans::{SortShuffleWriterExec, UnresolvedShuffleExec};
     use ballista_core::serde::BallistaCodec;
     use datafusion::arrow::compute::SortOptions;
-    use datafusion::arrow::datatypes::{DataType, Field, Schema};
-    use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
+
     use datafusion::execution::TaskContext;
     use datafusion::physical_expr::expressions::Column;
     use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode};
-    use datafusion::physical_plan::empty::EmptyExec;
-    use datafusion::physical_plan::expressions::lit;
+
     use datafusion::physical_plan::filter::FilterExec;
-    use datafusion::physical_plan::filter::FilterExecBuilder;
+
     use datafusion::physical_plan::joins::HashJoinExec;
     use datafusion::physical_plan::projection::ProjectionExec;
     use datafusion::physical_plan::sorts::sort::SortExec;
@@ -658,68 +768,51 @@ mod test {
     use std::sync::Arc;
     use uuid::Uuid;
 
-    macro_rules! downcast_exec {
-        ($exec: expr, $ty: ty) => {
-            $exec.as_any().downcast_ref::<$ty>().expect(&format!(
-                "Downcast to {} failed. Got {:?}",
-                stringify!($ty),
-                $exec
-            ))
-        };
-    }
-
-    /// Regression test for issue #1838: a `FilterExec` projecting to zero columns
-    /// is not serialization-safe in datafusion-proto 53.1.0 (the empty projection
-    /// decodes back as `None`, shifting downstream column indices). The planner
-    /// must rewrite it into a serde-safe equivalent that still emits zero columns.
     #[test]
-    fn empty_projection_filter_is_rewritten_serde_safe() -> Result<(), BallistaError> {
-        // 1-column input feeding a FilterExec that projects to ZERO columns
-        // (mirrors TPC-DS Q9's `FROM reason WHERE r_reason_sk = 1`).
-        let schema = Arc::new(Schema::new(vec![Field::new(
-            "r_reason_sk",
-            DataType::Int32,
-            false,
-        )]));
-        let input: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(schema));
-        let filter = FilterExecBuilder::new(lit(true), input)
-            .apply_projection(Some(vec![]))?
-            .build()?;
-        let plan: Arc<dyn ExecutionPlan> = Arc::new(filter);
-        assert_eq!(
-            plan.schema().fields().len(),
-            0,
-            "precondition: filter projects to zero columns"
-        );
+    fn multi_partition_empty_exec_is_rewritten_for_the_wire() {
+        use super::make_empty_exec_serde_safe;
+        use datafusion::arrow::datatypes::{DataType, Field, Schema};
+        use datafusion::physical_plan::empty::EmptyExec;
 
-        let serde_safe_plan = super::make_filter_projection_serde_safe(plan)?;
+        let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+        let plan: Arc<dyn ExecutionPlan> =
+            Arc::new(EmptyExec::new(schema).with_partitions(4));
+
+        let rewritten = make_empty_exec_serde_safe(plan).unwrap();
 
-        // schema is preserved
         assert_eq!(
-            serde_safe_plan.schema().fields().len(),
-            0,
-            "rewrite must preserve the zero-column output schema"
+            displayable(rewritten.as_ref()).indent(false).to_string(),
+            "RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1\n  EmptyExec\n"
         );
+    }
 
-        // FilterExec does not retain an empty projection
-        let mut has_empty_filter_projection = false;
-        serde_safe_plan.apply(|node| {
-            if let Some(f) = node.as_any().downcast_ref::<FilterExec>()
-                && f.projection()
-                    .as_ref()
-                    .map(|p| p.is_empty())
-                    .unwrap_or(false)
-            {
-                has_empty_filter_projection = true;
-            }
-            Ok(TreeNodeRecursion::Continue)
-        })?;
-        assert!(
-            !has_empty_filter_projection,
-            "FilterExec with empty projection must be rewritten to be serde-safe"
+    #[test]
+    fn single_partition_empty_exec_is_left_alone() {
+        use super::make_empty_exec_serde_safe;
+        use datafusion::arrow::datatypes::{DataType, Field, Schema};
+        use datafusion::physical_plan::empty::EmptyExec;
+
+        let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(schema));
+
+        let rewritten = make_empty_exec_serde_safe(plan).unwrap();
+
+        assert_eq!(
+            displayable(rewritten.as_ref()).indent(false).to_string(),
+            "EmptyExec\n"
         );
+    }
 
-        Ok(())
+    macro_rules! downcast_exec {
+        ($exec: expr, $ty: ty) => {
+            ($exec.as_ref() as &dyn ExecutionPlan)
+                .downcast_ref::<$ty>()
+                .expect(&format!(
+                    "Downcast to {} failed. Got {:?}",
+                    stringify!($ty),
+                    $exec
+                ))
+        };
     }
 
     #[tokio::test]
@@ -762,11 +855,11 @@ mod test {
           SortExec: expr=[l_returnflag@0 ASC NULLS LAST], preserve_partitioning=[true]
             ProjectionExec: expr=[l_returnflag@0 as l_returnflag, sum(lineitem.l_extendedprice * Int64(1))@1 as sum_disc_price]
               AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag], aggr=[sum(lineitem.l_extendedprice * Int64(1))]
-                UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0], 2)
+                UnresolvedShuffleExec: stage=1, partitioning: Hash([l_returnflag@0], 2)
 
         ShuffleWriterExec: partitioning: None
           SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST]
-            UnresolvedShuffleExec: partitioning: Hash([l_returnflag@0], 2)
+            UnresolvedShuffleExec: stage=2, partitioning: Hash([l_returnflag@0], 2)
         */
 
         assert_eq!(3, stages.len());
@@ -880,18 +973,18 @@ order by
         ShuffleWriterExec: partitioning: Hash([l_shipmode@0], 2)
           AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)]
             HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_shipmode@1, o_orderpriority@3]
-              UnresolvedShuffleExec: partitioning: Hash([l_orderkey@0], 2)
-              UnresolvedShuffleExec: partitioning: Hash([o_orderkey@0], 2)
+              UnresolvedShuffleExec: stage=1, partitioning: Hash([l_orderkey@0], 2)
+              UnresolvedShuffleExec: stage=2, partitioning: Hash([o_orderkey@0], 2)
 
         ShuffleWriterExec: partitioning: None
           SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true]
             ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count]
             AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)]
-              UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 2)
+              UnresolvedShuffleExec: stage=3, partitioning: Hash([l_shipmode@0], 2)
 
         ShuffleWriterExec: partitioning: None
           SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST]
-            UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 2)
+            UnresolvedShuffleExec: stage=4, partitioning: Hash([l_shipmode@0], 2)
         */
 
         assert_eq!(5, stages.len());
@@ -995,7 +1088,7 @@ order by
     async fn distributed_broadcast_join_plan() -> Result<(), BallistaError> {
         use datafusion::physical_plan::joins::PartitionMode;
 
-        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1)?;
+        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?;
 
         let df = ctx
             .sql("select count(*) from big join small on big.k = small.k")
@@ -1017,11 +1110,10 @@ order by
             let mut walker: Vec<Arc<dyn ExecutionPlan>> =
                 vec![stage.clone() as Arc<dyn ExecutionPlan>];
             while let Some(node) = walker.pop() {
-                if let Some(hj) = node.as_any().downcast_ref::<HashJoinExec>() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
                     assert_eq!(*hj.partition_mode(), PartitionMode::CollectLeft);
                     let left = hj.children()[0].clone();
                     let unresolved = left
-                        .as_any()
                         .downcast_ref::<UnresolvedShuffleExec>()
                         .expect("left input should be UnresolvedShuffleExec");
                     assert!(unresolved.broadcast, "left input should be broadcast");
@@ -1039,12 +1131,47 @@ order by
         Ok(())
     }
 
+    #[tokio::test]
+    async fn distributed_sort_merge_join_is_never_broadcast() -> Result<(), BallistaError>
+    {
+        // SMJ planned (prefer_hash_join=false). A SortMergeJoinExec is never a
+        // broadcast candidate, so it stays a SortMergeJoinExec over sorted
+        // inputs even though the small side is well under the threshold.
+        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false)?;
+
+        let df = ctx
+            .sql("select count(*) from big join small on big.k = small.k")
+            .await?;
+        let plan = df.into_optimized_plan()?;
+        let plan = ctx.state().create_physical_plan(&plan).await?;
+
+        let mut planner = DefaultDistributedPlanner::new();
+        let job_uuid = Uuid::new_v4();
+        let stages =
+            planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?;
+
+        // Stage 0 holds the join, still a SortMergeJoinExec over sorted inputs
+        // (no HashJoinExec, no broadcast UnresolvedShuffleExec).
+        assert_plan!(stages[0].as_ref(), @r"
+        ShuffleWriterExec: partitioning: None
+          AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
+            ProjectionExec: expr=[]
+              SortMergeJoinExec: join_type=Inner, on=[(k@0, k@0)]
+                SortExec: expr=[k@0 ASC], preserve_partitioning=[false]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                SortExec: expr=[k@0 ASC], preserve_partitioning=[false]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ");
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn distributed_join_plan_no_broadcast_when_threshold_zero()
     -> Result<(), BallistaError> {
         use datafusion::physical_plan::joins::PartitionMode;
 
-        let (ctx, options) = make_broadcast_test_ctx(0, 1)?;
+        let (ctx, options) = make_broadcast_test_ctx(0, 1, true)?;
 
         let df = ctx
             .sql("select count(*) from big join small on big.k = small.k")
@@ -1062,15 +1189,13 @@ order by
             let mut walker: Vec<Arc<dyn ExecutionPlan>> =
                 vec![stage.clone() as Arc<dyn ExecutionPlan>];
             while let Some(node) = walker.pop() {
-                if let Some(unresolved) =
-                    node.as_any().downcast_ref::<UnresolvedShuffleExec>()
-                {
+                if let Some(unresolved) = node.downcast_ref::<UnresolvedShuffleExec>() {
                     assert!(
                         !unresolved.broadcast,
                         "no broadcast reader expected with threshold=0"
                     );
                 }
-                if let Some(hj) = node.as_any().downcast_ref::<HashJoinExec>() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
                     assert_ne!(
                         *hj.partition_mode(),
                         PartitionMode::CollectLeft,
@@ -1084,6 +1209,346 @@ order by
         Ok(())
     }
 
+    // Across every expressible join type and both build-side orientations, any
+    // CollectLeft join the static planner promotes must be broadcast-safe: a
+    // CollectLeft join replicates the build (left) side to every probe task, so
+    // its (post-swap) join type must be driven by the probe side. `small` is
+    // under the threshold and `big` is not.
+    #[tokio::test]
+    async fn distributed_broadcast_only_promotes_safe_join_types()
+    -> Result<(), BallistaError> {
+        use crate::physical_optimizer::join_selection::collect_left_broadcast_safe;
+        use datafusion::physical_plan::joins::PartitionMode;
+
+        let sqls = [
+            "select * from big join small on big.k = small.k",
+            "select * from small join big on small.k = big.k",
+            "select * from big left join small on big.k = small.k",
+            "select * from small left join big on small.k = big.k",
+            "select * from big right join small on big.k = small.k",
+            "select * from small right join big on small.k = big.k",
+            "select * from big full join small on big.k = small.k",
+            "select * from small full join big on small.k = big.k",
+        ];
+
+        for sql in sqls {
+            let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?;
+            let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+            let plan = ctx.state().create_physical_plan(&plan).await?;
+
+            let mut planner = DefaultDistributedPlanner::new();
+            let job_uuid = Uuid::new_v4();
+            let stages = planner.plan_query_stages(
+                &job_uuid.to_string().into(),
+                plan,
+                &options,
+            )?;
+
+            for stage in &stages {
+                let mut walker: Vec<Arc<dyn ExecutionPlan>> =
+                    vec![stage.clone() as Arc<dyn ExecutionPlan>];
+                while let Some(node) = walker.pop() {
+                    if let Some(hj) = node.downcast_ref::<HashJoinExec>()
+                        && *hj.partition_mode() == PartitionMode::CollectLeft
+                    {
+                        assert!(
+                            collect_left_broadcast_safe(*hj.join_type()),
+                            "unsafe CollectLeft join_type={:?} promoted for `{sql}`",
+                            hj.join_type()
+                        );
+                    }
+                    walker.extend(node.children().iter().map(|c| (*c).clone()));
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    // When DataFusion's threshold is non-zero, DataFusion's own `JoinSelection`
+    // runs during `create_physical_plan` and can stamp `CollectLeft` on a LEFT
+    // join whose small side is already on the left (no swap, no join-type
+    // check). The distributed planner must demote such an unsafe `CollectLeft`
+    // join back to a partitioned join rather than broadcasting the outer side.
+    #[tokio::test]
+    async fn df_promoted_left_join_must_not_broadcast() -> Result<(), BallistaError> {
+        use ballista_core::extension::SessionConfigExt;
+        use datafusion::arrow::array::Int32Array;
+        use datafusion::arrow::datatypes::{DataType, Field, Schema};
+        use datafusion::arrow::record_batch::RecordBatch;
+        use datafusion::common::JoinType;
+        use datafusion::physical_plan::joins::PartitionMode;
+        use datafusion::prelude::SessionConfig;
+
+        let big_schema = Arc::new(Schema::new(vec![
+            Field::new("k", DataType::Int32, false),
+            Field::new("v", DataType::Int32, false),
+        ]));
+        let small_schema = Arc::new(Schema::new(vec![
+            Field::new("k", DataType::Int32, false),
+            Field::new("w", DataType::Int32, false),
+        ]));
+        let big_batch = RecordBatch::try_new(
+            big_schema,
+            vec![
+                Arc::new(Int32Array::from((0..10_000).collect::<Vec<_>>())),
+                Arc::new(Int32Array::from((0..10_000).collect::<Vec<_>>())),
+            ],
+        )
+        .map_err(|e| BallistaError::General(e.to_string()))?;
+        let small_batch = RecordBatch::try_new(
+            small_schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(Int32Array::from(vec![10, 20])),
+            ],
+        )
+        .map_err(|e| BallistaError::General(e.to_string()))?;
+
+        // Match the PR's production session config: DF and Ballista thresholds
+        // both at 10 MB so DF's `JoinSelection` can promote on its own.
+        let session_config = SessionConfig::new()
+            .with_target_partitions(2)
+            .set_bool("datafusion.optimizer.prefer_hash_join", true)
+            .set_usize(
+                "datafusion.optimizer.hash_join_single_partition_threshold",
+                10 * 1024 * 1024,
+            )
+            .with_ballista_broadcast_join_threshold_bytes(10 * 1024 * 1024);
+        let ctx = datafusion::prelude::SessionContext::new_with_config(session_config);
+        ctx.register_batch("big", big_batch)?;
+        ctx.register_batch("small", small_batch)?;
+        let options = ctx.state().config().options().clone();
+
+        let plan = ctx
+            .sql("select * from small left join big on small.k = big.k")
+            .await?
+            .into_optimized_plan()?;
+        let plan = ctx.state().create_physical_plan(&plan).await?;
+
+        let mut planner = DefaultDistributedPlanner::new();
+        let job_uuid = Uuid::new_v4();
+        let stages =
+            planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?;
+
+        for stage in &stages {
+            let mut walker: Vec<Arc<dyn ExecutionPlan>> =
+                vec![stage.clone() as Arc<dyn ExecutionPlan>];
+            while let Some(node) = walker.pop() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
+                    assert!(
+                        !(matches!(hj.join_type(), JoinType::Left)
+                            && *hj.partition_mode() == PartitionMode::CollectLeft),
+                        "LEFT join must not be CollectLeft (would broadcast the \
+                         outer side across probe tasks)"
+                    );
+                }
+                if let Some(unresolved) = node.downcast_ref::<UnresolvedShuffleExec>() {
+                    assert!(
+                        !unresolved.broadcast,
+                        "no broadcast reader expected for an unsafe LEFT join"
+                    );
+                }
+                walker.extend(node.children().iter().map(|c| (*c).clone()));
+            }
+        }
+        Ok(())
+    }
+
+    // Plans `big join small` with DataFusion's own hash-join threshold high
+    // enough for its `JoinSelection` to promote to `CollectLeft`, and the
+    // Ballista broadcast threshold set to `ballista_threshold_bytes`. Returns
+    // the partition modes of every `HashJoinExec` across the resulting stages.
+    async fn collect_left_modes_for_ballista_threshold(
+        ballista_threshold_bytes: usize,
+    ) -> Result<Vec<datafusion::physical_plan::joins::PartitionMode>, BallistaError> {
+        use ballista_core::extension::SessionConfigExt;
+        use datafusion::arrow::array::Int32Array;
+        use datafusion::arrow::datatypes::{DataType, Field, Schema};
+        use datafusion::arrow::record_batch::RecordBatch;
+        use datafusion::prelude::SessionConfig;
+
+        let big_schema = Arc::new(Schema::new(vec![
+            Field::new("k", DataType::Int32, false),
+            Field::new("v", DataType::Int32, false),
+        ]));
+        let small_schema = Arc::new(Schema::new(vec![
+            Field::new("k", DataType::Int32, false),
+            Field::new("w", DataType::Int32, false),
+        ]));
+        let big_batch = RecordBatch::try_new(
+            big_schema,
+            vec![
+                Arc::new(Int32Array::from((0..10_000).collect::<Vec<_>>())),
+                Arc::new(Int32Array::from((0..10_000).collect::<Vec<_>>())),
+            ],
+        )
+        .map_err(|e| BallistaError::General(e.to_string()))?;
+        let small_batch = RecordBatch::try_new(
+            small_schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(Int32Array::from(vec![10, 20])),
+            ],
+        )
+        .map_err(|e| BallistaError::General(e.to_string()))?;
+
+        // DF threshold high so DataFusion's JoinSelection promotes to
+        // CollectLeft on its own; the Ballista threshold is what we vary.
+        let session_config = SessionConfig::new()
+            .with_target_partitions(2)
+            .set_bool("datafusion.optimizer.prefer_hash_join", true)
+            .set_usize(
+                "datafusion.optimizer.hash_join_single_partition_threshold",
+                10 * 1024 * 1024,
+            )
+            .with_ballista_broadcast_join_threshold_bytes(ballista_threshold_bytes);
+        let ctx = datafusion::prelude::SessionContext::new_with_config(session_config);
+        ctx.register_batch("big", big_batch)?;
+        ctx.register_batch("small", small_batch)?;
+        let options = ctx.state().config().options().clone();
+
+        let plan = ctx
+            .sql("select * from big join small on big.k = small.k")
+            .await?
+            .into_optimized_plan()?;
+        let plan = ctx.state().create_physical_plan(&plan).await?;
+
+        let mut planner = DefaultDistributedPlanner::new();
+        let job_uuid = Uuid::new_v4();
+        let stages =
+            planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?;
+
+        let mut modes = vec![];
+        for stage in &stages {
+            let mut walker: Vec<Arc<dyn ExecutionPlan>> =
+                vec![stage.clone() as Arc<dyn ExecutionPlan>];
+            while let Some(node) = walker.pop() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
+                    modes.push(*hj.partition_mode());
+                }
+                walker.extend(node.children().iter().map(|c| (*c).clone()));
+            }
+        }
+        Ok(modes)
+    }
+
+    // DataFusion's `JoinSelection` promotes a small build side to `CollectLeft`
+    // using its own session threshold. When the Ballista broadcast threshold is
+    // lowered below that build size at runtime, the distributed planner must
+    // demote the `CollectLeft` join back to `Partitioned` so the Ballista key is
+    // authoritative; a high Ballista threshold leaves the broadcast in place.
+    #[tokio::test]
+    async fn df_collect_left_demoted_when_over_ballista_threshold()
+    -> Result<(), BallistaError> {
+        use datafusion::physical_plan::joins::PartitionMode;
+
+        let kept = collect_left_modes_for_ballista_threshold(10 * 1024 * 1024).await?;
+        assert!(
+            kept.contains(&PartitionMode::CollectLeft),
+            "a high Ballista threshold must keep the DataFusion-promoted CollectLeft join, got {kept:?}"
+        );
+
+        let demoted = collect_left_modes_for_ballista_threshold(8).await?;
+        assert!(
+            !demoted.contains(&PartitionMode::CollectLeft),
+            "a Ballista threshold below the build side must demote the CollectLeft join to Partitioned, got {demoted:?}"
+        );
+
+        Ok(())
+    }
+
+    // A LEFT join with the small side on the left builds (broadcasts) the left
+    // side and emits a null-padded row for every unmatched left row; each probe
+    // task would emit those independently. The guard must keep it repartitioned
+    // even though the small side is under the broadcast threshold.
+    #[tokio::test]
+    async fn distributed_left_join_small_build_not_broadcast() -> Result<(), BallistaError>
+    {
+        use datafusion::physical_plan::joins::PartitionMode;
+
+        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, true)?;
+
+        let df = ctx
+            .sql("select * from small left join big on small.k = big.k")
+            .await?;
+        let plan = df.into_optimized_plan()?;
+        let plan = ctx.state().create_physical_plan(&plan).await?;
+
+        let mut planner = DefaultDistributedPlanner::new();
+        let job_uuid = Uuid::new_v4();
+        let stages =
+            planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?;
+
+        for stage in &stages {
+            let mut walker: Vec<Arc<dyn ExecutionPlan>> =
+                vec![stage.clone() as Arc<dyn ExecutionPlan>];
+            while let Some(node) = walker.pop() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
+                    assert_ne!(
+                        *hj.partition_mode(),
+                        PartitionMode::CollectLeft,
+                        "LEFT join with broadcast build side must not be CollectLeft"
+                    );
+                }
+                if let Some(unresolved) = node.downcast_ref::<UnresolvedShuffleExec>() {
+                    assert!(
+                        !unresolved.broadcast,
+                        "no broadcast reader expected for unsafe LEFT join"
+                    );
+                }
+                walker.extend(node.children().iter().map(|c| (*c).clone()));
+            }
+        }
+
+        Ok(())
+    }
+
+    // The static planner only promotes `HashJoinExec`. With prefer_hash_join
+    // disabled the join plans as a `SortMergeJoinExec`, so no broadcast happens
+    // even though the build side is under the threshold.
+    #[tokio::test]
+    async fn distributed_no_broadcast_when_sort_merge_join() -> Result<(), BallistaError>
+    {
+        use datafusion::physical_plan::joins::PartitionMode;
+
+        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024, 1, false)?;
+
+        let df = ctx
+            .sql("select count(*) from big join small on big.k = small.k")
+            .await?;
+        let plan = df.into_optimized_plan()?;
+        let plan = ctx.state().create_physical_plan(&plan).await?;
+
+        let mut planner = DefaultDistributedPlanner::new();
+        let job_uuid = Uuid::new_v4();
+        let stages =
+            planner.plan_query_stages(&job_uuid.to_string().into(), plan, &options)?;
+
+        for stage in &stages {
+            let mut walker: Vec<Arc<dyn ExecutionPlan>> =
+                vec![stage.clone() as Arc<dyn ExecutionPlan>];
+            while let Some(node) = walker.pop() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
+                    assert_ne!(
+                        *hj.partition_mode(),
+                        PartitionMode::CollectLeft,
+                        "no CollectLeft expected when joins are sort-merge"
+                    );
+                }
+                if let Some(unresolved) = node.downcast_ref::<UnresolvedShuffleExec>() {
+                    assert!(
+                        !unresolved.broadcast,
+                        "no broadcast reader expected for sort-merge join"
+                    );
+                }
+                walker.extend(node.children().iter().map(|c| (*c).clone()));
+            }
+        }
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn distributed_broadcast_join_plan_multi_partition_build()
     -> Result<(), BallistaError> {
@@ -1091,7 +1556,7 @@ order by
         // broadcast build stage has 3 input partitions and writes 3 shuffle
         // files. The broadcast UnresolvedShuffleExec must report
         // upstream_partition_count = 3.
-        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024 * 1024, 3)?;
+        let (ctx, options) = make_broadcast_test_ctx(10 * 1024 * 1024 * 1024, 3, true)?;
 
         let df = ctx
             .sql("select count(*) from big join small on big.k = small.k")
@@ -1113,8 +1578,7 @@ order by
             let mut walker: Vec<Arc<dyn ExecutionPlan>> =
                 vec![stage.clone() as Arc<dyn ExecutionPlan>];
             while let Some(node) = walker.pop() {
-                if let Some(unresolved) =
-                    node.as_any().downcast_ref::<UnresolvedShuffleExec>()
+                if let Some(unresolved) = node.downcast_ref::<UnresolvedShuffleExec>()
                     && unresolved.broadcast
                 {
                     max_upstream = max_upstream.max(unresolved.upstream_partition_count);
@@ -1157,7 +1621,7 @@ order by
                 host: "localhost".to_string(),
                 port: 50050,
                 grpc_port: 50051,
-                specification: ExecutorSpecification::default().with_task_slots(1),
+                specification: ExecutorSpecification::default().with_vcores(1),
                 os_info: ExecutorOperatingSystemSpecification::default(),
             },
             partition_stats: PartitionStats::new(Some(10), None, Some(1)),
@@ -1184,7 +1648,6 @@ order by
 
         let resolved_child = resolved.children()[0].clone();
         let reader = resolved_child
-            .as_any()
             .downcast_ref::<ShuffleReaderExec>()
             .expect("expected resolved ShuffleReaderExec");
         assert!(reader.broadcast);
@@ -1214,7 +1677,6 @@ order by
         let rolled_back = crate::planner::rollback_resolved_shuffles(parent)?;
         let child = rolled_back.children()[0].clone();
         let unresolved = child
-            .as_any()
             .downcast_ref::<UnresolvedShuffleExec>()
             .expect("expected rolled-back UnresolvedShuffleExec");
         assert!(unresolved.broadcast);
@@ -1272,12 +1734,12 @@ order by
                 FilterExec: rank() PARTITION BY [lineitem.l_shipmode] ORDER BY [lineitem.l_shipdate DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 100
                   BoundedWindowAggExec: wdw=[rank() PARTITION BY [lineitem.l_shipmode] ORDER BY [lineitem.l_shipdate DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [lineitem.l_shipmode] ORDER BY [lineitem.l_shipdate DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
                     SortExec: expr=[l_shipmode@1 ASC NULLS LAST, l_shipdate@0 DESC], preserve_partitioning=[true]
-                      UnresolvedShuffleExec: partitioning: Hash([l_shipmode@1], 2)
+                      UnresolvedShuffleExec: stage=1, partitioning: Hash([l_shipmode@1], 2)
 
             Stage 2:
             ShuffleWriterExec: partitioning: None
               SortPreservingMergeExec: [l_shipdate@1 ASC NULLS LAST, rk@2 ASC NULLS LAST]
-                UnresolvedShuffleExec: partitioning: Hash([l_shipmode@0], 2)
+                UnresolvedShuffleExec: stage=2, partitioning: Hash([l_shipmode@0], 2)
 
         */
 
@@ -1290,7 +1752,7 @@ order by
         assert_eq!(2, partitioning.partition_count());
         let partition_col = match partitioning {
             Partitioning::Hash(exprs, 2) => match exprs.as_slice() {
-                [col] => col.as_any().downcast_ref::<Column>(),
+                [col] => col.downcast_ref::<Column>(),
                 _ => None,
             },
             _ => None,
@@ -1304,7 +1766,7 @@ order by
         let window = downcast_exec!(filter.children()[0], BoundedWindowAggExec);
         let partition_by = window.partition_keys();
         let partition_by = match partition_by[..] {
-            [ref col] => col.as_any().downcast_ref::<Column>(),
+            [ref col] => col.downcast_ref::<Column>(),
             _ => None,
         };
         assert_eq!(Some(&Column::new("l_shipmode", 1)), partition_by);
@@ -1321,7 +1783,7 @@ order by
                 );
                 assert_eq!(
                     Some(&Column::new("l_shipmode", 1)),
-                    expr1.expr.as_any().downcast_ref()
+                    expr1.expr.downcast_ref()
                 );
                 assert_eq!(
                     SortOptions {
@@ -1332,7 +1794,7 @@ order by
                 );
                 assert_eq!(
                     Some(&Column::new("l_shipdate", 0)),
-                    expr2.expr.as_any().downcast_ref()
+                    expr2.expr.downcast_ref()
                 );
             }
             _ => panic!("invalid sort {sort:?}"),
@@ -1386,6 +1848,7 @@ order by
     fn make_broadcast_test_ctx(
         threshold_bytes: usize,
         small_partitions: usize,
+        prefer_hash_join: bool,
     ) -> Result<
         (
             datafusion::prelude::SessionContext,
@@ -1433,11 +1896,12 @@ order by
 
         let session_config = SessionConfig::new()
             .with_target_partitions(2)
+            .with_ballista_broadcast_join_threshold_bytes(threshold_bytes)
             .set_usize(
                 "datafusion.optimizer.hash_join_single_partition_threshold",
                 0,
             )
-            .with_ballista_broadcast_join_threshold_bytes(threshold_bytes);
+            .set_bool("datafusion.optimizer.prefer_hash_join", prefer_hash_join);
         let ctx = datafusion::prelude::SessionContext::new_with_config(session_config);
         ctx.register_batch("big", big_batch)?;
 
diff --git a/ballista/scheduler/src/scheduler_process.rs b/ballista/scheduler/src/scheduler_process.rs
index 24bc58b9b0..70a2dd09d7 100644
--- a/ballista/scheduler/src/scheduler_process.rs
+++ b/ballista/scheduler/src/scheduler_process.rs
@@ -20,6 +20,7 @@ use crate::flight_proxy_service::BallistaFlightProxyService;
 
 #[cfg(feature = "rest-api")]
 use crate::api::get_routes;
+use crate::api::health_routes;
 use crate::api::route_disabled;
 use crate::cluster::BallistaCluster;
 use crate::config::SchedulerConfig;
@@ -92,7 +93,7 @@ pub async fn start_grpc_service<
     address: SocketAddr,
     scheduler: SchedulerServer<T, U>,
 ) -> ballista_core::error::Result<()> {
-    let config = &scheduler.state.config;
+    let config = scheduler.state.config.clone();
     let scheduler_grpc_server = SchedulerGrpcServer::new(scheduler.clone())
         .max_encoding_message_size(config.grpc_server_max_encoding_message_size as usize)
         .max_decoding_message_size(config.grpc_server_max_decoding_message_size as usize);
@@ -135,16 +136,21 @@ pub async fn start_grpc_service<
     let tonic =
         tonic.fallback(|| async { SchedulerErrorResponse::new(StatusCode::NOT_FOUND) });
 
+    let scheduler = Arc::new(scheduler);
+    let health = health_routes(scheduler.clone());
+
     #[cfg(feature = "rest-api")]
     let final_route = if config.disable_rest_api {
         tonic
             .merge(route_disabled(
                 "REST API has been disabled at startup".to_string(),
             ))
+            .merge(health)
             .into_make_service_with_connect_info::<SocketAddr>()
     } else {
-        let axum = get_routes(Arc::new(scheduler));
+        let axum = get_routes(scheduler);
         axum.merge(tonic)
+            .merge(health)
             .into_make_service_with_connect_info::<SocketAddr>()
     };
 
@@ -153,6 +159,7 @@ pub async fn start_grpc_service<
         .merge(route_disabled(
             "REST API has been disabled at compile time".to_string(),
         ))
+        .merge(health)
         .into_make_service_with_connect_info::<SocketAddr>();
 
     let listener = tokio::net::TcpListener::bind(&address)
diff --git a/ballista/scheduler/src/scheduler_server/event.rs b/ballista/scheduler/src/scheduler_server/event.rs
index c02c14a997..2cb7e718b5 100644
--- a/ballista/scheduler/src/scheduler_server/event.rs
+++ b/ballista/scheduler/src/scheduler_server/event.rs
@@ -18,12 +18,35 @@
 use std::fmt::{Debug, Formatter};
 
 use datafusion::logical_expr::LogicalPlan;
+use datafusion::physical_plan::ExecutionPlan;
 
 use crate::state::execution_graph::RunningTaskInfo;
 use ballista_core::{JobId, JobStatusSubscriber, serde::protobuf::TaskStatus};
 use datafusion::prelude::SessionContext;
 use std::sync::Arc;
 
+/// A plan submitted for execution by a client.
+///
+/// Most jobs are submitted as a [LogicalPlan] which the scheduler optimizes
+/// and turns into a physical plan itself. A client may instead submit an
+/// already-built physical plan directly, bypassing logical planning on the
+/// scheduler entirely - e.g. for plans containing custom operators that have
+/// no logical-plan representation. Physical-plan jobs always use the static
+/// distributed planner; the adaptive query planner (AQE) requires a logical
+/// plan and is not available for this path.
+// `LogicalPlan` is much larger than `Arc<dyn ExecutionPlan>`, but `SubmitPlan`
+// values are only ever handled by reference or already boxed by their callers
+// (e.g. `QueryStageSchedulerEvent::JobQueued::plan`), so the size difference
+// does not cause repeated large stack copies in practice.
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone)]
+pub enum SubmitPlan {
+    /// A logical plan that the scheduler will plan into a physical plan.
+    Logical(LogicalPlan),
+    /// An already-built physical plan supplied directly by the client.
+    Physical(Arc<dyn ExecutionPlan>),
+}
+
 /// Events that drive the query stage scheduler state machine.
 #[derive(Clone)]
 pub enum QueryStageSchedulerEvent {
@@ -35,8 +58,8 @@ pub enum QueryStageSchedulerEvent {
         job_name: String,
         /// Session context for the job.
         session_ctx: Arc<SessionContext>,
-        /// Logical plan to execute.
-        plan: Box<LogicalPlan>,
+        /// Plan to execute, either logical or an already-built physical plan.
+        plan: Box<SubmitPlan>,
         /// Timestamp when the job was queued.
         queued_at: u64,
         /// job status subscriber
diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs
index 14a6952dab..3b41e918c0 100644
--- a/ballista/scheduler/src/scheduler_server/grpc.rs
+++ b/ballista/scheduler/src/scheduler_server/grpc.rs
@@ -16,22 +16,23 @@
 // under the License.
 
 use axum::extract::ConnectInfo;
+use ballista_core::BALLISTA_PROTOCOL_VERSION;
 use ballista_core::config::BALLISTA_JOB_NAME;
 use ballista_core::error::{BallistaError, Result as BResult};
 use ballista_core::extension::SessionConfigHelperExt;
 use ballista_core::serde::protobuf::execute_query_params::Query;
 use ballista_core::serde::protobuf::scheduler_grpc_server::SchedulerGrpc;
 use ballista_core::serde::protobuf::{
-    AvailableTaskSlots, CancelJobParams, CancelJobResult, CleanJobDataParams,
+    AvailableVcores, CancelJobParams, CancelJobResult, CleanJobDataParams,
     CleanJobDataResult, CreateUpdateSessionParams, CreateUpdateSessionResult,
     ExecuteQueryFailureResult, ExecuteQueryParams, ExecuteQueryResult,
-    ExecuteQuerySuccessResult, ExecutorHeartbeat, ExecutorStoppedParams,
-    ExecutorStoppedResult, GetJobMetricsParams, GetJobMetricsResult, GetJobStatusParams,
-    GetJobStatusResult, HeartBeatParams, HeartBeatResult, JobStatus, KeyValuePair,
-    PollWorkParams, PollWorkResult, RegisterExecutorParams, RegisterExecutorResult,
-    RemoveSessionParams, RemoveSessionResult, UpdateTaskStatusParams,
-    UpdateTaskStatusResult, execute_query_failure_result, execute_query_result,
-    executor_metric::Metric,
+    ExecuteQuerySuccessResult, ExecutorHeartbeat, ExecutorRegistration,
+    ExecutorStoppedParams, ExecutorStoppedResult, GetJobMetricsParams,
+    GetJobMetricsResult, GetJobStatusParams, GetJobStatusResult, HeartBeatParams,
+    HeartBeatResult, JobStatus, KeyValuePair, PollWorkParams, PollWorkResult,
+    RegisterExecutorParams, RegisterExecutorResult, RemoveSessionParams,
+    RemoveSessionResult, UpdateTaskStatusParams, UpdateTaskStatusResult,
+    execute_query_failure_result, execute_query_result, executor_metric::Metric,
 };
 use ballista_core::serde::scheduler::ExecutorMetadata;
 use datafusion_proto::logical_plan::AsLogicalPlan;
@@ -51,9 +52,8 @@ use {
 
 use crate::cluster::{bind_task_bias, bind_task_round_robin};
 use crate::config::TaskDistributionPolicy;
-use crate::scheduler_server::event::QueryStageSchedulerEvent;
+use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan};
 use ballista_core::serde::protobuf::get_job_status_result::FlightProxy;
-use datafusion::logical_expr::LogicalPlan;
 use datafusion::physical_plan::{DisplayFormatType, ExecutionPlan};
 use datafusion::prelude::SessionContext;
 use std::ops::Deref;
@@ -61,8 +61,27 @@ use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tonic::{Request, Response, Status};
 
+use crate::metrics::record_protocol_mismatch;
 use crate::scheduler_server::SchedulerServer;
 
+/// Rejects an executor RPC whose `ballista_protocol_version` does not match
+/// the scheduler's compiled-in `BALLISTA_PROTOCOL_VERSION`. See the constant
+/// in `ballista_core` for the upgrade semantics.
+fn check_protocol_version(metadata: &ExecutorRegistration) -> Result<(), Status> {
+    if metadata.ballista_protocol_version == BALLISTA_PROTOCOL_VERSION {
+        return Ok(());
+    }
+    record_protocol_mismatch();
+    info!(
+        "Rejecting executor {}: protocol version mismatch (scheduler={}, executor={})",
+        metadata.id, BALLISTA_PROTOCOL_VERSION, metadata.ballista_protocol_version,
+    );
+    Err(Status::failed_precondition(format!(
+        "protocol version mismatch: scheduler={}, executor={}",
+        BALLISTA_PROTOCOL_VERSION, metadata.ballista_protocol_version,
+    )))
+}
+
 #[tonic::async_trait]
 impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
     for SchedulerServer<T, U>
@@ -83,11 +102,12 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
         let remote_addr = extract_connect_info(&request);
         if let PollWorkParams {
             metadata: Some(metadata),
-            num_free_slots,
+            num_free_vcores,
             task_status,
         } = request.into_inner()
         {
             trace!("Received poll_work request for {metadata:?}");
+            check_protocol_version(&metadata)?;
             let executor_id = metadata.id.clone();
 
             // It's not necessary.
@@ -118,35 +138,35 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
                 .map_err(|e| {
                     let msg = format!(
                         "Fail to update tasks status from executor {:?} due to {:?}",
-                        &executor_id, e
+                        executor_id, e
                     );
                     error!("{msg}");
                     Status::internal(msg)
                 })?;
 
-            let mut available_slots = [AvailableTaskSlots {
+            let mut budgets = [AvailableVcores {
                 executor_id: executor_id.clone(),
-                slots: num_free_slots,
+                vcores: num_free_vcores,
             }];
-            let available_slots = available_slots.iter_mut().collect();
+            let budgets = budgets.iter_mut().collect();
             let running_jobs = self.state.task_manager.get_running_job_cache();
             let schedulable_tasks = match self.state.config.task_distribution {
                 TaskDistributionPolicy::Bias => {
-                    bind_task_bias(available_slots, running_jobs, |_| false).await
+                    bind_task_bias(budgets, running_jobs, |_| false).await
                 }
                 TaskDistributionPolicy::RoundRobin => {
-                    bind_task_round_robin(available_slots, running_jobs, |_| false).await
+                    bind_task_round_robin(budgets, running_jobs, |_| false).await
                 }
 
                 TaskDistributionPolicy::Custom(ref policy) => policy
-                    .bind_tasks(available_slots, running_jobs)
+                    .bind_tasks(budgets, running_jobs)
                     .await
                     .map_err(|e| Status::internal(e.to_string()))?,
             };
 
             let mut tasks = vec![];
             for (_, task) in schedulable_tasks {
-                let job_id = task.partition.job_id.clone();
+                let job_id = task.key.job_id.clone();
                 match self.state.task_manager.prepare_task_definition(task) {
                     Ok(task_definition) => tasks.push(task_definition),
                     Err(e) => {
@@ -162,8 +182,9 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
                 .executor_manager
                 .drain_pending_cleanup_jobs(&executor_id)
                 .into_iter()
-                .map(|job_id| CleanJobDataParams {
+                .map(|(job_id, remove_stage_ids)| CleanJobDataParams {
                     job_id: job_id.into_inner(),
+                    remove_stage_ids,
                 })
                 .collect();
             Ok(Response::new(PollWorkResult {
@@ -186,6 +207,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
         } = request.into_inner()
         {
             info!("Received register executor request for {metadata:?}");
+            check_protocol_version(&metadata)?;
             let metadata = ExecutorMetadata {
                 id: metadata.id,
                 host: metadata
@@ -223,6 +245,19 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
         } = request.into_inner();
         trace!("Received heart beat request for {:?}", executor_id);
 
+        let Some(metadata_ref) = metadata.as_ref() else {
+            info!(
+                "Rejecting heartbeat from {executor_id}: missing registration metadata"
+            );
+            record_protocol_mismatch();
+            return Err(Status::failed_precondition(
+                "heartbeat missing registration metadata; \
+                 executors must include ExecutorRegistration \
+                 (see BALLISTA_PROTOCOL_VERSION)",
+            ));
+        };
+        check_protocol_version(metadata_ref)?;
+
         // If not registered, do registration first before saving heart beat
         if let Err(e) = self
             .state
@@ -327,7 +362,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
             .map_err(|e| {
                 let msg = format!(
                     "Fail to update tasks status from executor {:?} due to {:?}",
-                    &executor_id, e
+                    executor_id, e
                 );
                 error!("{msg}");
                 Status::internal(msg)
@@ -343,6 +378,11 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
         let session_params = request.into_inner();
 
         let session_config = self.state.session_manager.produce_config();
+        // TODO(c2): compute total cluster vcores (sum of vcores across
+        // registered executors from ClusterState.registered_executor_metadata())
+        // and inject it here so downstream rules like ParallelWindowDetectRule
+        // can size their output partition counts to actual cluster shape
+        // instead of hardcoding.
         let session_config =
             session_config.update_from_key_value_pair(&session_params.settings);
 
@@ -414,8 +454,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
             })?;
 
             debug!(
-                "Decoded logical plan for execution:\n{}",
-                plan.display_indent()
+                "Decoded plan for execution:\n{}",
+                Self::describe_submitted_plan(&plan)
             );
             log::trace!("setting job name: {job_name}");
 
@@ -430,7 +470,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
             });
 
             let job_id = self
-                .submit_job(&job_name, session_ctx, &plan, Some(subscriber))
+                .submit_plan(&job_name, session_ctx, &plan, Some(subscriber))
                 .await
                 .map_err(|e| {
                     let msg =
@@ -499,13 +539,13 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
             };
 
             debug!(
-                "Decoded logical plan for execution:\n{}",
-                plan.display_indent()
+                "Decoded plan for execution:\n{}",
+                Self::describe_submitted_plan(&plan)
             );
 
             log::trace!("setting job name: {job_name}");
             let job_id = self
-                .submit_job(&job_name, session_ctx, &plan, None)
+                .submit_plan(&job_name, session_ctx, &plan, None)
                 .await
                 .map_err(|e| {
                     let msg =
@@ -785,7 +825,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
         &self,
         query: Query,
         session_ctx: &SessionContext,
-    ) -> BResult<LogicalPlan> {
+    ) -> BResult<SubmitPlan> {
         match query {
             Query::LogicalPlan(message) => T::try_decode(message.as_slice())
                 .and_then(|m| {
@@ -794,8 +834,22 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
                         self.state.codec.logical_extension_codec(),
                     )
                 })
+                .map(SubmitPlan::Logical)
                 .map_err(|e| e.into()),
 
+            Query::PhysicalPlan(message) => {
+                let task_ctx = session_ctx.task_ctx();
+                U::try_decode(message.as_slice())
+                    .and_then(|m| {
+                        m.try_into_physical_plan(
+                            task_ctx.as_ref(),
+                            self.state.codec.physical_extension_codec(),
+                        )
+                    })
+                    .map(SubmitPlan::Physical)
+                    .map_err(|e| e.into())
+            }
+
             #[cfg(not(feature = "substrait"))]
             Query::SubstraitPlan(_) => {
                 Err(BallistaError::NotImplemented("Received query type \"Substrait\", enable \"substrait\" feature to support Substrait plans.".to_string()))
@@ -807,11 +861,24 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
                 let ctx = session_ctx.clone();
                 from_substrait_plan(&ctx.state(), &plan)
                     .await
+                    .map(SubmitPlan::Logical)
                     .map_err(|e| e.into())
             }
         }
     }
 
+    /// Returns a human-readable representation of a submitted plan, for logging.
+    fn describe_submitted_plan(plan: &SubmitPlan) -> String {
+        match plan {
+            SubmitPlan::Logical(plan) => plan.display_indent().to_string(),
+            SubmitPlan::Physical(plan) => {
+                datafusion::physical_plan::displayable(plan.as_ref())
+                    .indent(false)
+                    .to_string()
+            }
+        }
+    }
+
     fn flight_proxy_config(&self) -> Option<FlightProxy> {
         self.state
             .config
@@ -843,6 +910,7 @@ mod test {
 
     use crate::config::SchedulerConfig;
     use crate::metrics::default_metrics_collector;
+    use ballista_core::BALLISTA_PROTOCOL_VERSION;
     use ballista_core::error::BallistaError;
     use ballista_core::serde::BallistaCodec;
     use ballista_core::serde::protobuf::{
@@ -880,14 +948,13 @@ mod test {
             host: Some("http://localhost:8080".to_owned()),
             port: 0,
             grpc_port: 0,
-            specification: Some(
-                ExecutorSpecification::default().with_task_slots(2).into(),
-            ),
+            specification: Some(ExecutorSpecification::default().with_vcores(2).into()),
             os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+            ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
         };
         let request: Request<PollWorkParams> = Request::new(PollWorkParams {
             metadata: Some(exec_meta.clone()),
-            num_free_slots: 0,
+            num_free_vcores: 0,
             task_status: vec![],
         });
         let response = scheduler
@@ -913,12 +980,12 @@ mod test {
 
         assert_eq!(stored_executor.grpc_port, 0);
         assert_eq!(stored_executor.port, 0);
-        assert_eq!(stored_executor.specification.task_slots, 2);
+        assert_eq!(stored_executor.specification.vcores, 2);
         assert_eq!(stored_executor.host, "http://localhost:8080".to_owned());
 
         let request: Request<PollWorkParams> = Request::new(PollWorkParams {
             metadata: Some(exec_meta.clone()),
-            num_free_slots: 1,
+            num_free_vcores: 1,
             task_status: vec![],
         });
         let response = scheduler
@@ -945,7 +1012,7 @@ mod test {
 
         assert_eq!(stored_executor.grpc_port, 0);
         assert_eq!(stored_executor.port, 0);
-        assert_eq!(stored_executor.specification.task_slots, 2);
+        assert_eq!(stored_executor.specification.vcores, 2);
         assert_eq!(stored_executor.host, "http://localhost:8080".to_owned());
 
         Ok(())
@@ -971,10 +1038,9 @@ mod test {
             host: Some("http://localhost:8080".to_owned()),
             port: 0,
             grpc_port: 0,
-            specification: Some(
-                ExecutorSpecification::default().with_task_slots(2).into(),
-            ),
+            specification: Some(ExecutorSpecification::default().with_vcores(2).into()),
             os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+            ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
         };
 
         let request: Request<RegisterExecutorParams> =
@@ -1000,7 +1066,7 @@ mod test {
 
         assert_eq!(stored_executor.grpc_port, 0);
         assert_eq!(stored_executor.port, 0);
-        assert_eq!(stored_executor.specification.task_slots, 2);
+        assert_eq!(stored_executor.specification.vcores, 2);
         assert_eq!(stored_executor.host, "http://localhost:8080".to_owned());
 
         let request: Request<ExecutorStoppedParams> =
@@ -1059,10 +1125,9 @@ mod test {
             host: Some("http://localhost:8080".to_owned()),
             port: 0,
             grpc_port: 0,
-            specification: Some(
-                ExecutorSpecification::default().with_task_slots(2).into(),
-            ),
+            specification: Some(ExecutorSpecification::default().with_vcores(2).into()),
             os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+            ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
         };
 
         let request: Request<HeartBeatParams> = Request::new(HeartBeatParams {
@@ -1088,12 +1153,132 @@ mod test {
 
         assert_eq!(stored_executor.grpc_port, 0);
         assert_eq!(stored_executor.port, 0);
-        assert_eq!(stored_executor.specification.task_slots, 2);
+        assert_eq!(stored_executor.specification.vcores, 2);
         assert_eq!(stored_executor.host, "http://localhost:8080".to_owned());
 
         Ok(())
     }
 
+    async fn scheduler_for_protocol_test()
+    -> Result<SchedulerServer<LogicalPlanNode, PhysicalPlanNode>, BallistaError> {
+        let cluster = test_cluster_context();
+        let config = SchedulerConfig::default();
+        let mut scheduler: SchedulerServer<LogicalPlanNode, PhysicalPlanNode> =
+            SchedulerServer::new(
+                "localhost:50050".to_owned(),
+                cluster,
+                BallistaCodec::default(),
+                Arc::new(config),
+                default_metrics_collector().unwrap(),
+            );
+        scheduler.init().await?;
+        Ok(scheduler)
+    }
+
+    fn exec_meta_with_version(version: u32) -> ExecutorRegistration {
+        ExecutorRegistration {
+            id: "abc".to_owned(),
+            host: Some("http://localhost:8080".to_owned()),
+            port: 0,
+            grpc_port: 0,
+            specification: Some(ExecutorSpecification::default().with_vcores(2).into()),
+            os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+            ballista_protocol_version: version,
+        }
+    }
+
+    #[tokio::test]
+    async fn register_rejects_protocol_mismatch() -> Result<(), BallistaError> {
+        let scheduler = scheduler_for_protocol_test().await?;
+        // Version 0 is what an old executor without the field defaults to;
+        // the scheduler must reject it.
+        let request = Request::new(RegisterExecutorParams {
+            metadata: Some(exec_meta_with_version(0)),
+        });
+
+        let status = scheduler
+            .register_executor(request)
+            .await
+            .expect_err("register with mismatched version should be rejected");
+        assert_eq!(status.code(), tonic::Code::FailedPrecondition);
+        assert!(
+            status.message().contains("protocol version mismatch"),
+            "unexpected message: {}",
+            status.message()
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn heartbeat_rejects_protocol_mismatch() -> Result<(), BallistaError> {
+        let scheduler = scheduler_for_protocol_test().await?;
+        let meta = exec_meta_with_version(BALLISTA_PROTOCOL_VERSION + 1);
+        let request = Request::new(HeartBeatParams {
+            executor_id: meta.id.clone(),
+            metrics: vec![],
+            status: Some(ExecutorStatus {
+                status: Some(executor_status::Status::Active("".to_string())),
+            }),
+            metadata: Some(meta),
+        });
+        let status = scheduler
+            .heart_beat_from_executor(request)
+            .await
+            .expect_err("heartbeat with mismatched version should be rejected");
+        assert_eq!(status.code(), tonic::Code::FailedPrecondition);
+        assert!(status.message().contains("protocol version mismatch"));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn heartbeat_rejects_missing_metadata() -> Result<(), BallistaError> {
+        let scheduler = scheduler_for_protocol_test().await?;
+        let request = Request::new(HeartBeatParams {
+            executor_id: "abc".to_owned(),
+            metrics: vec![],
+            status: Some(ExecutorStatus {
+                status: Some(executor_status::Status::Active("".to_string())),
+            }),
+            metadata: None,
+        });
+        let status = scheduler
+            .heart_beat_from_executor(request)
+            .await
+            .expect_err("heartbeat with no metadata should be rejected");
+        assert_eq!(status.code(), tonic::Code::FailedPrecondition);
+        assert!(status.message().contains("missing registration metadata"));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn poll_work_rejects_protocol_mismatch() -> Result<(), BallistaError> {
+        let config = SchedulerConfig {
+            scheduling_policy: ballista_core::config::TaskSchedulingPolicy::PullStaged,
+            ..Default::default()
+        };
+        let mut scheduler: SchedulerServer<LogicalPlanNode, PhysicalPlanNode> =
+            SchedulerServer::new(
+                "localhost:50050".to_owned(),
+                test_cluster_context(),
+                BallistaCodec::default(),
+                Arc::new(config),
+                default_metrics_collector().unwrap(),
+            );
+        scheduler.init().await?;
+
+        let request = Request::new(PollWorkParams {
+            metadata: Some(exec_meta_with_version(0)),
+            num_free_vcores: 0,
+            task_status: vec![],
+        });
+        let status = scheduler
+            .poll_work(request)
+            .await
+            .expect_err("poll_work with mismatched version should be rejected");
+        assert_eq!(status.code(), tonic::Code::FailedPrecondition);
+        Ok(())
+    }
+
     #[tokio::test]
     #[ignore]
     async fn test_expired_executor() -> Result<(), BallistaError> {
@@ -1115,10 +1300,9 @@ mod test {
             host: Some("http://localhost:8080".to_owned()),
             port: 0,
             grpc_port: 0,
-            specification: Some(
-                ExecutorSpecification::default().with_task_slots(2).into(),
-            ),
+            specification: Some(ExecutorSpecification::default().with_vcores(2).into()),
             os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+            ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
         };
 
         let request: Request<RegisterExecutorParams> =
@@ -1144,7 +1328,7 @@ mod test {
 
         assert_eq!(stored_executor.grpc_port, 0);
         assert_eq!(stored_executor.port, 0);
-        assert_eq!(stored_executor.specification.task_slots, 2);
+        assert_eq!(stored_executor.specification.vcores, 2);
         assert_eq!(stored_executor.host, "http://localhost:8080".to_owned());
 
         // heartbeat from the executor
@@ -1204,10 +1388,9 @@ mod test {
             host: Some("http://localhost:8080".to_owned()),
             port: 0,
             grpc_port: 0,
-            specification: Some(
-                ExecutorSpecification::default().with_task_slots(2).into(),
-            ),
+            specification: Some(ExecutorSpecification::default().with_vcores(2).into()),
             os_info: Some(ExecutorOperatingSystemSpecification::default().into()),
+            ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
         };
 
         let request: Request<RegisterExecutorParams> =
@@ -1233,7 +1416,7 @@ mod test {
 
         assert_eq!(stored_executor.grpc_port, 0);
         assert_eq!(stored_executor.port, 0);
-        assert_eq!(stored_executor.specification.task_slots, 2);
+        assert_eq!(stored_executor.specification.vcores, 2);
         assert_eq!(stored_executor.host, "http://localhost:8080".to_owned());
 
         // Context strictly used for values-based query serialization to avoid
diff --git a/ballista/scheduler/src/scheduler_server/mod.rs b/ballista/scheduler/src/scheduler_server/mod.rs
index 17b955a98c..66ed01b505 100644
--- a/ballista/scheduler/src/scheduler_server/mod.rs
+++ b/ballista/scheduler/src/scheduler_server/mod.rs
@@ -26,6 +26,7 @@ use ballista_core::{JobId, JobStatusSubscriber};
 
 use datafusion::execution::context::SessionState;
 use datafusion::logical_expr::LogicalPlan;
+use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::{SessionConfig, SessionContext};
 use datafusion_proto::logical_plan::AsLogicalPlan;
 use datafusion_proto::physical_plan::AsExecutionPlan;
@@ -36,7 +37,7 @@ use crate::metrics::SchedulerMetricsCollector;
 use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata};
 use log::{debug, error, warn};
 
-use crate::scheduler_server::event::QueryStageSchedulerEvent;
+use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan};
 use crate::scheduler_server::query_stage_scheduler::QueryStageScheduler;
 
 use crate::state::executor_manager::ExecutorManager;
@@ -181,6 +182,15 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
     pub fn running_job_number(&self) -> usize {
         self.state.task_manager.running_job_number()
     }
+
+    /// True when at least `min_ready_executors` executors currently have
+    /// live heartbeats. Embedders can call this from their own health/readiness
+    /// handler and AND it with whatever app-specific state they track. The
+    /// built-in `/readyz` endpoint (see `api::health`) reports the same value.
+    pub fn is_ready(&self) -> bool {
+        let alive = self.state.executor_manager.get_alive_executors().len();
+        alive >= self.state.config.min_ready_executors
+    }
     #[cfg(feature = "rest-api")]
     pub(crate) fn metrics_collector(&self) -> &dyn SchedulerMetricsCollector {
         self.query_stage_scheduler.metrics_collector()
@@ -217,12 +227,12 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
         Ok(())
     }
 
-    /// Submits a job to executor returning job_id
-    pub async fn submit_job(
+    /// Shared entry point for logical and physical submissions.
+    pub async fn submit_plan(
         &self,
         job_name: &str,
         ctx: Arc<SessionContext>,
-        plan: &LogicalPlan,
+        plan: &SubmitPlan,
         subscriber: Option<JobStatusSubscriber>,
     ) -> Result<JobId> {
         log::debug!("Received submit request for job {job_name}");
@@ -242,6 +252,37 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
         Ok(job_id)
     }
 
+    /// Submit a logical plan for distributed execution, returning the job id.
+    pub async fn submit_job(
+        &self,
+        job_name: &str,
+        ctx: Arc<SessionContext>,
+        plan: &LogicalPlan,
+        subscriber: Option<JobStatusSubscriber>,
+    ) -> Result<JobId> {
+        self.submit_plan(
+            job_name,
+            ctx,
+            &SubmitPlan::Logical(plan.clone()),
+            subscriber,
+        )
+        .await
+    }
+
+    /// Submit an already-built physical plan for distributed execution. The physical
+    /// plan is planned with the static distributed planner; adaptive query planning
+    /// (AQE) is not available for this path.
+    pub async fn submit_physical_plan(
+        &self,
+        job_name: &str,
+        ctx: Arc<SessionContext>,
+        plan: Arc<dyn ExecutionPlan>,
+        subscriber: Option<JobStatusSubscriber>,
+    ) -> Result<JobId> {
+        self.submit_plan(job_name, ctx, &SubmitPlan::Physical(plan), subscriber)
+            .await
+    }
+
     /// It just send task status update event to the channel,
     /// and will not guarantee the event processing completed after return
     pub(crate) async fn update_task_status(
@@ -371,8 +412,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
     async fn do_register_executor(&self, metadata: ExecutorMetadata) -> Result<()> {
         let executor_data = ExecutorData {
             executor_id: metadata.id.clone(),
-            total_task_slots: metadata.specification.task_slots,
-            available_task_slots: metadata.specification.task_slots,
+            total_vcores: metadata.specification.vcores,
+            available_vcores: metadata.specification.vcores,
         };
 
         // Save the executor to state
@@ -451,11 +492,11 @@ mod test {
         let plan = test_plan();
         // this test will fail when AQE scheduling is used.
         // as AQE will fold plan due to empty scan
-        let task_slots = 4;
+        let total_vcores = 4;
 
         let scheduler = test_scheduler(TaskSchedulingPolicy::PullStaged).await?;
 
-        let executors = test_executors(task_slots);
+        let executors = test_executors(total_vcores);
         for (executor_metadata, executor_data) in executors {
             scheduler
                 .state
@@ -465,7 +506,7 @@ mod test {
         }
 
         let config =
-            SessionConfig::new_with_ballista().with_target_partitions(task_slots);
+            SessionConfig::new_with_ballista().with_target_partitions(total_vcores);
 
         let ctx = scheduler
             .state
@@ -484,6 +525,7 @@ mod test {
         // Submit job
         scheduler
             .state
+            .task_manager
             .submit_job(&job_id, "", ctx, &plan, 0, None)
             .await
             .expect("submitting plan");
@@ -516,11 +558,10 @@ mod test {
 
                 // Complete the task
                 let task_status = TaskStatus {
-                    task_id: task.task_id as u32,
-                    job_id: task.partition.job_id.clone().into(),
-                    stage_id: task.partition.stage_id as u32,
+                    task_id: task.key.task_id as u32,
+                    job_id: task.key.job_id.clone().into(),
+                    stage_id: task.key.stage_id as u32,
                     stage_attempt_num: task.stage_attempt_num as u32,
-                    partition_id: task.partition.partition_id as u32,
                     launch_time: 0,
                     start_exec_time: 0,
                     end_exec_time: 0,
@@ -557,6 +598,60 @@ mod test {
         Ok(())
     }
 
+    // Verify a job can be submitted as an already-built physical plan.
+    #[tokio::test]
+    async fn test_submit_physical_plan() -> Result<()> {
+        let logical_plan = test_plan();
+        let total_vcores = 4;
+
+        let scheduler = test_scheduler(TaskSchedulingPolicy::PullStaged).await?;
+
+        let executors = test_executors(total_vcores);
+        for (executor_metadata, executor_data) in executors {
+            scheduler
+                .state
+                .executor_manager
+                .register_executor(executor_metadata, executor_data)
+                .await?;
+        }
+
+        let config =
+            SessionConfig::new_with_ballista().with_target_partitions(total_vcores);
+
+        let ctx = scheduler
+            .state
+            .session_manager
+            .create_or_update_session("session_id", &config)
+            .await?;
+
+        // Plan the logical plan into a physical plan and submit it directly.
+        let physical_plan = ctx.state().create_physical_plan(&logical_plan).await?;
+
+        let job_id: JobId = "physical_job".into();
+
+        scheduler
+            .state
+            .task_manager
+            .queue_job(&job_id, "", timestamp_millis())?;
+
+        scheduler
+            .state
+            .task_manager
+            .submit_physical_plan(&job_id, "", ctx, physical_plan, 0, None)
+            .await
+            .expect("submitting physical plan");
+
+        assert!(
+            scheduler
+                .state
+                .task_manager
+                .get_active_execution_graph(&job_id)
+                .is_some()
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_push_scheduling() -> Result<()> {
         let plan = test_plan();
@@ -661,19 +756,13 @@ mod test {
             |_executor_id: String, task: MultiTaskDefinition| {
                 let mut statuses = vec![];
 
-                for TaskId {
-                    task_id,
-                    partition_id,
-                    ..
-                } in task.task_ids
-                {
+                for TaskId { task_id, .. } in task.task_ids {
                     let timestamp = timestamp_millis();
                     statuses.push(TaskStatus {
                         task_id,
                         job_id: task.job_id.clone(),
                         stage_id: task.stage_id,
                         stage_attempt_num: task.stage_attempt_num,
-                        partition_id,
                         launch_time: timestamp,
                         start_exec_time: timestamp,
                         end_exec_time: timestamp,
@@ -737,19 +826,13 @@ mod test {
             |_executor_id: String, task: MultiTaskDefinition| {
                 let mut statuses = vec![];
 
-                for TaskId {
-                    task_id,
-                    partition_id,
-                    ..
-                } in task.task_ids
-                {
+                for TaskId { task_id, .. } in task.task_ids {
                     let timestamp = timestamp_millis();
                     statuses.push(TaskStatus {
                         task_id,
                         job_id: task.job_id.clone(),
                         stage_id: task.stage_id,
                         stage_attempt_num: task.stage_attempt_num,
-                        partition_id,
                         launch_time: timestamp,
                         start_exec_time: timestamp,
                         end_exec_time: timestamp,
@@ -950,7 +1033,7 @@ mod test {
     }
 
     fn test_executors(num_partitions: usize) -> Vec<(ExecutorMetadata, ExecutorData)> {
-        let task_slots = (num_partitions as u32).div_ceil(2);
+        let vcores = (num_partitions as u32).div_ceil(2);
 
         vec![
             (
@@ -959,14 +1042,13 @@ mod test {
                     host: "localhost1".to_string(),
                     port: 8080,
                     grpc_port: 9090,
-                    specification: ExecutorSpecification::default()
-                        .with_task_slots(task_slots),
+                    specification: ExecutorSpecification::default().with_vcores(vcores),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 ExecutorData {
                     executor_id: "executor-1".to_owned(),
-                    total_task_slots: task_slots,
-                    available_task_slots: task_slots,
+                    total_vcores: vcores,
+                    available_vcores: vcores,
                 },
             ),
             (
@@ -976,13 +1058,13 @@ mod test {
                     port: 8080,
                     grpc_port: 9090,
                     specification: ExecutorSpecification::default()
-                        .with_task_slots(num_partitions as u32 - task_slots),
+                        .with_vcores(num_partitions as u32 - vcores),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 ExecutorData {
                     executor_id: "executor-2".to_owned(),
-                    total_task_slots: num_partitions as u32 - task_slots,
-                    available_task_slots: num_partitions as u32 - task_slots,
+                    total_vcores: num_partitions as u32 - vcores,
+                    available_vcores: num_partitions as u32 - vcores,
                 },
             ),
         ]
diff --git a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs
index 0d84950bc7..ee5174853a 100644
--- a/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs
+++ b/ballista/scheduler/src/scheduler_server/query_stage_scheduler.rs
@@ -18,6 +18,7 @@
 use std::sync::Arc;
 use std::time::Duration;
 
+use ballista_core::JobId;
 use ballista_core::serde::protobuf::{FailedJob, JobStatus};
 use log::{debug, error, info, trace, warn};
 
@@ -58,6 +59,22 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> QueryStageSchedul
             config,
         }
     }
+
+    async fn abort_job(&self, job_id: &JobId, failure_reason: String) -> Result<()> {
+        let executor_manager = self.state.executor_manager.clone();
+        self.state
+            .task_manager
+            .abort_job(job_id, failure_reason, move |running_tasks| async move {
+                if running_tasks.is_empty() {
+                    Ok(())
+                } else {
+                    executor_manager.cancel_running_tasks(running_tasks).await
+                }
+            })
+            .await?;
+        Ok(())
+    }
+
     #[cfg(feature = "rest-api")]
     pub(crate) fn metrics_collector(&self) -> &dyn SchedulerMetricsCollector {
         self.metrics_collector.as_ref()
@@ -146,7 +163,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
                             }
                         }
 
-                        error!("{}", &fail_message);
+                        error!("{}", fail_message);
                         QueryStageSchedulerEvent::JobPlanningFailed {
                             job_id,
                             fail_message,
@@ -211,10 +228,18 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
                     .record_completed(&job_id, queued_at, completed_at);
 
                 info!("Job finished successfully: [{job_id}]");
-                if let Err(e) = self.state.task_manager.succeed_job(&job_id).await {
-                    error!("Fail to invoke succeed_job for job {job_id} due to {e:?}");
-                }
-                self.state.clean_up_successful_job(job_id);
+                let intermediate_stage_ids =
+                    match self.state.task_manager.succeed_job(&job_id).await {
+                        Ok(ids) => ids,
+                        Err(e) => {
+                            error!(
+                                "Fail to invoke succeed_job for job {job_id} due to {e:?}"
+                            );
+                            vec![]
+                        }
+                    };
+                self.state
+                    .clean_up_successful_job(job_id, intermediate_stage_ids);
             }
             QueryStageSchedulerEvent::JobRunningFailed {
                 job_id,
@@ -226,24 +251,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
                     .record_failed(&job_id, queued_at, failed_at);
 
                 error!("Job failed: [{job_id}]");
-                match self
-                    .state
-                    .task_manager
-                    .abort_job(&job_id, fail_message)
-                    .await
-                {
-                    Ok((running_tasks, _pending_tasks)) => {
-                        if !running_tasks.is_empty() {
-                            event_sender
-                                .post_event(QueryStageSchedulerEvent::CancelTasks(
-                                    running_tasks,
-                                ))
-                                .await?;
-                        }
-                    }
-                    Err(e) => {
-                        error!("Fail to invoke abort_job for job {job_id} due to {e:?}");
-                    }
+                if let Err(e) = self.abort_job(&job_id, fail_message).await {
+                    error!("Fail to abort job {job_id} due to {e:?}");
                 }
                 self.state.clean_up_failed_job(job_id);
             }
@@ -257,17 +266,8 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
                 self.metrics_collector.record_cancelled(&job_id);
 
                 info!("Job cancelled: [{job_id}]");
-                match self.state.task_manager.cancel_job(&job_id).await {
-                    Ok((running_tasks, _pending_tasks)) => {
-                        event_sender
-                            .post_event(QueryStageSchedulerEvent::CancelTasks(
-                                running_tasks,
-                            ))
-                            .await?;
-                    }
-                    Err(e) => {
-                        error!("Fail to invoke cancel_job for job {job_id} due to {e:?}");
-                    }
+                if let Err(e) = self.abort_job(&job_id, "Cancelled".to_owned()).await {
+                    error!("Fail to cancel job {job_id} due to {e:?}");
                 }
                 self.state.clean_up_failed_job(job_id);
             }
@@ -278,9 +278,19 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
 
                 let num_status = tasks_status.len();
                 if self.state.config.is_push_staged_scheduling() {
+                    // Refund the vcores each completing task consumed at bind
+                    // time (see `bind_one` in `cluster/mod.rs`). Refunding
+                    // one vcore per task would leak leftovers under the
+                    // multi-partition-task model, draining executor budgets
+                    // to 1 vcore over the course of a query.
+                    let vcores_freed = self
+                        .state
+                        .task_manager
+                        .sum_vcores_for_statuses(&tasks_status)
+                        .await;
                     self.state
                         .executor_manager
-                        .unbind_tasks(vec![(executor_id.clone(), num_status as u32)])
+                        .unbind_tasks(vec![(executor_id.clone(), vcores_freed)])
                         .await?;
                 }
                 match self
diff --git a/ballista/scheduler/src/state/aqe/adapter.rs b/ballista/scheduler/src/state/aqe/adapter.rs
index f751d81b97..2040982341 100644
--- a/ballista/scheduler/src/state/aqe/adapter.rs
+++ b/ballista/scheduler/src/state/aqe/adapter.rs
@@ -45,7 +45,7 @@ impl BallistaAdapter {
         &mut self,
         plan: Arc<dyn ExecutionPlan>,
     ) -> datafusion::error::Result<Transformed<Arc<dyn ExecutionPlan>>> {
-        if let Some(exchange) = plan.as_any().downcast_ref::<ExchangeExec>() {
+        if let Some(exchange) = plan.downcast_ref::<ExchangeExec>() {
             let schema = exchange.schema().clone();
             let partitions = exchange.shuffle_partitions().ok_or_else(|| {
                 DataFusionError::Execution(
@@ -119,7 +119,7 @@ impl BallistaAdapter {
         job_id: &JobId,
         config: &ConfigOptions,
     ) -> datafusion::error::Result<AdaptiveStageInfo> {
-        if let Some(root) = plan.as_any().downcast_ref::<ExchangeExec>() {
+        if let Some(root) = plan.downcast_ref::<ExchangeExec>() {
             let mut adapter = BallistaAdapter::default();
             let plan = root
                 .input()
@@ -146,8 +146,7 @@ impl BallistaAdapter {
                 plan: writer,
                 inputs: adapter.inputs,
             })
-        } else if let Some(root) = plan.as_any().downcast_ref::<AdaptiveDatafusionExec>()
-        {
+        } else if let Some(root) = plan.downcast_ref::<AdaptiveDatafusionExec>() {
             let mut adapter = BallistaAdapter::default();
             let plan = root
                 .input()
diff --git a/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs b/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs
index 1a1c3962c4..9bc7451ae7 100644
--- a/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs
+++ b/ballista/scheduler/src/state/aqe/execution_plan/adaptive.rs
@@ -22,7 +22,6 @@ use datafusion::{
     physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties},
 };
 use parking_lot::Mutex;
-use std::any::Any;
 use std::fmt::Formatter;
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, atomic::AtomicI64};
@@ -126,10 +125,6 @@ impl ExecutionPlan for AdaptiveDatafusionExec {
         "AdaptiveDatafusionExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.input.properties()
     }
diff --git a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs
index 3c7853c26c..9391709ea6 100644
--- a/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs
+++ b/ballista/scheduler/src/state/aqe/execution_plan/dynamic_join.rs
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use ballista_core::config::BallistaConfig;
 use datafusion::{
     arrow::compute::SortOptions,
     common::{JoinType, NullEquality, Result, exec_err, internal_err},
@@ -32,7 +33,9 @@ use datafusion::{
 use log::debug;
 use std::sync::Arc;
 
+use crate::physical_optimizer::join_selection::collect_left_broadcast_safe;
 use crate::state::aqe::execution_plan::ExchangeExec;
+use crate::state::aqe::optimizer_rule::join_selection::SelectJoinRule;
 
 /// has children of this join been
 /// repartitioned
@@ -65,10 +68,6 @@ impl ExecutionPlan for DynamicJoinSelectionExec {
         "DynamicJoinSelectionExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -216,25 +215,58 @@ impl DynamicJoinSelectionExec {
         config: &ConfigOptions,
     ) -> Result<JoinSelectionAction> {
         let prefer_hash_join = config.optimizer.prefer_hash_join;
-        let threshold_collect_left_join_bytes =
-            config.optimizer.hash_join_single_partition_threshold;
-        let threshold_collect_left_join_rows =
-            config.optimizer.hash_join_single_partition_threshold_rows;
-
-        let partition_mode = if Self::supports_collect_by_thresholds(
+        // Both broadcast thresholds come from Ballista's own config so that a
+        // single set of keys governs broadcast selection under both the static
+        // planner (`maybe_promote_to_broadcast`) and AQE. A byte threshold of 0
+        // disables broadcast promotion entirely, matching the static planner.
+        // The row threshold is only consulted as a fallback when byte-size
+        // statistics are absent.
+        let bc = config
+            .extensions
+            .get::<BallistaConfig>()
+            .cloned()
+            .unwrap_or_default();
+        let threshold_collect_left_join_bytes = bc.broadcast_join_threshold_bytes();
+        let threshold_collect_left_join_rows = bc.broadcast_join_threshold_rows();
+
+        // The `!= 0` guard sits here rather than inside
+        // `supports_collect_by_thresholds`: that helper falls back to the row
+        // check when the byte threshold is 0/absent, so gating at this level is
+        // what makes a 0 threshold disable both the byte and row broadcast paths.
+        let under_threshold = threshold_collect_left_join_bytes != 0
+            && (Self::supports_collect_by_thresholds(
+                self.left.as_ref(),
+                threshold_collect_left_join_bytes,
+                threshold_collect_left_join_rows,
+            ) || Self::supports_collect_by_thresholds(
+                self.right.as_ref(),
+                threshold_collect_left_join_bytes,
+                threshold_collect_left_join_rows,
+            ));
+
+        // The resolver collects the smaller side onto the build (left) input,
+        // swapping the join type when the right side is the smaller one (the
+        // `swap_inputs` calls in `SelectJoinRule`). A `CollectLeft` join then
+        // broadcasts that build side to every probe task, which is only correct
+        // for join types that never emit rows on behalf of the build side. Use
+        // the same swap decision as the resolver to determine the resulting join
+        // type and skip the broadcast when it would be unsafe.
+        let build_side_join_type = if SelectJoinRule::supports_swap_join_order(
             self.left.as_ref(),
-            threshold_collect_left_join_bytes,
-            threshold_collect_left_join_rows,
-        ) || Self::supports_collect_by_thresholds(
             self.right.as_ref(),
-            threshold_collect_left_join_bytes,
-            threshold_collect_left_join_rows,
-        ) {
-            PartitionMode::CollectLeft
+        )? {
+            self.join_type.swap()
         } else {
-            PartitionMode::Partitioned
+            self.join_type
         };
 
+        let partition_mode =
+            if under_threshold && collect_left_broadcast_safe(build_side_join_type) {
+                PartitionMode::CollectLeft
+            } else {
+                PartitionMode::Partitioned
+            };
+
         let stats_left = self.left.partition_statistics(None)?;
         let stats_right = self.right.partition_statistics(None)?;
 
@@ -383,16 +415,14 @@ impl DynamicJoinSelectionExec {
     /// its children.
     pub(crate) fn upstream_resolved(&self) -> bool {
         fn has_join_or_unresolved_exchange(plan: &Arc<dyn ExecutionPlan>) -> bool {
-            if let Some(exchange) = plan.as_any().downcast_ref::<ExchangeExec>() {
+            if let Some(exchange) = plan.downcast_ref::<ExchangeExec>() {
                 // this should be fine, once we find first resolved
                 // exchange it should not have any unresolved shuffles later
                 // nor dynamic joins
                 return !exchange.shuffle_created();
             };
 
-            plan.as_any()
-                .downcast_ref::<DynamicJoinSelectionExec>()
-                .is_some()
+            plan.downcast_ref::<DynamicJoinSelectionExec>().is_some()
                 || plan
                     .children()
                     .iter()
@@ -430,12 +460,250 @@ mod tests {
     use super::*;
     use crate::state::aqe::execution_plan::ExchangeExec;
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::common::{ColumnStatistics, Statistics, stats::Precision};
+    use datafusion::config::ExtensionOptions;
     use datafusion::physical_plan::empty::EmptyExec;
+    use datafusion::physical_plan::expressions::Column;
+    use datafusion::physical_plan::test::exec::StatisticsExec;
 
     fn test_schema() -> Arc<Schema> {
         Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]))
     }
 
+    fn stats_exec(num_rows: usize) -> Arc<dyn ExecutionPlan> {
+        Arc::new(StatisticsExec::new(
+            Statistics {
+                num_rows: Precision::Inexact(num_rows),
+                total_byte_size: Precision::Inexact(num_rows * 16),
+                column_statistics: vec![ColumnStatistics::new_unknown()],
+            },
+            Schema::new(vec![Field::new("k", DataType::Int32, false)]),
+        ))
+    }
+
+    /// A source that reports a row count but no byte-size statistic, so the
+    /// resolver must fall back to the row-count broadcast threshold.
+    fn stats_exec_rows_only(num_rows: usize) -> Arc<dyn ExecutionPlan> {
+        Arc::new(StatisticsExec::new(
+            Statistics {
+                num_rows: Precision::Inexact(num_rows),
+                total_byte_size: Precision::Absent,
+                column_statistics: vec![ColumnStatistics::new_unknown()],
+            },
+            Schema::new(vec![Field::new("k", DataType::Int32, false)]),
+        ))
+    }
+
+    /// Core driver: runs the join-strategy decision for the given children with
+    /// explicit Ballista broadcast thresholds.
+    fn run_to_actual_join(
+        left: Arc<dyn ExecutionPlan>,
+        right: Arc<dyn ExecutionPlan>,
+        join_type: JoinType,
+        prefer_hash_join: bool,
+        broadcast_threshold_bytes: usize,
+        broadcast_threshold_rows: usize,
+    ) -> JoinSelectionAction {
+        let on: JoinOn =
+            vec![(Arc::new(Column::new("k", 0)), Arc::new(Column::new("k", 0)))];
+        let dj = DynamicJoinSelectionExec {
+            properties: Arc::clone(left.properties()),
+            left,
+            right,
+            on,
+            filter: None,
+            join_type,
+            projection: None,
+            null_equality: NullEquality::NullEqualsNothing,
+            selection_state: JoinInputState::Unknown,
+            null_aware: false,
+            plan_id: 0,
+        };
+        let mut config = ConfigOptions::new();
+        config.optimizer.prefer_hash_join = prefer_hash_join;
+        let mut bc = BallistaConfig::default();
+        bc.set(
+            "optimizer.broadcast_join_threshold_bytes",
+            &broadcast_threshold_bytes.to_string(),
+        )
+        .unwrap();
+        bc.set(
+            "optimizer.broadcast_join_threshold_rows",
+            &broadcast_threshold_rows.to_string(),
+        )
+        .unwrap();
+        config.extensions.insert(bc);
+        dj.to_actual_join(&config).unwrap()
+    }
+
+    /// Run the resolver's join-strategy decision for `join_type` with `left_rows`
+    /// and `right_rows` on the two sides, both small enough to be collectable.
+    fn actual_join_for(
+        join_type: JoinType,
+        left_rows: usize,
+        right_rows: usize,
+        prefer_hash_join: bool,
+    ) -> JoinSelectionAction {
+        // 10 MiB is large enough for the tiny StatisticsExec sides below to be
+        // collectable, matching the default broadcast threshold.
+        actual_join_with_threshold(
+            join_type,
+            left_rows,
+            right_rows,
+            prefer_hash_join,
+            10 * 1024 * 1024,
+        )
+    }
+
+    /// Same as [`actual_join_for`] but with an explicit
+    /// `broadcast_join_threshold_bytes`, so tests can prove the Ballista config
+    /// drives the byte cutoff (0 disables broadcast promotion).
+    fn actual_join_with_threshold(
+        join_type: JoinType,
+        left_rows: usize,
+        right_rows: usize,
+        prefer_hash_join: bool,
+        broadcast_threshold_bytes: usize,
+    ) -> JoinSelectionAction {
+        run_to_actual_join(
+            stats_exec(left_rows),
+            stats_exec(right_rows),
+            join_type,
+            prefer_hash_join,
+            broadcast_threshold_bytes,
+            // Large row threshold: byte stats are present here, so the row path
+            // is never consulted; keep it out of the way of the byte decision.
+            1_000_000,
+        )
+    }
+
+    fn is_collected(action: &JoinSelectionAction) -> bool {
+        matches!(
+            action,
+            JoinSelectionAction::CollectLeft(_) | JoinSelectionAction::LateCollectLeft(_)
+        )
+    }
+
+    // With equal-sized sides (no swap) the resolver must collect a small build
+    // side into a CollectLeft broadcast only for join types whose output is
+    // driven by the probe side; everything that emits build-side rows must be
+    // repartitioned instead. `prefer_hash_join` only selects the repartitioned
+    // fallback (hash vs sort-merge), so the broadcast-safety decision must be
+    // identical under both settings.
+    #[test]
+    fn to_actual_join_collects_only_broadcast_safe_join_types() {
+        for prefer_hash_join in [true, false] {
+            for join_type in [
+                JoinType::Inner,
+                JoinType::Left,
+                JoinType::Right,
+                JoinType::Full,
+                JoinType::LeftSemi,
+                JoinType::RightSemi,
+                JoinType::LeftAnti,
+                JoinType::RightAnti,
+                JoinType::LeftMark,
+                JoinType::RightMark,
+            ] {
+                let action = actual_join_for(join_type, 100, 100, prefer_hash_join);
+                assert_eq!(
+                    is_collected(&action),
+                    collect_left_broadcast_safe(join_type),
+                    "join_type {join_type:?} (prefer_hash_join={prefer_hash_join}): collected={}, expected safe={}",
+                    is_collected(&action),
+                    collect_left_broadcast_safe(join_type),
+                );
+            }
+        }
+    }
+
+    // Safety is evaluated on the join type *after* the build-side swap. A LEFT
+    // join with the smaller side on the right swaps to a (safe) RIGHT join and is
+    // collected; a RIGHT join with the smaller side on the right swaps to an
+    // (unsafe) LEFT join and must be repartitioned.
+    #[test]
+    fn to_actual_join_uses_post_swap_join_type() {
+        assert!(
+            is_collected(&actual_join_for(JoinType::Left, 1000, 10, true)),
+            "LEFT with small right swaps to a safe RIGHT join and should collect"
+        );
+        assert!(
+            !is_collected(&actual_join_for(JoinType::Right, 1000, 10, true)),
+            "RIGHT with small right swaps to an unsafe LEFT join and must repartition"
+        );
+    }
+
+    // The byte cutoff comes from `ballista.optimizer.broadcast_join_threshold_bytes`,
+    // not DataFusion's `hash_join_single_partition_threshold`. `stats_exec(n)`
+    // reports `n * 16` bytes, so the smaller side here is 1600 bytes. A threshold
+    // above it collects; a threshold below it repartitions.
+    #[test]
+    fn broadcast_threshold_bytes_drives_collect_decision() {
+        // smaller (left) side = 100 * 16 = 1600 bytes, larger = 16000 bytes
+        assert!(
+            is_collected(&actual_join_with_threshold(
+                JoinType::Inner,
+                100,
+                1000,
+                true,
+                2000,
+            )),
+            "smaller side (1600 B) is under a 2000 B threshold and must collect"
+        );
+        assert!(
+            !is_collected(&actual_join_with_threshold(
+                JoinType::Inner,
+                100,
+                1000,
+                true,
+                1000,
+            )),
+            "neither side is under a 1000 B threshold, so the join must repartition"
+        );
+    }
+
+    // When byte-size statistics are absent, the decision falls back to the
+    // Ballista row-count threshold (`broadcast_join_threshold_rows`), not
+    // DataFusion's `hash_join_single_partition_threshold_rows`.
+    #[test]
+    fn broadcast_threshold_rows_drives_fallback_decision() {
+        // Byte-size is Absent on both sides, so only the row threshold applies.
+        // Smaller side = 100 rows. A generous byte threshold keeps the byte
+        // guard open so the row fallback is what decides.
+        assert!(
+            is_collected(&run_to_actual_join(
+                stats_exec_rows_only(100),
+                stats_exec_rows_only(1000),
+                JoinType::Inner,
+                true,
+                10 * 1024 * 1024,
+                200,
+            )),
+            "smaller side (100 rows) is under a 200-row threshold and must collect"
+        );
+        assert!(
+            !is_collected(&run_to_actual_join(
+                stats_exec_rows_only(100),
+                stats_exec_rows_only(1000),
+                JoinType::Inner,
+                true,
+                10 * 1024 * 1024,
+                50,
+            )),
+            "neither side is under a 50-row threshold, so the join must repartition"
+        );
+    }
+
+    // A threshold of 0 disables broadcast promotion entirely, matching the static
+    // planner, even for a side small enough to collect under any positive cutoff.
+    #[test]
+    fn zero_broadcast_threshold_disables_collect() {
+        assert!(
+            !is_collected(&actual_join_with_threshold(JoinType::Inner, 1, 1, true, 0,)),
+            "broadcast_join_threshold_bytes=0 must disable CollectLeft promotion"
+        );
+    }
+
     /// Constructs a minimal `DynamicJoinSelectionExec` around the given children.
     /// Properties are borrowed from the left child — correct enough for unit tests
     /// that only exercise `children_resolved`.
diff --git a/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs b/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs
index 28f1011e45..63cc181502 100644
--- a/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs
+++ b/ballista/scheduler/src/state/aqe/execution_plan/exchange.rs
@@ -29,7 +29,6 @@ use datafusion::{
 };
 use log::trace;
 use parking_lot::Mutex;
-use std::any::Any;
 use std::ops::Deref;
 use std::sync::{Arc, atomic::AtomicI64};
 
@@ -326,10 +325,6 @@ impl ExecutionPlan for ExchangeExec {
         "ExchangeExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -381,7 +376,7 @@ impl ExecutionPlan for ExchangeExec {
         ))
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         let schema = self.input.schema();
         match self.shuffle_partitions.lock().deref() {
             //
@@ -406,7 +401,7 @@ impl ExecutionPlan for ExchangeExec {
                         "shuffle reader at stage: {:?} and partition {} returned statistics: {:?}",
                         self.stage_id, idx, stat_for_partition
                     );
-                    stat_for_partition
+                    stat_for_partition.map(Arc::new)
                 } else {
                     let stats_for_partitions = stats_for_partitions(
                         schema.fields().len(),
@@ -419,10 +414,10 @@ impl ExecutionPlan for ExchangeExec {
                         "shuffle reader at stage: {:?} returned statistics for all partitions: {:?}",
                         self.stage_id, stats_for_partitions
                     );
-                    Ok(stats_for_partitions)
+                    Ok(Arc::new(stats_for_partitions))
                 }
             }
-            None => Ok(Statistics::new_unknown(&schema)),
+            None => Ok(Arc::new(Statistics::new_unknown(&schema))),
         }
     }
 }
diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs
index 74cedb1f40..deec100639 100644
--- a/ballista/scheduler/src/state/aqe/mod.rs
+++ b/ballista/scheduler/src/state/aqe/mod.rs
@@ -113,8 +113,6 @@ pub(crate) struct AdaptiveExecutionGraph {
 
     /// Locations of this `ExecutionGraph` final output locations
     output_locations: Vec<PartitionLocation>,
-    /// Task ID generator, generate unique TID in the execution graph
-    task_id_gen: usize,
     /// Failed stage attempts, record the failed stage attempts to limit the retry times.
     /// Map from Stage ID -> Set<Stage_ATTPMPT_NUM>
     failed_stage_attempts: HashMap<usize, HashSet<usize>>,
@@ -191,7 +189,6 @@ impl AdaptiveExecutionGraph {
             end_time: 0,
             stages,
             output_locations: vec![],
-            task_id_gen: 0,
             failed_stage_attempts: HashMap::new(),
             session_config,
             logical_plan,
@@ -218,13 +215,6 @@ impl AdaptiveExecutionGraph {
         Ok((stage_id, stage))
     }
 
-    #[cfg(test)]
-    fn next_task_id(&mut self) -> usize {
-        let new_tid = self.task_id_gen;
-        self.task_id_gen += 1;
-        new_tid
-    }
-
     /// Processing stage status update after task status changing
     fn processing_stages_update(
         &mut self,
@@ -656,28 +646,25 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                             );
                             continue;
                         }
-                        let partition_id = task_status.clone().partition_id as usize;
+                        let task_id = task_status.task_id as usize;
                         let task_identity = format!(
-                            "TID {} {}/{}.{}/{}",
-                            task_status.task_id,
-                            job_id,
-                            stage_id,
-                            task_stage_attempt_num,
-                            partition_id
+                            "TID {}/{}.{}/{}",
+                            job_id, stage_id, task_stage_attempt_num, task_id
                         );
-                        let operator_metrics = task_status.metrics.clone();
 
-                        if !running_stage
-                            .update_task_info(partition_id, task_status.clone())
-                        {
+                        if !running_stage.update_task_info(task_id, task_status.clone()) {
                             continue;
                         }
+
+                        let TaskStatus {
+                            status,
+                            metrics: operator_metrics,
+                            ..
+                        } = task_status;
                         //
                         // handle task failure
                         //
-                        if let Some(task_status::Status::Failed(failed_task)) =
-                            task_status.status
-                        {
+                        if let Some(task_status::Status::Failed(failed_task)) = status {
                             let failed_reason = failed_task.failed_reason;
 
                             match failed_reason {
@@ -749,16 +736,16 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                                     if failed_task.retryable
                                         && failed_task.count_to_failures
                                     {
-                                        if running_stage.task_failure_number(partition_id)
+                                        if running_stage.task_failure_number(task_id)
                                             < max_task_failures
                                         {
                                             // TODO add new struct to track all the failed task infos
                                             // The failure TaskInfo is ignored and set to None here
-                                            running_stage.reset_task_info(partition_id);
+                                            running_stage.reset_task_info(task_id);
                                         } else {
                                             let error_msg = format!(
                                                 "Task {} in Stage {} failed {} times, fail the stage, most recent failure reason: {:?}",
-                                                partition_id,
+                                                task_id,
                                                 stage_id,
                                                 max_task_failures,
                                                 failed_task.error
@@ -769,12 +756,12 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                                     } else if failed_task.retryable {
                                         // TODO add new struct to track all the failed task infos
                                         // The failure TaskInfo is ignored and set to None here
-                                        running_stage.reset_task_info(partition_id);
+                                        running_stage.reset_task_info(task_id);
                                     }
                                 }
                                 None => {
                                     let error_msg = format!(
-                                        "Task {partition_id} in Stage {stage_id} failed with unknown failure reasons, fail the stage"
+                                        "Task {task_id} in Stage {stage_id} failed with unknown failure reasons, fail the stage"
                                     );
                                     error!("{error_msg}");
                                     failed_stages.insert(stage_id, error_msg);
@@ -786,16 +773,16 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                         //
                         else if let Some(task_status::Status::Successful(
                             successful_task,
-                        )) = task_status.status
+                        )) = status
                         {
-                            // update task metrics for successfu task
+                            // update task metrics for successful task
                             running_stage
-                                .update_task_metrics(partition_id, operator_metrics)?;
+                                .update_task_metrics(task_id, operator_metrics)?;
 
                             locations.append(
                                 &mut crate::state::execution_graph::partition_to_location(
                                     &job_id,
-                                    partition_id,
+                                    task_id,
                                     stage_id,
                                     executor,
                                     successful_task.partitions,
@@ -848,7 +835,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                         stage_id,
                         stage_task_statuses
                             .into_iter()
-                            .map(|task_status| task_status.partition_id)
+                            .map(|task_status| task_status.task_id)
                             .collect::<Vec<_>>(),
                     );
                 }
@@ -865,17 +852,22 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                 .insert(*stage_id, HashSet::from_iter(attempts.iter().copied()));
         }
 
-        for (stage_id, missing_parts) in &resubmit_successful_stages {
+        // Values coming out of `remove_input_partitions` are task_ids
+        // (append slots) under the multi-partition-task model (see the mirror
+        // site in `execution_graph.rs` for the TODO on partition-id semantics).
+        for (stage_id, missing_task_ids) in &resubmit_successful_stages {
             if let Some(stage) = self.stages.get_mut(stage_id) {
                 if let ExecutionStage::Successful(success_stage) = stage {
-                    for partition in missing_parts {
-                        if *partition > success_stage.partitions {
+                    for task_id in missing_task_ids {
+                        if *task_id >= success_stage.task_infos.len() {
                             return Err(BallistaError::Internal(format!(
-                                "Invalid partition ID {} in map stage {}",
-                                *partition, stage_id
+                                "Invalid task_id {} in map stage {} (task_infos has {} entries)",
+                                *task_id,
+                                stage_id,
+                                success_stage.task_infos.len()
                             )));
                         }
-                        let task_info = &mut success_stage.task_infos[*partition];
+                        let task_info = &mut success_stage.task_infos[*task_id];
                         // Update the task info to failed
                         task_info.task_status = task_status::Status::Failed(FailedTask {
                             error: "FetchPartitionError in parent stage".to_owned(),
@@ -896,17 +888,19 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
             }
         }
 
-        for (stage_id, missing_parts) in &reset_running_stages {
+        for (stage_id, missing_task_ids) in &reset_running_stages {
             if let Some(stage) = self.stages.get_mut(stage_id) {
                 if let ExecutionStage::Running(running_stage) = stage {
-                    for partition in missing_parts {
-                        if *partition > running_stage.partitions {
+                    for task_id in missing_task_ids {
+                        if *task_id >= running_stage.task_infos.len() {
                             return Err(BallistaError::Internal(format!(
-                                "Invalid partition ID {} in map stage {}",
-                                *partition, stage_id
+                                "Invalid task_id {} in map stage {} (task_infos has {} entries)",
+                                *task_id,
+                                stage_id,
+                                running_stage.task_infos.len()
                             )));
                         }
-                        running_stage.reset_task_info(*partition);
+                        running_stage.reset_task_info(*task_id);
                     }
                 } else {
                     warn!(
@@ -955,14 +949,11 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                     stage
                         .running_tasks()
                         .into_iter()
-                        .map(|(task_id, stage_id, partition_id, executor_id)| {
-                            RunningTaskInfo {
-                                task_id,
-                                job_id: self.job_id.clone(),
-                                stage_id,
-                                partition_id,
-                                executor_id,
-                            }
+                        .map(|(task_id, stage_id, executor_id)| RunningTaskInfo {
+                            task_id,
+                            job_id: self.job_id.clone(),
+                            stage_id,
+                            executor_id,
                         })
                         .collect::<Vec<RunningTaskInfo>>()
                 } else {
@@ -986,10 +977,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
             .sum()
     }
 
-    fn fetch_running_stage(
-        &mut self,
-        black_list: &[usize],
-    ) -> Option<(&mut RunningStage, &mut usize)> {
+    fn fetch_running_stage(&mut self, black_list: &[usize]) -> Option<&mut RunningStage> {
         if matches!(
             self.status,
             JobStatus {
@@ -1006,7 +994,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
             if let Some(ExecutionStage::Running(running_stage)) =
                 self.stages.get_mut(&running_stage_id)
             {
-                Some((running_stage, &mut self.task_id_gen))
+                Some(running_stage)
             } else {
                 warn!("Fail to find running stage with id {running_stage_id}");
                 None
@@ -1111,15 +1099,12 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
             let running_tasks = stage
                 .running_tasks()
                 .into_iter()
-                .map(
-                    |(task_id, stage_id, partition_id, executor_id)| RunningTaskInfo {
-                        task_id,
-                        job_id: self.job_id.clone(),
-                        stage_id,
-                        partition_id,
-                        executor_id,
-                    },
-                )
+                .map(|(task_id, stage_id, executor_id)| RunningTaskInfo {
+                    task_id,
+                    job_id: self.job_id.clone(),
+                    stage_id,
+                    executor_id,
+                })
                 .collect();
             self.stages.insert(
                 stage_id,
@@ -1253,19 +1238,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
         let job_id = self.job_id.clone();
         let session_id = self.session_id.clone();
 
-        let find_candidate = self.stages.iter().any(|(_stage_id, stage)| {
-            if let ExecutionStage::Running(stage) = stage {
-                stage.available_tasks() > 0
-            } else {
-                false
-            }
-        });
-        let next_task_id = if find_candidate {
-            Some(self.next_task_id())
-        } else {
-            None
-        };
-
         let mut next_task = self.stages.iter_mut().find(|(_stage_id, stage)| {
             if let ExecutionStage::Running(stage) = stage {
                 stage.available_tasks() > 0
@@ -1274,25 +1246,25 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
             }
         }).map(|(stage_id, stage)| {
             if let ExecutionStage::Running(stage) = stage {
-                use ballista_core::serde::scheduler::PartitionId;
-
-                let (partition_id, _) = stage
-                    .task_infos
+                use ballista_core::serde::scheduler::TaskKey;
+
+                // pop_next_task hands out a single-partition task — the
+                // bind path sized to `exec.vcores` lives in cluster/mod.rs.
+                let input_partition_ids = stage.pending.next_slice(1);
+                if input_partition_ids.is_empty() {
+                    return Err(BallistaError::Internal(format!(
+                        "Error getting next task for job {job_id}: Stage {stage_id} is ready but has no pending tasks"
+                    )));
+                }
+                // task_id is the append slot in `task_infos` — assigned as
+                // `task_infos.len()` at bind time. `(job_id, stage_id, task_id)`
+                // is globally unique.
+                let task_id = stage.task_infos.len();
+                let task_attempt = input_partition_ids
                     .iter()
-                    .enumerate()
-                    .find(|(_partition, info)| info.is_none())
-                    .ok_or_else(|| {
-                        BallistaError::Internal(format!("Error getting next task for job {job_id}: Stage {stage_id} is ready but has no pending tasks"))
-                    })?;
-
-                let partition = PartitionId {
-                    job_id,
-                    stage_id: *stage_id,
-                    partition_id,
-                };
-
-                let task_id = next_task_id.unwrap();
-                let task_attempt = stage.task_failure_numbers[partition_id];
+                    .map(|pid| stage.task_failure_numbers[*pid])
+                    .max()
+                    .unwrap_or(0);
                 let task_info = crate::state::execution_graph::TaskInfo {
                     task_id,
                     scheduled_time: timestamp_millis() as u128,
@@ -1304,17 +1276,25 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
                     task_status: task_status::Status::Running(ballista_core::serde::protobuf::RunningTask {
                         executor_id: executor_id.to_owned()
                     }),
+                    global_input_partition_ids: input_partition_ids.clone(),
+                    vcores_consumed: input_partition_ids.len() as u32,
                 };
+                stage.task_infos.push(task_info);
 
-                // Set the task info to Running for new task
-                stage.task_infos[partition_id] = Some(task_info);
+                let key = TaskKey {
+                    job_id,
+                    stage_id: *stage_id,
+                    task_id,
+                };
 
+                let vcores_consumed = input_partition_ids.len() as u32;
                 Ok(crate::state::execution_graph::TaskDescription {
                     session_id,
-                    partition,
+                    key,
                     stage_attempt_num: stage.stage_attempt_num,
-                    task_id,
                     task_attempt,
+                    global_input_partition_ids: input_partition_ids,
+                    vcores_consumed,
                     plan: stage.plan.clone(),
                     session_config: self.session_config.clone()
                 })
diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs
index a2b8f7b2e1..bef85f3fc4 100644
--- a/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs
+++ b/ballista/scheduler/src/state/aqe/optimizer_rule/chaos_exec.rs
@@ -177,7 +177,7 @@ mod tests {
             let mut count = 0usize;
             result
                 .apply(|node| {
-                    if node.as_any().downcast_ref::<ChaosExec>().is_some() {
+                    if node.downcast_ref::<ChaosExec>().is_some() {
                         count += 1;
                     }
                     Ok(TreeNodeRecursion::Continue)
@@ -193,7 +193,6 @@ mod tests {
         let plan = leaf_exec();
         let result = rule.optimize(plan, &chaos_config(0.42, 42)).unwrap();
         let chaos = result
-            .as_any()
             .downcast_ref::<ChaosExec>()
             .expect("single-node plan always wraps root");
         assert!(
diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs
index 4686fde0fd..5947be4c3d 100644
--- a/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs
+++ b/ballista/scheduler/src/state/aqe/optimizer_rule/coalesce_partitions.rs
@@ -155,7 +155,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule {
         );
 
         // Get the subtree below the root. Two root kinds, same outcome.
-        let input = if let Some(ex) = plan.as_any().downcast_ref::<ExchangeExec>() {
+        let input = if let Some(ex) = plan.downcast_ref::<ExchangeExec>() {
             debug!(
                 "[coalesce-rule] root=ExchangeExec plan_id={} stage_id={:?} stage_resolved={}",
                 ex.plan_id,
@@ -163,7 +163,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule {
                 ex.shuffle_partitions().is_some(),
             );
             ex.input().clone()
-        } else if let Some(adp) = plan.as_any().downcast_ref::<AdaptiveDatafusionExec>() {
+        } else if let Some(adp) = plan.downcast_ref::<AdaptiveDatafusionExec>() {
             debug!(
                 "[coalesce-rule] root=AdaptiveDatafusionExec stage_id={:?}",
                 adp.stage_id(),
@@ -182,7 +182,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule {
         // *this* stage's group, they belong to whatever stage wrote them.
         let mut leaves: Vec<Arc<dyn ExecutionPlan>> = Vec::new();
         input.apply(|node| {
-            if node.as_any().is::<ExchangeExec>() {
+            if node.is::<ExchangeExec>() {
                 leaves.push(node.clone());
                 Ok(TreeNodeRecursion::Jump)
             } else {
@@ -192,8 +192,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule {
 
         // Helper: downcast each Arc back to &ExchangeExec.
         fn as_exchange(arc: &Arc<dyn ExecutionPlan>) -> &ExchangeExec {
-            arc.as_any()
-                .downcast_ref::<ExchangeExec>()
+            arc.downcast_ref::<ExchangeExec>()
                 .expect("filtered to ExchangeExec above")
         }
 
diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs
index 7b60e1e43d..a07312d14e 100644
--- a/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs
+++ b/ballista/scheduler/src/state/aqe/optimizer_rule/distributed_exchange.rs
@@ -45,12 +45,9 @@ impl DistributedExchangeRule {
         &self,
         execution_plan: Arc<dyn ExecutionPlan>,
     ) -> datafusion::error::Result<Transformed<Arc<dyn ExecutionPlan>>> {
-        if let Some(coalesce) = execution_plan
-            .as_any()
-            .downcast_ref::<CoalescePartitionsExec>()
-        {
+        if let Some(coalesce) = execution_plan.downcast_ref::<CoalescePartitionsExec>() {
             let input = coalesce.input();
-            if input.as_any().downcast_ref::<ExchangeExec>().is_none()
+            if input.downcast_ref::<ExchangeExec>().is_none()
                 && !matches!(nearest_exchange_status(input), ExchangeStatus::Unresolved)
             {
                 let exchange_exec = ExchangeExec::new(
@@ -63,12 +60,11 @@ impl DistributedExchangeRule {
                     execution_plan.with_new_children(vec![Arc::new(exchange_exec)])?,
                 ));
             }
-        } else if let Some(sort_preserving_merge) = execution_plan
-            .as_any()
-            .downcast_ref::<SortPreservingMergeExec>(
-        ) {
+        } else if let Some(sort_preserving_merge) =
+            execution_plan.downcast_ref::<SortPreservingMergeExec>()
+        {
             let input = sort_preserving_merge.input();
-            if input.as_any().downcast_ref::<ExchangeExec>().is_none()
+            if input.downcast_ref::<ExchangeExec>().is_none()
                 && !matches!(nearest_exchange_status(input), ExchangeStatus::Unresolved)
             {
                 let exchange_exec = ExchangeExec::new(
@@ -81,8 +77,7 @@ impl DistributedExchangeRule {
                     execution_plan.with_new_children(vec![Arc::new(exchange_exec)])?,
                 ));
             }
-        } else if let Some(repartition) =
-            execution_plan.as_any().downcast_ref::<RepartitionExec>()
+        } else if let Some(repartition) = execution_plan.downcast_ref::<RepartitionExec>()
             && let execution_plan::Partitioning::Hash(_, _) = repartition.partitioning()
         {
             let input = repartition.input();
@@ -112,7 +107,6 @@ impl PhysicalOptimizerRule for DistributedExchangeRule {
 
         if result
             .data
-            .as_any()
             .downcast_ref::<AdaptiveDatafusionExec>()
             .is_some()
         {
@@ -143,7 +137,7 @@ impl PhysicalOptimizerRule for DistributedExchangeRule {
 /// (short-circuits), `Resolved` if every branch that has an exchange has a resolved
 /// one, and `None` if no exchange is found anywhere.
 fn nearest_exchange_status(plan: &Arc<dyn ExecutionPlan>) -> ExchangeStatus {
-    if let Some(exchange) = plan.as_any().downcast_ref::<ExchangeExec>() {
+    if let Some(exchange) = plan.downcast_ref::<ExchangeExec>() {
         if exchange.shuffle_created() && !exchange.inactive_stage {
             ExchangeStatus::Resolved
         } else {
@@ -241,25 +235,18 @@ mod tests {
 
         let result = rule.optimize(coalesce, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let coalesce_out = adaptive
             .input()
-            .as_any()
             .downcast_ref::<CoalescePartitionsExec>()
             .unwrap();
         let child = coalesce_out.children()[0];
         assert!(
-            child.as_any().downcast_ref::<ExchangeExec>().is_some(),
+            child.downcast_ref::<ExchangeExec>().is_some(),
             "direct child should remain ExchangeExec"
         );
         assert!(
-            child.children()[0]
-                .as_any()
-                .downcast_ref::<ExchangeExec>()
-                .is_none(),
+            child.children()[0].downcast_ref::<ExchangeExec>().is_none(),
             "ExchangeExec should not wrap another ExchangeExec"
         );
     }
@@ -277,18 +264,13 @@ mod tests {
 
         let result = rule.optimize(outer, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let outer_coalesce = adaptive
             .input()
-            .as_any()
             .downcast_ref::<CoalescePartitionsExec>()
             .unwrap();
         assert!(
             outer_coalesce.children()[0]
-                .as_any()
                 .downcast_ref::<ExchangeExec>()
                 .is_none(),
             "should not inject ExchangeExec when unresolved exchange is in subtree"
@@ -307,18 +289,13 @@ mod tests {
 
         let result = rule.optimize(outer, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let outer_coalesce = adaptive
             .input()
-            .as_any()
             .downcast_ref::<CoalescePartitionsExec>()
             .unwrap();
         assert!(
             outer_coalesce.children()[0]
-                .as_any()
                 .downcast_ref::<ExchangeExec>()
                 .is_some(),
             "should inject ExchangeExec when subtree only has resolved exchanges"
@@ -334,20 +311,13 @@ mod tests {
 
         let result = rule.optimize(input, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let spm = adaptive
             .input()
-            .as_any()
             .downcast_ref::<SortPreservingMergeExec>()
             .expect("child should be SortPreservingMergeExec");
         assert!(
-            spm.children()[0]
-                .as_any()
-                .downcast_ref::<ExchangeExec>()
-                .is_some(),
+            spm.children()[0].downcast_ref::<ExchangeExec>().is_some(),
             "SortPreservingMergeExec should have ExchangeExec injected as its child"
         );
     }
@@ -359,23 +329,14 @@ mod tests {
 
         let result = rule.optimize(input, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let spm = adaptive
             .input()
-            .as_any()
             .downcast_ref::<SortPreservingMergeExec>()
             .unwrap();
         let child = spm.children()[0];
-        assert!(child.as_any().downcast_ref::<ExchangeExec>().is_some());
-        assert!(
-            child.children()[0]
-                .as_any()
-                .downcast_ref::<ExchangeExec>()
-                .is_none()
-        );
+        assert!(child.downcast_ref::<ExchangeExec>().is_some());
+        assert!(child.children()[0].downcast_ref::<ExchangeExec>().is_none());
     }
 
     #[test]
@@ -388,20 +349,13 @@ mod tests {
 
         let result = rule.optimize(input, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let spm = adaptive
             .input()
-            .as_any()
             .downcast_ref::<SortPreservingMergeExec>()
             .unwrap();
         assert!(
-            spm.children()[0]
-                .as_any()
-                .downcast_ref::<ExchangeExec>()
-                .is_none(),
+            spm.children()[0].downcast_ref::<ExchangeExec>().is_none(),
             "should not inject ExchangeExec when unresolved exchange is in subtree"
         );
     }
@@ -419,13 +373,9 @@ mod tests {
 
         let result = rule.optimize(repartition, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         let exchange = adaptive
             .input()
-            .as_any()
             .downcast_ref::<ExchangeExec>()
             .expect("Hash RepartitionExec should be replaced with ExchangeExec");
         assert!(
@@ -444,16 +394,9 @@ mod tests {
 
         let result = rule.optimize(repartition, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         assert!(
-            adaptive
-                .input()
-                .as_any()
-                .downcast_ref::<RepartitionExec>()
-                .is_some(),
+            adaptive.input().downcast_ref::<RepartitionExec>().is_some(),
             "RoundRobin repartition should be kept as-is (not replaced)"
         );
     }
@@ -472,16 +415,9 @@ mod tests {
 
         let result = rule.optimize(repartition, &config()).unwrap();
 
-        let adaptive = result
-            .as_any()
-            .downcast_ref::<AdaptiveDatafusionExec>()
-            .unwrap();
+        let adaptive = result.downcast_ref::<AdaptiveDatafusionExec>().unwrap();
         assert!(
-            adaptive
-                .input()
-                .as_any()
-                .downcast_ref::<RepartitionExec>()
-                .is_some(),
+            adaptive.input().downcast_ref::<RepartitionExec>().is_some(),
             "Hash repartition should be kept when input has an unresolved exchange"
         );
     }
@@ -493,10 +429,7 @@ mod tests {
         let rule = DistributedExchangeRule::default();
         let result = rule.optimize(leaf_exec(), &config()).unwrap();
         assert!(
-            result
-                .as_any()
-                .downcast_ref::<AdaptiveDatafusionExec>()
-                .is_some(),
+            result.downcast_ref::<AdaptiveDatafusionExec>().is_some(),
             "optimize should always wrap the result in AdaptiveDatafusionExec"
         );
     }
@@ -510,13 +443,11 @@ mod tests {
         let result = rule.optimize(adaptive, &config()).unwrap();
 
         let outer = result
-            .as_any()
             .downcast_ref::<AdaptiveDatafusionExec>()
             .expect("result should be AdaptiveDatafusionExec");
         assert!(
             outer
                 .input()
-                .as_any()
                 .downcast_ref::<AdaptiveDatafusionExec>()
                 .is_none(),
             "existing AdaptiveDatafusionExec should not be wrapped in another one"
@@ -609,15 +540,12 @@ mod tests {
             )
             .unwrap();
         let exchange1 = result1
-            .as_any()
             .downcast_ref::<AdaptiveDatafusionExec>()
             .unwrap()
             .input()
-            .as_any()
             .downcast_ref::<CoalescePartitionsExec>()
             .unwrap()
             .children()[0]
-            .as_any()
             .downcast_ref::<ExchangeExec>()
             .unwrap();
         assert_eq!(
@@ -632,15 +560,12 @@ mod tests {
             )
             .unwrap();
         let exchange2 = result2
-            .as_any()
             .downcast_ref::<AdaptiveDatafusionExec>()
             .unwrap()
             .input()
-            .as_any()
             .downcast_ref::<CoalescePartitionsExec>()
             .unwrap()
             .children()[0]
-            .as_any()
             .downcast_ref::<ExchangeExec>()
             .unwrap();
         assert_eq!(
diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs
index 848b0ca0ec..53f44dae61 100644
--- a/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs
+++ b/ballista/scheduler/src/state/aqe/optimizer_rule/join_selection.rs
@@ -61,13 +61,11 @@ impl PhysicalOptimizerRule for DelayJoinSelectionRule {
             .unwrap_or_default();
         if bc.adaptive_join_enabled() {
             let result = plan.transform_up(|node| {
-                if let Some(hj) = node.as_any().downcast_ref::<HashJoinExec>() {
+                if let Some(hj) = node.downcast_ref::<HashJoinExec>() {
                     let dynamic =
                         DynamicJoinSelectionExec::from_hash_join(hj, self.plan_id())?;
                     Ok(Transformed::yes(dynamic as Arc<dyn ExecutionPlan>))
-                } else if let Some(smj) =
-                    node.as_any().downcast_ref::<SortMergeJoinExec>()
-                {
+                } else if let Some(smj) = node.downcast_ref::<SortMergeJoinExec>() {
                     let dynamic =
                         DynamicJoinSelectionExec::from_sort_join(smj, self.plan_id())?;
                     Ok(Transformed::yes(dynamic as Arc<dyn ExecutionPlan>))
@@ -146,7 +144,7 @@ impl PhysicalOptimizerRule for SelectJoinRule {
         if bc.adaptive_join_enabled() {
             let result = plan.transform_up(|node| {
                 if let Some(dynamic_join) =
-                    node.as_any().downcast_ref::<DynamicJoinSelectionExec>()
+                    node.downcast_ref::<DynamicJoinSelectionExec>()
                 {
                     if dynamic_join.upstream_resolved() {
                         match dynamic_join.to_actual_join(config)? {
@@ -161,7 +159,6 @@ impl PhysicalOptimizerRule for SelectJoinRule {
                                     let right = hash_join_exec.right.clone();
 
                                     let right = right
-                                    .as_any()
                                     .downcast_ref::<ExchangeExec>()
                                     .ok_or(DataFusionError::Execution(
                                         "ExchangeExec is expected at this point (right)"
@@ -174,19 +171,17 @@ impl PhysicalOptimizerRule for SelectJoinRule {
                                         .with_new_children(vec![left, right])?;
 
                                     let join = join
-                                    .as_any()
-                                    .downcast_ref::<HashJoinExec>()
-                                    .ok_or(DataFusionError::Execution(
-                                    "HashJoinExec is expected at this point (right)"
-                                        .to_owned(),
-                                ))?;
+                                        .downcast_ref::<HashJoinExec>()
+                                        .ok_or(DataFusionError::Execution(
+                                        "HashJoinExec is expected at this point (right)"
+                                            .to_owned(),
+                                    ))?;
                                     let join =
                                         join.swap_inputs(PartitionMode::CollectLeft)?;
                                     Ok(Transformed::yes(join))
                                 } else {
                                     let left = hash_join_exec
                                     .left
-                                    .as_any()
                                     .downcast_ref::<ExchangeExec>()
                                     .ok_or(DataFusionError::Execution(
                                         "ExchangeExec is expected at this point (left)"
@@ -216,7 +211,7 @@ impl PhysicalOptimizerRule for SelectJoinRule {
 
                                 let exec = plan.transform_up(|p| {
                                     if let Some(hash_join) =
-                                        p.as_any().downcast_ref::<HashJoinExec>()
+                                        p.downcast_ref::<HashJoinExec>()
                                         && matches!(
                                             hash_join.partition_mode(),
                                             PartitionMode::CollectLeft
@@ -224,10 +219,9 @@ impl PhysicalOptimizerRule for SelectJoinRule {
                                     {
                                         if let Some(data_source) = hash_join
                                             .left
-                                            .as_any()
-                                            .downcast_ref::<DataSourceExec>()
-                                            && let Ok(Some(left)) =
-                                                data_source.repartitioned(1, config)
+                                            .downcast_ref::<DataSourceExec>(
+                                        ) && let Ok(Some(left)) =
+                                            data_source.repartitioned(1, config)
                                         {
                                             let right = hash_join.right.clone();
                                             let p =
@@ -242,7 +236,6 @@ impl PhysicalOptimizerRule for SelectJoinRule {
                                         {
                                             let left = if let Some(exchange) = hash_join
                                                 .left
-                                                .as_any()
                                                 .downcast_ref::<ExchangeExec>()
                                             {
                                                 Arc::new(
diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
index 86002c316f..87d52a5bd8 100644
--- a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
+++ b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
@@ -41,7 +41,7 @@ use std::sync::Arc;
 
 macro_rules! is_empty_exec {
     ($e:expr) => {
-        $e.as_any().downcast_ref::<EmptyExec>().is_some()
+        $e.downcast_ref::<EmptyExec>().is_some()
     };
 }
 
@@ -75,45 +75,45 @@ impl PropagateEmptyExecRule {
     fn transform(
         plan: Arc<dyn ExecutionPlan>,
     ) -> datafusion::error::Result<Transformed<Arc<dyn ExecutionPlan>>> {
-        if let Some(filter) = plan.as_any().downcast_ref::<FilterExec>()
+        if let Some(filter) = plan.downcast_ref::<FilterExec>()
             && is_empty_exec!(filter.input())
         {
             Ok(Transformed::yes(filter.input().clone()))
-        } else if let Some(coalesce) = plan.as_any().downcast_ref::<CoalesceBatchesExec>()
+        } else if let Some(coalesce) = plan.downcast_ref::<CoalesceBatchesExec>()
             && is_empty_exec!(coalesce.input())
         {
             Ok(Transformed::yes(coalesce.input().clone()))
-        } else if let Some(exchange) = plan.as_any().downcast_ref::<ExchangeExec>()
+        } else if let Some(exchange) = plan.downcast_ref::<ExchangeExec>()
             && is_empty_exec!(exchange.input())
         {
             Ok(Transformed::yes(exchange.input().clone()))
-        } else if let Some(projection) = plan.as_any().downcast_ref::<ProjectionExec>()
+        } else if let Some(projection) = plan.downcast_ref::<ProjectionExec>()
             && is_empty_exec!(projection.input())
         {
             empty_exec!(projection)
-        } else if let Some(limit) = plan.as_any().downcast_ref::<GlobalLimitExec>()
+        } else if let Some(limit) = plan.downcast_ref::<GlobalLimitExec>()
             && is_empty_exec!(limit.input())
         {
             Ok(Transformed::yes(limit.input().clone()))
-        } else if let Some(limit) = plan.as_any().downcast_ref::<LocalLimitExec>()
+        } else if let Some(limit) = plan.downcast_ref::<LocalLimitExec>()
             && is_empty_exec!(limit.input())
         {
             Ok(Transformed::yes(limit.input().clone()))
-        } else if let Some(aggregation) = plan.as_any().downcast_ref::<AggregateExec>()
+        } else if let Some(aggregation) = plan.downcast_ref::<AggregateExec>()
             && is_empty_exec!(aggregation.input())
         {
             empty_exec!(aggregation)
-        } else if let Some(repartition) = plan.as_any().downcast_ref::<RepartitionExec>()
+        } else if let Some(repartition) = plan.downcast_ref::<RepartitionExec>()
             && is_empty_exec!(repartition.input())
         {
             empty_exec!(repartition)
         } else if let Some(coalesce_partition) =
-            plan.as_any().downcast_ref::<CoalescePartitionsExec>()
+            plan.downcast_ref::<CoalescePartitionsExec>()
             && is_empty_exec!(coalesce_partition.input())
         {
             empty_exec!(coalesce_partition)
         } else if let Some(sort_preserving_merge) =
-            plan.as_any().downcast_ref::<SortPreservingMergeExec>()
+            plan.downcast_ref::<SortPreservingMergeExec>()
             && is_empty_exec!(sort_preserving_merge.input())
         {
             empty_exec!(sort_preserving_merge)
@@ -180,7 +180,7 @@ impl PropagateEmptyExecRule {
                 }
                 _ => Ok(Transformed::no(plan)),
             }
-        } else if let Some(exchange) = plan.as_any().downcast_ref::<ExchangeExec>() {
+        } else if let Some(exchange) = plan.downcast_ref::<ExchangeExec>() {
             let stats = exchange.partition_statistics(None)?;
             match stats.num_rows {
                 Precision::Exact(0) => empty_exec!(plan),
@@ -213,7 +213,7 @@ impl PhysicalOptimizerRule for PropagateEmptyExecRule {
 
 /// Extracting the physical join information from the incoming [ExecutionPlan]
 pub fn as_join(plan: &Arc<dyn ExecutionPlan>) -> Option<JoinInfo<'_>> {
-    let any = plan.as_any();
+    let any = plan.as_ref();
 
     if let Some(join) = any.downcast_ref::<HashJoinExec>() {
         return Some(JoinInfo {
@@ -422,7 +422,7 @@ mod tests {
     /// Assert the result is an EmptyExec.
     fn assert_empty(result: &Arc<dyn ExecutionPlan>) {
         assert!(
-            result.as_any().downcast_ref::<EmptyExec>().is_some(),
+            result.downcast_ref::<EmptyExec>().is_some(),
             "expected EmptyExec, got {:?}",
             result.name()
         );
@@ -431,7 +431,7 @@ mod tests {
     /// Assert the result is a ProjectionExec (null-padded surviving side).
     fn assert_projection(result: &Arc<dyn ExecutionPlan>) {
         assert!(
-            result.as_any().downcast_ref::<ProjectionExec>().is_some(),
+            result.downcast_ref::<ProjectionExec>().is_some(),
             "expected ProjectionExec, got {:?}",
             result.name()
         );
@@ -441,8 +441,8 @@ mod tests {
     /// i.e. the join was left untouched.
     fn assert_untouched(result: &Arc<dyn ExecutionPlan>) {
         assert!(
-            result.as_any().downcast_ref::<EmptyExec>().is_none()
-                && result.as_any().downcast_ref::<ProjectionExec>().is_none(),
+            result.downcast_ref::<EmptyExec>().is_none()
+                && result.downcast_ref::<ProjectionExec>().is_none(),
             "expected join to be untouched, got {:?}",
             result.name()
         );
@@ -732,7 +732,7 @@ mod tests {
         let plan = hash_join(Arc::clone(&left), empty_stats_exec(), JoinType::LeftAnti);
         let result = transform(plan);
         // Should be the left child itself, not wrapped in anything
-        assert!(result.as_any().downcast_ref::<StatisticsExec>().is_some());
+        assert!(result.downcast_ref::<StatisticsExec>().is_some());
     }
 
     #[test]
@@ -761,7 +761,7 @@ mod tests {
         let right = non_empty_stats_exec();
         let plan = hash_join(empty_stats_exec(), Arc::clone(&right), JoinType::RightAnti);
         let result = transform(plan);
-        assert!(result.as_any().downcast_ref::<StatisticsExec>().is_some());
+        assert!(result.downcast_ref::<StatisticsExec>().is_some());
     }
 
     #[test]
@@ -857,7 +857,7 @@ mod tests {
             JoinType::LeftAnti,
         );
         let result = transform(plan);
-        assert!(result.as_any().downcast_ref::<StatisticsExec>().is_some());
+        assert!(result.downcast_ref::<StatisticsExec>().is_some());
     }
 
     #[test]
@@ -868,7 +868,7 @@ mod tests {
             JoinType::RightAnti,
         );
         let result = transform(plan);
-        assert!(result.as_any().downcast_ref::<StatisticsExec>().is_some());
+        assert!(result.downcast_ref::<StatisticsExec>().is_some());
     }
 
     // ── unknown stats — never optimised ─────────────────────────────────────
@@ -945,7 +945,7 @@ mod tests {
             .optimize(plan.clone(), &ConfigOptions::default())
             .unwrap();
 
-        let empty_exec = result.as_any().downcast_ref::<EmptyExec>();
+        let empty_exec = result.downcast_ref::<EmptyExec>();
         assert!(empty_exec.is_some(), "expected EmptyExec");
         assert_eq!(
             empty_exec
diff --git a/ballista/scheduler/src/state/aqe/planner.rs b/ballista/scheduler/src/state/aqe/planner.rs
index 81afd9d5df..67d303761a 100644
--- a/ballista/scheduler/src/state/aqe/planner.rs
+++ b/ballista/scheduler/src/state/aqe/planner.rs
@@ -30,13 +30,14 @@ use ballista_core::serde::scheduler::PartitionLocation;
 use datafusion::common;
 use datafusion::common::{HashMap, exec_err};
 use datafusion::error::DataFusionError;
+#[cfg(test)]
+use datafusion::execution::config::SessionConfig;
 use datafusion::execution::context::SessionContext;
 use datafusion::execution::{SessionState, SessionStateBuilder};
 use datafusion::logical_expr::LogicalPlan;
 use datafusion::physical_optimizer::PhysicalOptimizerRule;
 use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable};
 use datafusion::physical_planner::DefaultPhysicalPlanner;
-use datafusion::prelude::SessionConfig;
 use log::debug;
 use std::collections::HashSet;
 use std::fmt::{Debug, Formatter};
@@ -85,8 +86,7 @@ impl AdaptivePlanner {
     /// Creates a new `AdaptivePlanner` with the specified physical optimizer rules.
     ///
     /// # Arguments:
-    ///
-    /// * `session_config` - The session configuration for the job.
+    /// * `state_builder` -  Session state builder,
     /// * `plan` - The physical execution plan for the job.
     /// * `job_name` - The name of the job.
     /// * `physical_optimizer_rules` - A list of physical optimizer rules to apply.
@@ -94,13 +94,13 @@ impl AdaptivePlanner {
     /// # Returns
     /// A new instance of `AdaptivePlanner` or an error if the initialization fails.
     pub fn try_new_with_optimizers(
-        session_config: &SessionConfig,
+        state_builder: SessionStateBuilder,
         plan: Arc<dyn ExecutionPlan>,
         job_name: String,
         physical_optimizer_rules: Vec<PhysicalOptimizerRuleRef>,
     ) -> common::Result<Self> {
         let session_state =
-            Self::create_session_state(session_config, physical_optimizer_rules);
+            Self::create_session_state(state_builder, physical_optimizer_rules);
         let planner = DefaultPhysicalPlanner::default();
         let plan = planner.optimize_physical_plan(plan, &session_state, |_, _| {})?;
 
@@ -131,8 +131,10 @@ impl AdaptivePlanner {
         job_name: String,
     ) -> common::Result<Self> {
         let plan_id_generator = Arc::new(AtomicUsize::new(0));
+        let state_builder = SessionStateBuilder::new_with_default_features()
+            .with_config(session_config.clone());
         Self::try_new_with_optimizers(
-            session_config,
+            state_builder,
             plan,
             job_name,
             Self::default_optimizers(plan_id_generator),
@@ -159,12 +161,16 @@ impl AdaptivePlanner {
         // running standard set of optimizers, which will
         // after each stage.
         let plan_id_generator = Arc::new(AtomicUsize::new(0));
-        let state = Self::create_session_state(
-            ctx.state().config(),
+
+        let plan_preparation_state_builder = SessionStateBuilder::from(ctx.state());
+        let plan_preparation_state = Self::create_session_state(
+            plan_preparation_state_builder,
             Self::plan_preparation_optimizers(plan_id_generator.clone()),
         );
 
-        let plan = state.create_physical_plan(logical_plan).await?;
+        let plan = plan_preparation_state
+            .create_physical_plan(logical_plan)
+            .await?;
 
         // Note: the signature requires a JobId, but we are passing a JobName. The below is a
         // dirty fix but this seems like a bug or a design flaw.
@@ -173,8 +179,9 @@ impl AdaptivePlanner {
             .await
             .map_err(|e| DataFusionError::Execution(e.to_string()))?;
 
+        let state_builder = SessionStateBuilder::from(ctx.state());
         Self::try_new_with_optimizers(
-            ctx.state().config(),
+            state_builder,
             plan,
             job_name,
             Self::default_optimizers(plan_id_generator),
@@ -216,8 +223,8 @@ impl AdaptivePlanner {
             .as_ref()
             .map(|stage| {
                 (
-                    stage.as_any().downcast_ref::<ExchangeExec>(),
-                    stage.as_any().downcast_ref::<AdaptiveDatafusionExec>(),
+                    stage.downcast_ref::<ExchangeExec>(),
+                    stage.downcast_ref::<AdaptiveDatafusionExec>(),
                 )
             }) {
             Some((Some(stage), None)) => {
@@ -266,7 +273,6 @@ impl AdaptivePlanner {
             ),
         )?;
         let is_broadcast = stage
-            .as_any()
             .downcast_ref::<ExchangeExec>()
             .map(|e| e.broadcast)
             .unwrap_or(false);
@@ -395,7 +401,7 @@ impl AdaptivePlanner {
         if !runnable_stages.is_empty() {
             let mut runnable = Vec::new();
             for exec in runnable_stages.into_iter() {
-                match exec.as_any().downcast_ref::<ExchangeExec>() {
+                match exec.downcast_ref::<ExchangeExec>() {
                     Some(exchange) if exchange.inactive_stage => continue,
                     Some(exchange) if exchange.stage_id().is_none() => {
                         exchange.set_stage_id(self.stage_id_generator);
@@ -418,9 +424,7 @@ impl AdaptivePlanner {
             }
 
             Ok(Some(runnable))
-        } else if let Some(root) =
-            self.plan.as_any().downcast_ref::<AdaptiveDatafusionExec>()
-        {
+        } else if let Some(root) = self.plan.downcast_ref::<AdaptiveDatafusionExec>() {
             // shuffle writer has finished
             // there is no more runnable stages
             if root.shuffle_created() {
@@ -462,8 +466,7 @@ impl AdaptivePlanner {
         runnable_stages
             .into_iter()
             .map(|exec| {
-                exec.as_any()
-                    .downcast_ref::<ExchangeExec>()
+                exec.downcast_ref::<ExchangeExec>()
                     .ok_or_else(|| {
                         datafusion::common::DataFusionError::Plan(
                             "ExchangeExec expected".into(),
@@ -543,20 +546,20 @@ impl AdaptivePlanner {
     /// Creates a session state with the given configuration and optimizer rules.
     ///
     /// # Arguments
-    /// * `session_config` - The session configuration.
+    /// * `session_builder` - The session builder.
     /// * `physical_optimizers` - A list of physical optimizer rules.
     ///
     /// # Returns
     /// A new `SessionState` instance.
     fn create_session_state(
-        session_config: &SessionConfig,
+        builder: SessionStateBuilder,
         physical_optimizers: Vec<PhysicalOptimizerRuleRef>,
     ) -> SessionState {
-        SessionStateBuilder::new_with_default_features()
+        builder
             .with_physical_optimizer_rules(physical_optimizers)
-            .with_config(session_config.clone())
             .build()
     }
+
     /// Recursively finds runnable exchanges in the execution plan.
     ///
     /// # Arguments
@@ -569,7 +572,7 @@ impl AdaptivePlanner {
         node: &Arc<dyn ExecutionPlan>,
         runnable_stages: &mut Vec<Arc<dyn ExecutionPlan>>,
     ) -> bool {
-        if let Some(exchange) = node.as_any().downcast_ref::<ExchangeExec>()
+        if let Some(exchange) = node.downcast_ref::<ExchangeExec>()
             && exchange.shuffle_created()
         {
             // we found exchange which has partitions resolved or this stage is not
@@ -577,7 +580,7 @@ impl AdaptivePlanner {
             // all runnable children has been run
 
             false
-        } else if let Some(exchange) = node.as_any().downcast_ref::<ExchangeExec>()
+        } else if let Some(exchange) = node.downcast_ref::<ExchangeExec>()
             && !exchange.shuffle_created()
         {
             // we found exchange which has not been resolved (run)
diff --git a/ballista/scheduler/src/state/aqe/test/alter_stages.rs b/ballista/scheduler/src/state/aqe/test/alter_stages.rs
index fcf48e9cfa..4ba896fffe 100644
--- a/ballista/scheduler/src/state/aqe/test/alter_stages.rs
+++ b/ballista/scheduler/src/state/aqe/test/alter_stages.rs
@@ -37,7 +37,6 @@ use datafusion::physical_plan::test::exec::StatisticsExec;
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,
 };
-use std::any::Any;
 use std::collections::HashSet;
 use std::fmt::Formatter;
 use std::sync::Arc;
@@ -73,7 +72,8 @@ async fn should_propagate_empty_stage() -> datafusion::error::Result<()> {
     assert_eq!(1, stages.len());
     assert_plan!(stages.first().unwrap().plan.as_ref(),  @ r"
     ShuffleWriterExec: partitioning: None
-      EmptyExec
+      RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+        EmptyExec
     ");
     planner.finalise_stage_internal(1, mock_partitions_with_statistics_no_data())?;
 
@@ -147,7 +147,8 @@ async fn should_propagate_empty_stage_and_remove() -> datafusion::error::Result<
     assert_eq!(1, stages.len());
     assert_plan!(stages.first().unwrap().plan.as_ref(),  @ r"
     ShuffleWriterExec: partitioning: None
-      EmptyExec
+      RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+        EmptyExec
     ");
     planner.finalise_stage_internal(1, mock_partitions_with_statistics_no_data())?;
 
@@ -211,45 +212,42 @@ async fn should_support_join_re_ordering() -> datafusion::error::Result<()> {
 
     // join ordering changes as build side is bigger than probe side
     // after exchange statistic updated.
-    assert_plan!(planner.current_plan(),  @ r"
+    assert_plan!(planner.current_plan(),  @ "
     AdaptiveDatafusionExec: is_final=false, plan_id=2, stage_id=pending, stage_resolved=false
-      ProjectionExec: expr=[big_col@1 as big_col, big_col@0 as big_col]
-        HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)]
-          ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true
-            CooperativeExec
-              MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
-          ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true
-            CooperativeExec
-              MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
+      HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)], projection=[big_col@1, big_col@0]
+        ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true
+          CooperativeExec
+            MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
+        ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true
+          CooperativeExec
+            MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
     ");
 
     let stages = planner.runnable_stages()?.unwrap();
     assert_eq!(1, stages.len());
 
-    assert_plan!(planner.current_plan(),  @ r"
+    assert_plan!(planner.current_plan(),  @ "
     AdaptiveDatafusionExec: is_final=true, plan_id=2, stage_id=2, stage_resolved=false
-      ProjectionExec: expr=[big_col@1 as big_col, big_col@0 as big_col]
-        HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)]
-          ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true
-            CooperativeExec
-              MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
-          ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true
-            CooperativeExec
-              MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
+      HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)], projection=[big_col@1, big_col@0]
+        ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true
+          CooperativeExec
+            MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
+        ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true
+          CooperativeExec
+            MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
     ");
 
     planner.finalise_stage_internal(2, small_statistics_exchange())?;
 
-    assert_plan!(planner.current_plan(),  @ r"
+    assert_plan!(planner.current_plan(),  @ "
     AdaptiveDatafusionExec: is_final=true, plan_id=2, stage_id=2, stage_resolved=true
-      ProjectionExec: expr=[big_col@1 as big_col, big_col@0 as big_col]
-        HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)]
-          ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true
-            CooperativeExec
-              MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
-          ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true
-            CooperativeExec
-              MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
+      HashJoinExec: mode=Partitioned, join_type=Inner, on=[(big_col@0, big_col@0)], projection=[big_col@1, big_col@0]
+        ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=1, stage_id=1, stage_resolved=true
+          CooperativeExec
+            MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
+        ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0, stage_id=0, stage_resolved=true
+          CooperativeExec
+            MockPartitionedScan: num_partitions=2, statistics=[Rows=Exact(262144), Bytes=Exact(2097152), [(Col[0]:)]]
     ");
 
     Ok(())
@@ -524,7 +522,7 @@ fn small_statistics_exchange() -> Vec<Vec<PartitionLocation>> {
             host: "".to_string(),
             port: 0,
             grpc_port: 0,
-            specification: ExecutorSpecification::default().with_task_slots(0),
+            specification: ExecutorSpecification::default().with_vcores(0),
             os_info: ExecutorOperatingSystemSpecification::default(),
         },
         // next few properties are needed
@@ -556,7 +554,7 @@ fn big_statistics_exchange() -> Vec<Vec<PartitionLocation>> {
             host: "".to_string(),
             port: 0,
             grpc_port: 0,
-            specification: ExecutorSpecification::default().with_task_slots(0),
+            specification: ExecutorSpecification::default().with_vcores(0),
             os_info: ExecutorOperatingSystemSpecification::default(),
         },
 
@@ -649,10 +647,6 @@ impl ExecutionPlan for MockPartitionedScan {
         "MockPartitionedScan"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.plan_properties
     }
@@ -679,7 +673,7 @@ impl ExecutionPlan for MockPartitionedScan {
     fn partition_statistics(
         &self,
         _partition: Option<usize>,
-    ) -> datafusion::common::Result<Statistics> {
-        Ok(self.statistics.clone())
+    ) -> datafusion::common::Result<Arc<Statistics>> {
+        Ok(Arc::new(self.statistics.clone()))
     }
 }
diff --git a/ballista/scheduler/src/state/aqe/test/coalesce_rule.rs b/ballista/scheduler/src/state/aqe/test/coalesce_rule.rs
index 665eee88f8..0ac47ae721 100644
--- a/ballista/scheduler/src/state/aqe/test/coalesce_rule.rs
+++ b/ballista/scheduler/src/state/aqe/test/coalesce_rule.rs
@@ -106,7 +106,7 @@ fn partitions_with_byte_sizes(
                     host: "".to_string(),
                     port: 0,
                     grpc_port: 0,
-                    specification: ExecutorSpecification::default().with_task_slots(0),
+                    specification: ExecutorSpecification::default().with_vcores(0),
                     os_info: ExecutorOperatingSystemSpecification::default(),
                 },
                 partition_stats: PartitionStats::new(Some(1), None, Some(bytes)),
diff --git a/ballista/scheduler/src/state/aqe/test/job_failure.rs b/ballista/scheduler/src/state/aqe/test/job_failure.rs
new file mode 100644
index 0000000000..f468e55b8e
--- /dev/null
+++ b/ballista/scheduler/src/state/aqe/test/job_failure.rs
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Job-failure lifecycle tests for the adaptive execution graph.
+
+use crate::state::aqe::AdaptiveExecutionGraph;
+use crate::state::execution_graph::ExecutionGraph;
+use crate::state::execution_stage::ExecutionStage;
+use crate::test_utils::mock_executor;
+use ballista_core::error::Result;
+use ballista_core::serde::protobuf::{
+    FailedTask, JobStatus, failed_task, job_status, task_status,
+};
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::execution::context::{SessionConfig, SessionContext};
+use datafusion::functions_aggregate::sum::sum;
+use datafusion::logical_expr::SortExpr;
+use datafusion::prelude::{JoinType, col};
+use datafusion::test_util::scan_empty_with_partitions;
+
+/// Builds an adaptive graph for a join (two concurrent leaf stages).
+async fn test_join_plan(partition: usize) -> AdaptiveExecutionGraph {
+    let mut config = SessionConfig::new().with_target_partitions(partition);
+    config
+        .options_mut()
+        .optimizer
+        .enable_round_robin_repartition = false;
+    let ctx = SessionContext::new_with_config(config);
+
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Utf8, false),
+        Field::new("gmv", DataType::UInt64, false),
+    ]);
+
+    let left_plan = scan_empty_with_partitions(Some("left"), &schema, None, 2).unwrap();
+    let right_plan = scan_empty_with_partitions(Some("right"), &schema, None, 2)
+        .unwrap()
+        .build()
+        .unwrap();
+    let sort_expr = SortExpr::new(col("id"), false, false);
+    let logical_plan = left_plan
+        .join(right_plan, JoinType::Inner, (vec!["id"], vec!["id"]), None)
+        .unwrap()
+        .aggregate(vec![col("left.id")], vec![sum(col("left.gmv"))])
+        .unwrap()
+        .sort(vec![sort_expr])
+        .unwrap()
+        .build()
+        .unwrap();
+
+    AdaptiveExecutionGraph::try_new(
+        "localhost:50050",
+        &"job".into(),
+        "",
+        &ctx,
+        &logical_plan,
+        0,
+    )
+    .await
+    .unwrap()
+}
+
+// Same contract as the static graph's test: aborting transitions every running
+// stage to Failed and returns its in-flight tasks for cancellation.
+#[tokio::test]
+async fn test_abort_running_cancels_stages_and_returns_inflight_tasks() -> Result<()> {
+    let executor = mock_executor("executor-id1".to_string());
+    let mut graph = test_join_plan(2).await;
+
+    // Call revive to move the two leaf Resolved stages to Running
+    graph.revive();
+    assert!(
+        graph.running_stages().len() >= 2,
+        "expected two concurrently running leaf stages, found {:?}",
+        graph.running_stages()
+    );
+
+    // Dispatch a task so there is an in-flight task to cancel
+    let _task = graph.pop_next_task(&executor.id)?.unwrap();
+
+    // Aborting cancels every running stage and returns its in-flight tasks
+    let cancelled = graph.abort_running("job aborted".to_string());
+
+    assert!(
+        !cancelled.is_empty(),
+        "abort_running must return the in-flight tasks to cancel"
+    );
+    assert!(
+        graph.running_stages().is_empty(),
+        "every running stage must be cancelled, found {:?}",
+        graph.running_stages()
+    );
+    assert!(
+        matches!(
+            graph.status(),
+            JobStatus {
+                status: Some(job_status::Status::Failed(_)),
+                ..
+            }
+        ),
+        "the job must be Failed after abort"
+    );
+
+    // In-flight tasks of the cancelled stage are recorded as Failed(TaskKilled)
+    let has_killed_task = graph.stages.values().any(|stage| match stage {
+        ExecutionStage::Failed(failed) => failed.task_infos.iter().any(|info| {
+            matches!(
+                &info.task_status,
+                task_status::Status::Failed(FailedTask {
+                    failed_reason: Some(failed_task::FailedReason::TaskKilled(_)),
+                    ..
+                })
+            )
+        }),
+        _ => false,
+    });
+    assert!(
+        has_killed_task,
+        "in-flight tasks must be recorded as Failed(TaskKilled) after abort"
+    );
+
+    Ok(())
+}
diff --git a/ballista/scheduler/src/state/aqe/test/join_selection.rs b/ballista/scheduler/src/state/aqe/test/join_selection.rs
index 7dba88e81f..aa4aac860c 100644
--- a/ballista/scheduler/src/state/aqe/test/join_selection.rs
+++ b/ballista/scheduler/src/state/aqe/test/join_selection.rs
@@ -19,6 +19,7 @@ use crate::{
     assert_plan,
     state::aqe::{planner::AdaptivePlanner, test::mock_partitions_with_statistics},
 };
+use ballista_core::extension::SessionConfigExt;
 use datafusion::{
     arrow::{
         array::{Int32Array, RecordBatch},
@@ -85,6 +86,10 @@ fn make_ctx_without_collect_left(prefer_hash_join: bool) -> SessionContext {
             "datafusion.optimizer.hash_join_single_partition_threshold_rows",
             0,
         )
+        // AQE join selection reads the broadcast cutoff from Ballista's own
+        // config; a byte threshold of 0 disables CollectLeft promotion, keeping
+        // these joins repartitioned.
+        .with_ballista_broadcast_join_threshold_bytes(0)
         .with_target_partitions(4)
         .with_round_robin_repartition(false);
     let state = SessionStateBuilder::new_with_default_features()
@@ -141,6 +146,38 @@ async fn test_hash_join_two_tables_coalesce() -> datafusion::common::Result<()>
     Ok(())
 }
 
+// A `CollectLeft` join broadcasts (replicates) the build/left side to every
+// probe task, each of which sees only its slice of the probe/right side. For a
+// LEFT join that emits a null-padded row for every unmatched left row, each
+// task would emit those rows independently, producing duplicate/spurious rows
+// (apache/datafusion-ballista#1055). The resolver must therefore keep an
+// unsafe-to-broadcast join type repartitioned instead of collecting it.
+#[tokio::test]
+async fn test_left_join_not_collected_left() -> datafusion::common::Result<()> {
+    let ctx = make_ctx(true);
+    register_2tables(&ctx);
+
+    let lp = ctx
+        .sql("SELECT t1.id, t2.val FROM t1 LEFT JOIN t2 ON t1.id = t2.id")
+        .await
+        .unwrap()
+        .into_optimized_plan()
+        .unwrap();
+
+    let planner = AdaptivePlanner::try_new(&ctx, &lp, "test_job".to_owned()).await?;
+    let plan = planner.current_plan();
+    assert_plan!(plan, @ r"
+    AdaptiveDatafusionExec: is_final=false, plan_id=3, stage_id=pending, stage_resolved=false
+      ProjectionExec: expr=[id@0 as id, val@2 as val]
+        DynamicJoinSelectionExec: plan_id=0, join_type=Left, on=[(id@0, id@0)] repartitioned=true
+          ExchangeExec: partitioning=Hash([id@0], 4), plan_id=1, stage_id=pending, stage_resolved=false
+            DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1]
+          ExchangeExec: partitioning=Hash([id@0], 4), plan_id=2, stage_id=pending, stage_resolved=false
+            DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1]
+    ");
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_hash_join_two_tables_repartition() -> datafusion::common::Result<()> {
     let ctx = make_ctx_without_collect_left(true);
diff --git a/ballista/scheduler/src/state/aqe/test/mod.rs b/ballista/scheduler/src/state/aqe/test/mod.rs
index c1237f216d..a8640d3e30 100644
--- a/ballista/scheduler/src/state/aqe/test/mod.rs
+++ b/ballista/scheduler/src/state/aqe/test/mod.rs
@@ -19,6 +19,8 @@
 mod alter_stages;
 /// Functional tests for the CoalescePartitionsRule end-to-end through the planner
 mod coalesce_rule;
+/// Job-failure lifecycle tests for the adaptive graph
+mod job_failure;
 /// covers join selection tests
 mod join_selection;
 /// Tests if plan is going to be split to stages correctly
@@ -52,7 +54,7 @@ pub(crate) fn mock_partitions_with_statistics() -> Vec<Vec<PartitionLocation>> {
             host: "".to_string(),
             port: 0,
             grpc_port: 0,
-            specification: ExecutorSpecification::default().with_task_slots(0),
+            specification: ExecutorSpecification::default().with_vcores(0),
             os_info: ExecutorOperatingSystemSpecification::default(),
         },
         // next few properties are needed
@@ -77,7 +79,7 @@ pub(crate) fn mock_partitions_with_statistics_no_data() -> Vec<Vec<PartitionLoca
             host: "".to_string(),
             port: 0,
             grpc_port: 0,
-            specification: ExecutorSpecification::default().with_task_slots(0),
+            specification: ExecutorSpecification::default().with_vcores(0),
             os_info: ExecutorOperatingSystemSpecification::default(),
         },
         // next few properties are needed
diff --git a/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs b/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs
index 528a912fb8..b115f273c5 100644
--- a/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs
+++ b/ballista/scheduler/src/state/aqe/test/plan_to_stages.rs
@@ -25,6 +25,7 @@ use crate::state::aqe::test::{
 use ballista_core::execution_plans::SortShuffleWriterExec;
 use datafusion::arrow::datatypes::{DataType, Field, Schema};
 use datafusion::common::ColumnStatistics;
+use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_plan::Statistics;
 use datafusion::physical_plan::test::exec::StatisticsExec;
 use std::collections::HashSet;
@@ -475,7 +476,7 @@ async fn should_use_sort_shuffle_when_enabled() -> datafusion::error::Result<()>
 
     let plan = stages.first().unwrap().plan.as_ref();
     assert!(
-        plan.as_any()
+        (plan as &dyn ExecutionPlan)
             .downcast_ref::<SortShuffleWriterExec>()
             .is_some(),
         "expected SortShuffleWriterExec when sort shuffle is enabled, got plan: {plan:?}"
@@ -505,7 +506,7 @@ async fn should_use_sort_shuffle_by_default() -> datafusion::error::Result<()> {
 
     let plan = stages.first().unwrap().plan.as_ref();
     assert!(
-        plan.as_any()
+        (plan as &dyn ExecutionPlan)
             .downcast_ref::<SortShuffleWriterExec>()
             .is_some(),
         "expected SortShuffleWriterExec by default, got plan: {plan:?}"
diff --git a/ballista/scheduler/src/state/distributed_explain.rs b/ballista/scheduler/src/state/distributed_explain.rs
index 4292950329..c4408be468 100644
--- a/ballista/scheduler/src/state/distributed_explain.rs
+++ b/ballista/scheduler/src/state/distributed_explain.rs
@@ -195,7 +195,7 @@ pub(crate) async fn handle_explain_plan(
     plan: Arc<dyn ExecutionPlan>,
 ) -> ballista_core::error::Result<Arc<dyn ExecutionPlan>> {
     if let LogicalPlan::Explain(explain_plan) = &logical_plan
-        && let Some(explain) = plan.as_any().downcast_ref::<ExplainExec>()
+        && let Some(explain) = plan.downcast_ref::<ExplainExec>()
     {
         let inner_plan = explain_plan.plan.clone();
         let plans = explain.stringified_plans();
diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs
index 72fd05f8f9..dd5704e769 100644
--- a/ballista/scheduler/src/state/execution_graph.rs
+++ b/ballista/scheduler/src/state/execution_graph.rs
@@ -40,7 +40,7 @@ use ballista_core::serde::protobuf::{
 };
 use ballista_core::serde::protobuf::{RunningTask, task_status};
 use ballista_core::serde::scheduler::{
-    ExecutorMetadata, PartitionId, PartitionLocation, PartitionStats,
+    ExecutorMetadata, PartitionId, PartitionLocation, PartitionStats, TaskKey,
 };
 
 use crate::display::print_stage_metrics;
@@ -159,12 +159,11 @@ pub trait ExecutionGraph: Debug {
 
     /// Fetches a running stage that has available tasks, excluding stages in the blacklist.
     ///
-    /// Returns a mutable reference to the running stage and the task ID generator
-    /// if a suitable stage is found.
-    fn fetch_running_stage(
-        &mut self,
-        black_list: &[usize],
-    ) -> Option<(&mut RunningStage, &mut usize)>;
+    /// Returns a mutable reference to the running stage if a suitable
+    /// stage is found. task_id is assigned per-stage as
+    /// `task_infos.len()` at bind time (see `bind_one`), so no external
+    /// generator is needed.
+    fn fetch_running_stage(&mut self, black_list: &[usize]) -> Option<&mut RunningStage>;
 
     /// Updates the job status.
     fn update_status(&mut self, status: JobStatus);
@@ -219,6 +218,18 @@ pub trait ExecutionGraph: Debug {
     /// fail job with error message
     fn fail_job(&mut self, error: String);
 
+    /// Abort a running job: fail it, transition every running stage to Failed,
+    /// and return the in-flight tasks that should be cancelled. Used for both the
+    /// failure and cancellation teardown paths.
+    fn abort_running(&mut self, error: String) -> Vec<RunningTaskInfo> {
+        let running_tasks = self.running_tasks();
+        self.fail_job(error.clone());
+        for stage_id in self.running_stages() {
+            self.fail_stage(stage_id, error.clone());
+        }
+        running_tasks
+    }
+
     /// Marks the job as successfully completed.
     ///
     /// This should only be called after all stages have completed successfully.
@@ -228,6 +239,30 @@ pub trait ExecutionGraph: Debug {
     /// Exposes executions stages and stage id's
     fn stages(&self) -> &HashMap<usize, ExecutionStage>;
 
+    /// Stage ids of all non-final (intermediate) stages — those whose
+    /// `output_links` is non-empty. The final stage(s) are excluded.
+    fn intermediate_stage_ids(&self) -> Vec<u32> {
+        self.stages()
+            .iter()
+            .filter(|(_, stage)| !stage.output_links().is_empty())
+            .map(|(stage_id, _)| *stage_id as u32)
+            .collect()
+    }
+
+    /// Vcores a task consumed from the executor's budget at bind time.
+    /// Usually equals `global_input_partition_ids.len()`, but for collapse
+    /// tasks that monopolize the executor it is capped at the budget
+    /// available when they were bound. Returns `None` if the stage or task
+    /// is unknown — e.g. the stage was evicted or the `task_id` (append
+    /// slot in `task_infos`) is out of range.
+    fn task_vcores(&self, stage_id: usize, task_id: usize) -> Option<u32> {
+        self.stages()
+            .get(&stage_id)
+            .and_then(|s| s.task_infos())
+            .and_then(|infos| infos.get(task_id))
+            .map(|ti| ti.vcores_consumed)
+    }
+
     /// returns next task to run
     /// (used for testing only)
     #[cfg(test)]
@@ -265,8 +300,6 @@ pub struct StaticExecutionGraph {
     stages: HashMap<usize, ExecutionStage>,
     /// Locations of this `ExecutionGraph` final output locations
     output_locations: Vec<PartitionLocation>,
-    /// Task ID generator, generate unique TID in the execution graph
-    task_id_gen: usize,
     /// Failed stage attempts, record the failed stage attempts to limit the retry times.
     /// Map from Stage ID -> Set<Stage_ATTPMPT_NUM>
     failed_stage_attempts: HashMap<usize, HashSet<usize>>,
@@ -284,14 +317,13 @@ pub struct StaticExecutionGraph {
 /// when an executor is lost or a job is cancelled.
 #[derive(Clone, Debug)]
 pub struct RunningTaskInfo {
-    /// Unique identifier for this task within the execution graph.
+    /// Append-order slot of this task in `RunningStage.task_infos`;
+    /// `(job_id, stage_id, task_id)` is globally unique.
     pub task_id: usize,
     /// The job ID this task belongs to.
     pub job_id: JobId,
     /// The stage ID this task belongs to.
     pub stage_id: usize,
-    /// The partition this task is processing.
-    pub partition_id: usize,
     /// The executor ID where this task is running.
     pub executor_id: String,
 }
@@ -341,7 +373,6 @@ impl StaticExecutionGraph {
             end_time: 0,
             stages,
             output_locations: vec![],
-            task_id_gen: 0,
             failed_stage_attempts: HashMap::new(),
             session_config,
             logical_plan,
@@ -349,13 +380,6 @@ impl StaticExecutionGraph {
         })
     }
 
-    #[cfg(test)]
-    fn next_task_id(&mut self) -> usize {
-        let new_tid = self.task_id_gen;
-        self.task_id_gen += 1;
-        new_tid
-    }
-
     /// Processing stage status update after task status changing
     fn processing_stages_update(
         &mut self,
@@ -779,26 +803,22 @@ impl ExecutionGraph for StaticExecutionGraph {
                             );
                             continue;
                         }
-                        let partition_id = task_status.clone().partition_id as usize;
+                        let task_id = task_status.task_id as usize;
                         let task_identity = format!(
-                            "TID {} {}/{}.{}/{}",
-                            task_status.task_id,
-                            job_id,
-                            stage_id,
-                            task_stage_attempt_num,
-                            partition_id
+                            "TID {}/{}.{}/{}",
+                            job_id, stage_id, task_stage_attempt_num, task_id
                         );
-                        let operator_metrics = task_status.metrics.clone();
-
-                        if !running_stage
-                            .update_task_info(partition_id, task_status.clone())
-                        {
+                        if !running_stage.update_task_info(task_id, task_status.clone()) {
                             continue;
                         }
 
-                        if let Some(task_status::Status::Failed(failed_task)) =
-                            task_status.status
-                        {
+                        let TaskStatus {
+                            status,
+                            metrics: operator_metrics,
+                            ..
+                        } = task_status;
+
+                        if let Some(task_status::Status::Failed(failed_task)) = status {
                             let failed_reason = failed_task.failed_reason;
 
                             match failed_reason {
@@ -870,16 +890,37 @@ impl ExecutionGraph for StaticExecutionGraph {
                                     if failed_task.retryable
                                         && failed_task.count_to_failures
                                     {
-                                        if running_stage.task_failure_number(partition_id)
+                                        if running_stage.task_failure_number(task_id)
                                             < max_task_failures
                                         {
                                             // TODO add new struct to track all the failed task infos
                                             // The failure TaskInfo is ignored and set to None here
-                                            running_stage.reset_task_info(partition_id);
+                                            running_stage.reset_task_info(task_id);
                                         } else {
+                                            // Report the *partitions* that hit the failure
+                                            // ceiling — task_id (the append slot) isn't
+                                            // user-meaningful under the append-only retries
+                                            // model (retries get fresh slots), but
+                                            // per-partition failure counters are the
+                                            // durable identity.
+                                            let over_limit: Vec<usize> = running_stage
+                                                .task_infos[task_id]
+                                                .global_input_partition_ids
+                                                .iter()
+                                                .copied()
+                                                .filter(|p| {
+                                                    running_stage.task_failure_numbers[*p]
+                                                        >= max_task_failures
+                                                })
+                                                .collect();
+                                            let subject = if over_limit.len() == 1 {
+                                                format!("Task {}", over_limit[0])
+                                            } else {
+                                                format!("Tasks {over_limit:?}")
+                                            };
                                             let error_msg = format!(
-                                                "Task {} in Stage {} failed {} times, fail the stage, most recent failure reason: {:?}",
-                                                partition_id,
+                                                "{} in Stage {} failed {} times, fail the stage, most recent failure reason: {:?}",
+                                                subject,
                                                 stage_id,
                                                 max_task_failures,
                                                 failed_task.error
@@ -890,12 +931,12 @@ impl ExecutionGraph for StaticExecutionGraph {
                                     } else if failed_task.retryable {
                                         // TODO add new struct to track all the failed task infos
                                         // The failure TaskInfo is ignored and set to None here
-                                        running_stage.reset_task_info(partition_id);
+                                        running_stage.reset_task_info(task_id);
                                     }
                                 }
                                 None => {
                                     let error_msg = format!(
-                                        "Task {partition_id} in Stage {stage_id} failed with unknown failure reasons, fail the stage"
+                                        "Task {task_id} in Stage {stage_id} failed with unknown failure reasons, fail the stage"
                                     );
                                     error!("{error_msg}");
                                     failed_stages.insert(stage_id, error_msg);
@@ -903,15 +944,15 @@ impl ExecutionGraph for StaticExecutionGraph {
                             }
                         } else if let Some(task_status::Status::Successful(
                             successful_task,
-                        )) = task_status.status
+                        )) = status
                         {
-                            // update task metrics for successfu task
+                            // update task metrics for successful task
                             running_stage
-                                .update_task_metrics(partition_id, operator_metrics)?;
+                                .update_task_metrics(task_id, operator_metrics)?;
 
                             locations.append(&mut partition_to_location(
                                 &job_id,
-                                partition_id,
+                                task_id,
                                 stage_id,
                                 executor,
                                 successful_task.partitions,
@@ -954,14 +995,10 @@ impl ExecutionGraph for StaticExecutionGraph {
                     for task_status in stage_task_statuses.into_iter() {
                         let task_stage_attempt_num =
                             task_status.stage_attempt_num as usize;
-                        let partition_id = task_status.clone().partition_id as usize;
+                        let task_id = task_status.task_id as usize;
                         let task_identity = format!(
-                            "TID {} {}/{}.{}/{}",
-                            task_status.task_id,
-                            job_id,
-                            stage_id,
-                            task_stage_attempt_num,
-                            partition_id
+                            "TID {}/{}.{}/{}",
+                            job_id, stage_id, task_stage_attempt_num, task_id
                         );
                         let mut should_ignore = true;
                         // handle delayed failed tasks if the stage's next attempt is still in UnResolved status.
@@ -1034,7 +1071,7 @@ impl ExecutionGraph for StaticExecutionGraph {
                         stage_id,
                         stage_task_statuses
                             .into_iter()
-                            .map(|task_status| task_status.partition_id)
+                            .map(|task_status| task_status.task_id)
                             .collect::<Vec<_>>(),
                     );
                 }
@@ -1051,17 +1088,32 @@ impl ExecutionGraph for StaticExecutionGraph {
                 .insert(*stage_id, HashSet::from_iter(attempts.iter().copied()));
         }
 
-        for (stage_id, missing_parts) in &resubmit_successful_stages {
+        // The values in `resubmit_successful_stages` and `reset_running_stages`
+        // come out of `remove_input_partitions` collecting
+        // `loc.map_partition_id`, which under the multi-partition-task model
+        // is the source task's `task_id` (see `partition_to_location`).
+        // So the set members are task_ids (append slots), not partition ids —
+        // bounds check against `task_infos.len()` and pass through to
+        // `reset_task_info` / `task_infos[...]` directly.
+        //
+        // TODO: switch to partition-id semantics — push only the actually-lost
+        // partition_ids back into `stage.pending`, avoiding the "redo the
+        // whole map task's slice" waste on partial-loss scenarios. Needs
+        // `remove_input_partitions` to expose partition_ids (from the outer
+        // map key) and a way to partial-reset a still-Running task.
+        for (stage_id, missing_task_ids) in &resubmit_successful_stages {
             if let Some(stage) = self.stages.get_mut(stage_id) {
                 if let ExecutionStage::Successful(success_stage) = stage {
-                    for partition in missing_parts {
-                        if *partition > success_stage.partitions {
+                    for task_id in missing_task_ids {
+                        if *task_id >= success_stage.task_infos.len() {
                             return Err(BallistaError::Internal(format!(
-                                "Invalid partition ID {} in map stage {}",
-                                *partition, stage_id
+                                "Invalid task_id {} in map stage {} (task_infos has {} entries)",
+                                *task_id,
+                                stage_id,
+                                success_stage.task_infos.len()
                             )));
                         }
-                        let task_info = &mut success_stage.task_infos[*partition];
+                        let task_info = &mut success_stage.task_infos[*task_id];
                         // Update the task info to failed
                         task_info.task_status = task_status::Status::Failed(FailedTask {
                             error: "FetchPartitionError in parent stage".to_owned(),
@@ -1082,17 +1134,19 @@ impl ExecutionGraph for StaticExecutionGraph {
             }
         }
 
-        for (stage_id, missing_parts) in &reset_running_stages {
+        for (stage_id, missing_task_ids) in &reset_running_stages {
             if let Some(stage) = self.stages.get_mut(stage_id) {
                 if let ExecutionStage::Running(running_stage) = stage {
-                    for partition in missing_parts {
-                        if *partition > running_stage.partitions {
+                    for task_id in missing_task_ids {
+                        if *task_id >= running_stage.task_infos.len() {
                             return Err(BallistaError::Internal(format!(
-                                "Invalid partition ID {} in map stage {}",
-                                *partition, stage_id
+                                "Invalid task_id {} in map stage {} (task_infos has {} entries)",
+                                *task_id,
+                                stage_id,
+                                running_stage.task_infos.len()
                             )));
                         }
-                        running_stage.reset_task_info(*partition);
+                        running_stage.reset_task_info(*task_id);
                     }
                 } else {
                     warn!(
@@ -1141,14 +1195,11 @@ impl ExecutionGraph for StaticExecutionGraph {
                     stage
                         .running_tasks()
                         .into_iter()
-                        .map(|(task_id, stage_id, partition_id, executor_id)| {
-                            RunningTaskInfo {
-                                task_id,
-                                job_id: self.job_id.clone(),
-                                stage_id,
-                                partition_id,
-                                executor_id,
-                            }
+                        .map(|(task_id, stage_id, executor_id)| RunningTaskInfo {
+                            task_id,
+                            job_id: self.job_id.clone(),
+                            stage_id,
+                            executor_id,
                         })
                         .collect::<Vec<RunningTaskInfo>>()
                 } else {
@@ -1172,10 +1223,7 @@ impl ExecutionGraph for StaticExecutionGraph {
             .sum()
     }
 
-    fn fetch_running_stage(
-        &mut self,
-        black_list: &[usize],
-    ) -> Option<(&mut RunningStage, &mut usize)> {
+    fn fetch_running_stage(&mut self, black_list: &[usize]) -> Option<&mut RunningStage> {
         if matches!(
             self.status,
             JobStatus {
@@ -1192,7 +1240,7 @@ impl ExecutionGraph for StaticExecutionGraph {
             if let Some(ExecutionStage::Running(running_stage)) =
                 self.stages.get_mut(&running_stage_id)
             {
-                Some((running_stage, &mut self.task_id_gen))
+                Some(running_stage)
             } else {
                 warn!("Fail to find running stage with id {running_stage_id}");
                 None
@@ -1297,15 +1345,12 @@ impl ExecutionGraph for StaticExecutionGraph {
             let running_tasks = stage
                 .running_tasks()
                 .into_iter()
-                .map(
-                    |(task_id, stage_id, partition_id, executor_id)| RunningTaskInfo {
-                        task_id,
-                        job_id: self.job_id.clone(),
-                        stage_id,
-                        partition_id,
-                        executor_id,
-                    },
-                )
+                .map(|(task_id, stage_id, executor_id)| RunningTaskInfo {
+                    task_id,
+                    job_id: self.job_id.clone(),
+                    stage_id,
+                    executor_id,
+                })
                 .collect();
             self.stages.insert(
                 stage_id,
@@ -1432,19 +1477,6 @@ impl ExecutionGraph for StaticExecutionGraph {
         let job_id = self.job_id.clone();
         let session_id = self.session_id.clone();
 
-        let find_candidate = self.stages.iter().any(|(_stage_id, stage)| {
-            if let ExecutionStage::Running(stage) = stage {
-                stage.available_tasks() > 0
-            } else {
-                false
-            }
-        });
-        let next_task_id = if find_candidate {
-            Some(self.next_task_id())
-        } else {
-            None
-        };
-
         let mut next_task = self.stages.iter_mut().find(|(_stage_id, stage)| {
             if let ExecutionStage::Running(stage) = stage {
                 stage.available_tasks() > 0
@@ -1453,23 +1485,23 @@ impl ExecutionGraph for StaticExecutionGraph {
             }
         }).map(|(stage_id, stage)| {
             if let ExecutionStage::Running(stage) = stage {
-                let (partition_id, _) = stage
-                    .task_infos
+                // pop_next_task hands out a single-partition task — bind path
+                // sized to `exec.vcores` lives in `cluster::bind_task_*`.
+                let input_partition_ids = stage.pending.next_slice(1);
+                if input_partition_ids.is_empty() {
+                    return Err(BallistaError::Internal(format!(
+                        "Error getting next task for job {job_id}: Stage {stage_id} is ready but has no pending tasks"
+                    )));
+                }
+                // task_id is the append slot in `task_infos` — assigned as
+                // `task_infos.len()` at bind time. `(job_id, stage_id, task_id)`
+                // is globally unique.
+                let task_id = stage.task_infos.len();
+                let task_attempt = input_partition_ids
                     .iter()
-                    .enumerate()
-                    .find(|(_partition, info)| info.is_none())
-                    .ok_or_else(|| {
-                        BallistaError::Internal(format!("Error getting next task for job {job_id}: Stage {stage_id} is ready but has no pending tasks"))
-                    })?;
-
-                let partition = PartitionId {
-                    job_id,
-                    stage_id: *stage_id,
-                    partition_id,
-                };
-
-                let task_id = next_task_id.unwrap();
-                let task_attempt = stage.task_failure_numbers[partition_id];
+                    .map(|pid| stage.task_failure_numbers[*pid])
+                    .max()
+                    .unwrap_or(0);
                 let task_info = TaskInfo {
                     task_id,
                     scheduled_time: SystemTime::now()
@@ -1484,17 +1516,25 @@ impl ExecutionGraph for StaticExecutionGraph {
                     task_status: task_status::Status::Running(RunningTask {
                         executor_id: executor_id.to_owned()
                     }),
+                    global_input_partition_ids: input_partition_ids.clone(),
+                    vcores_consumed: input_partition_ids.len() as u32,
                 };
+                stage.task_infos.push(task_info);
 
-                // Set the task info to Running for new task
-                stage.task_infos[partition_id] = Some(task_info);
+                let key = TaskKey {
+                    job_id,
+                    stage_id: *stage_id,
+                    task_id,
+                };
 
+                let vcores_consumed = input_partition_ids.len() as u32;
                 Ok(TaskDescription {
                     session_id,
-                    partition,
+                    key,
                     stage_attempt_num: stage.stage_attempt_num,
-                    task_id,
                     task_attempt,
+                    global_input_partition_ids: input_partition_ids,
+                    vcores_consumed,
                     plan: stage.plan.clone(),
                     session_config: self.session_config.clone()
                 })
@@ -1537,7 +1577,9 @@ impl Debug for StaticExecutionGraph {
     }
 }
 
-/// Creates a new `TaskInfo` for a task that is about to be scheduled on an executor.
+/// Creates a new `TaskInfo` for a task that is about to be scheduled on an
+/// executor. The caller sets `global_input_partition_ids` to the partitions this task
+/// will process (bind loops draw the slice from `stage.pending`).
 pub fn create_task_info(executor_id: String, task_id: usize) -> TaskInfo {
     TaskInfo {
         task_id,
@@ -1545,12 +1587,13 @@ pub fn create_task_info(executor_id: String, task_id: usize) -> TaskInfo {
             .duration_since(UNIX_EPOCH)
             .unwrap()
             .as_millis(),
-        // Those times will be updated when the task finish
         launch_time: 0,
         start_exec_time: 0,
         end_exec_time: 0,
         finish_time: 0,
         task_status: task_status::Status::Running(RunningTask { executor_id }),
+        global_input_partition_ids: vec![],
+        vcores_consumed: 0,
     }
 }
 
@@ -1633,14 +1676,12 @@ impl ExecutionPlanVisitor for ExecutionStageBuilder {
         plan: &dyn ExecutionPlan,
     ) -> std::result::Result<bool, Self::Error> {
         // Handle both ShuffleWriterExec and SortShuffleWriterExec
-        if let Some(shuffle_write) = plan.as_any().downcast_ref::<ShuffleWriterExec>() {
+        if let Some(shuffle_write) = plan.downcast_ref::<ShuffleWriterExec>() {
             self.current_stage_id = shuffle_write.stage_id();
-        } else if let Some(shuffle_write) =
-            plan.as_any().downcast_ref::<SortShuffleWriterExec>()
-        {
+        } else if let Some(shuffle_write) = plan.downcast_ref::<SortShuffleWriterExec>() {
             self.current_stage_id = shuffle_write.stage_id();
         } else if let Some(unresolved_shuffle) =
-            plan.as_any().downcast_ref::<UnresolvedShuffleExec>()
+            plan.downcast_ref::<UnresolvedShuffleExec>()
         {
             if let Some(output_links) =
                 self.output_links.get_mut(&unresolved_shuffle.stage_id)
@@ -1668,20 +1709,29 @@ impl ExecutionPlanVisitor for ExecutionStageBuilder {
 
 /// Represents the basic unit of work for the Ballista executor.
 ///
-/// A `TaskDescription` contains all the information needed to execute
-/// one partition of one stage on a single executor task slot.
+/// One `TaskDescription` drives all of `global_input_partition_ids`'s partitions
+/// through one plan-Arc on the assigned executor.
 #[derive(Clone)]
 pub struct TaskDescription {
     /// The session ID associated with this task's job.
     pub session_id: String,
-    /// The partition identifier (job_id, stage_id, partition_id).
-    pub partition: PartitionId,
+    /// Task locator: `(job_id, stage_id, task_id)`. `task_id` is this task's
+    /// append-order slot in `RunningStage.task_infos`.
+    pub key: TaskKey,
     /// The attempt number for this stage (for retry tracking).
     pub stage_attempt_num: usize,
-    /// Unique task ID within the execution graph.
-    pub task_id: usize,
     /// The attempt number for this specific task (for retry tracking).
     pub task_attempt: usize,
+    /// The partitions (real plan input indices) this task will process.
+    /// Populated at bind time from the stage's `PendingPartitions` cursor
+    /// sized to the assigned executor's free vcores. Baked into `plan`
+    /// via `task_builder::restrict_plan_to_partitions` before dispatch.
+    pub global_input_partition_ids: Vec<usize>,
+    /// Vcores this task consumed from the executor's budget at bind time
+    /// (`min(global_input_partition_ids.len(), budget.vcores)` for non-collapse
+    /// stages, `1` for collapse stages). Forwarded to the executor over the
+    /// wire so the memory pool can be sized proportionally.
+    pub vcores_consumed: u32,
     /// The physical execution plan to run for this task.
     pub plan: Arc<dyn ExecutionPlan>,
     /// Session configuration for this task's execution context.
@@ -1693,13 +1743,12 @@ impl Debug for TaskDescription {
         let plan = DisplayableExecutionPlan::new(self.plan.as_ref()).indent(false);
         write!(
             f,
-            "TaskDescription[session_id: {},job: {}, stage: {}.{}, partition: {} task_id {}, task attempt {}]\n{}",
+            "TaskDescription[session_id: {},job: {}, stage: {}.{}, task_id: {}, task attempt {}]\n{}",
             self.session_id,
-            self.partition.job_id,
-            self.partition.stage_id,
+            self.key.job_id,
+            self.key.stage_id,
             self.stage_attempt_num,
-            self.partition.partition_id,
-            self.task_id,
+            self.key.task_id,
             self.task_attempt,
             plan
         )
@@ -1710,18 +1759,14 @@ impl TaskDescription {
     /// Returns the number of output partitions this task will produce.
     pub fn get_output_partition_number(&self) -> usize {
         // Try ShuffleWriterExec first
-        if let Some(shuffle_writer) =
-            self.plan.as_any().downcast_ref::<ShuffleWriterExec>()
-        {
+        if let Some(shuffle_writer) = self.plan.downcast_ref::<ShuffleWriterExec>() {
             return shuffle_writer
                 .shuffle_output_partitioning()
                 .map(|partitioning| partitioning.partition_count())
                 .unwrap_or(1);
         }
         // Try SortShuffleWriterExec
-        if let Some(shuffle_writer) =
-            self.plan.as_any().downcast_ref::<SortShuffleWriterExec>()
-        {
+        if let Some(shuffle_writer) = self.plan.downcast_ref::<SortShuffleWriterExec>() {
             return shuffle_writer
                 .shuffle_output_partitioning()
                 .partition_count();
@@ -1767,10 +1812,11 @@ mod test {
     use ballista_core::error::Result;
     use ballista_core::serde::protobuf::{
         self, ExecutionError, FailedTask, FetchPartitionError, IoError, JobStatus,
-        TaskKilled, failed_task, job_status,
+        TaskKilled, failed_task, job_status, task_status,
     };
 
     use crate::state::execution_graph::ExecutionGraph;
+    use crate::state::execution_stage::ExecutionStage;
     use crate::test_utils::{
         mock_completed_task, mock_executor, mock_failed_task,
         revive_graph_and_complete_next_stage,
@@ -1779,6 +1825,33 @@ mod test {
         test_union_all_plan, test_union_plan,
     };
 
+    #[tokio::test]
+    async fn test_intermediate_stage_ids() {
+        // A simple aggregation produces a 2-stage graph: one intermediate
+        // stage (non-empty output_links) feeding one final stage (empty
+        // output_links).
+        let graph = test_aggregation_plan(4).await;
+
+        assert_eq!(graph.stages().len(), 2);
+
+        // Exactly one final stage.
+        let final_count = graph
+            .stages()
+            .values()
+            .filter(|s| s.output_links().is_empty())
+            .count();
+        assert_eq!(final_count, 1);
+
+        // Intermediate = all - final = exactly one stage, and none of the
+        // returned ids is a final stage.
+        let intermediate = graph.intermediate_stage_ids();
+        assert_eq!(intermediate.len(), 1);
+        for id in &intermediate {
+            let stage = graph.stages().get(&(*id as usize)).unwrap();
+            assert!(!stage.output_links().is_empty());
+        }
+    }
+
     #[tokio::test]
     async fn test_fail_job_sets_end_time_and_failed_metadata() -> Result<()> {
         let mut graph = test_aggregation_plan(4).await;
@@ -2114,12 +2187,17 @@ mod test {
         assert_eq!(agg_graph.available_tasks(), 1);
 
         let mut last_attempt = 0;
-        // 2rd task's attempts
+        // 2rd task's attempts.
+        //
+        // Under the append-only task_infos model, each retry gets a fresh
+        // task_id (rather than reusing the original task's slot). The
+        // global_input_partition_ids is what stably identifies "which task is being
+        // retried" — assert on that instead of task_id.
         for attempt in 1..5 {
             if let Some(task2_attempt) = agg_graph.pop_next_task(&executor.id)? {
                 assert_eq!(
-                    task2_attempt.partition.partition_id,
-                    task2.partition.partition_id
+                    task2_attempt.global_input_partition_ids,
+                    task2.global_input_partition_ids
                 );
                 assert_eq!(task2_attempt.task_attempt, attempt);
                 last_attempt = task2_attempt.task_attempt;
@@ -2161,6 +2239,70 @@ mod test {
         Ok(())
     }
 
+    // Aborting a running job (failure or cancellation) must transition every
+    // running stage to Failed and return its in-flight tasks for cancellation.
+    // `abort_running` is the shared teardown invoked by `abort_job`.
+    #[tokio::test]
+    async fn test_abort_running_cancels_stages_and_returns_inflight_tasks() -> Result<()>
+    {
+        let executor = mock_executor("executor-id1".to_string());
+        let mut graph = test_join_plan(2).await;
+
+        // Call revive to move the two leaf Resolved stages to Running
+        graph.revive();
+        assert!(
+            graph.running_stages().len() >= 2,
+            "expected two concurrently running leaf stages, found {:?}",
+            graph.running_stages()
+        );
+
+        // Dispatch a task so there is an in-flight task to cancel
+        let _task = graph.pop_next_task(&executor.id)?.unwrap();
+
+        // Aborting cancels every running stage and returns its in-flight tasks
+        let cancelled = graph.abort_running("job aborted".to_string());
+
+        assert!(
+            !cancelled.is_empty(),
+            "abort_running must return the in-flight tasks to cancel"
+        );
+        assert!(
+            graph.running_stages().is_empty(),
+            "every running stage must be cancelled, found {:?}",
+            graph.running_stages()
+        );
+        assert!(
+            matches!(
+                graph.status(),
+                JobStatus {
+                    status: Some(job_status::Status::Failed(_)),
+                    ..
+                }
+            ),
+            "the job must be Failed after abort"
+        );
+
+        // In-flight tasks of the cancelled stage are recorded as Failed(TaskKilled)
+        let has_killed_task = graph.stages.values().any(|stage| match stage {
+            ExecutionStage::Failed(failed) => failed.task_infos.iter().any(|info| {
+                matches!(
+                    &info.task_status,
+                    task_status::Status::Failed(FailedTask {
+                        failed_reason: Some(failed_task::FailedReason::TaskKilled(_)),
+                        ..
+                    })
+                )
+            }),
+            _ => false,
+        });
+        assert!(
+            has_killed_task,
+            "in-flight tasks must be recorded as Failed(TaskKilled) after abort"
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_long_delayed_failed_task_after_executor_lost() -> Result<()> {
         let executor1 = mock_executor("executor-id1".to_string());
diff --git a/ballista/scheduler/src/state/execution_graph_dot.rs b/ballista/scheduler/src/state/execution_graph_dot.rs
index f43772dd0f..eaef4dc5e9 100644
--- a/ballista/scheduler/src/state/execution_graph_dot.rs
+++ b/ballista/scheduler/src/state/execution_graph_dot.rs
@@ -148,7 +148,7 @@ fn write_plan_recursive(
     let node_name = format!("{prefix}_{i}");
     let display_name = get_operator_name(plan);
 
-    if let Some(reader) = plan.as_any().downcast_ref::<ShuffleReaderExec>() {
+    if let Some(reader) = plan.downcast_ref::<ShuffleReaderExec>() {
         for part in &reader.partition {
             for loc in part {
                 state
@@ -156,7 +156,7 @@ fn write_plan_recursive(
                     .insert(node_name.clone(), loc.partition_id.stage_id);
             }
         }
-    } else if let Some(reader) = plan.as_any().downcast_ref::<UnresolvedShuffleExec>() {
+    } else if let Some(reader) = plan.downcast_ref::<UnresolvedShuffleExec>() {
         state.readers.insert(node_name.clone(), reader.stage_id);
     }
 
@@ -230,9 +230,9 @@ fn sanitize(str: &str, max_len: Option<usize>) -> String {
 }
 #[allow(deprecated)]
 fn get_operator_name(plan: &dyn ExecutionPlan) -> String {
-    if let Some(exec) = plan.as_any().downcast_ref::<FilterExec>() {
+    if let Some(exec) = plan.downcast_ref::<FilterExec>() {
         format!("Filter: {}", exec.predicate())
-    } else if let Some(exec) = plan.as_any().downcast_ref::<ProjectionExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<ProjectionExec>() {
         let expr = exec
             .expr()
             .iter()
@@ -241,7 +241,7 @@ fn get_operator_name(plan: &dyn ExecutionPlan) -> String {
             .collect::<Vec<String>>()
             .join(", ");
         format!("Projection: {}", sanitize_dot_label(&expr))
-    } else if let Some(exec) = plan.as_any().downcast_ref::<SortExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<SortExec>() {
         let sort_expr = exec
             .expr()
             .iter()
@@ -257,7 +257,7 @@ fn get_operator_name(plan: &dyn ExecutionPlan) -> String {
             .collect::<Vec<String>>()
             .join(", ");
         format!("Sort: {}", sanitize_dot_label(&sort_expr))
-    } else if let Some(exec) = plan.as_any().downcast_ref::<AggregateExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<AggregateExec>() {
         let group_exprs_with_alias = exec.group_expr().expr();
         let group_expr = group_exprs_with_alias
             .iter()
@@ -277,19 +277,19 @@ aggr=[{}]",
             sanitize_dot_label(&group_expr),
             sanitize_dot_label(&aggr_expr)
         )
-    } else if let Some(exec) = plan.as_any().downcast_ref::<CoalesceBatchesExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<CoalesceBatchesExec>() {
         format!("CoalesceBatches [batchSize={}]", exec.target_batch_size())
-    } else if let Some(exec) = plan.as_any().downcast_ref::<CoalescePartitionsExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<CoalescePartitionsExec>() {
         format!(
             "CoalescePartitions [{}]",
             format_partitioning(exec.properties().output_partitioning().clone())
         )
-    } else if let Some(exec) = plan.as_any().downcast_ref::<RepartitionExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<RepartitionExec>() {
         format!(
             "RepartitionExec [{}]",
             format_partitioning(exec.properties().output_partitioning().clone())
         )
-    } else if let Some(exec) = plan.as_any().downcast_ref::<HashJoinExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<HashJoinExec>() {
         let join_expr = exec
             .on()
             .iter()
@@ -308,49 +308,46 @@ filter_expr={}",
             sanitize_dot_label(&join_expr),
             sanitize_dot_label(&filter_expr)
         )
-    } else if plan.as_any().downcast_ref::<CrossJoinExec>().is_some() {
+    } else if plan.downcast_ref::<CrossJoinExec>().is_some() {
         "CrossJoin".to_string()
-    } else if plan.as_any().downcast_ref::<UnionExec>().is_some() {
+    } else if plan.downcast_ref::<UnionExec>().is_some() {
         "Union".to_string()
-    } else if let Some(exec) = plan.as_any().downcast_ref::<UnresolvedShuffleExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<UnresolvedShuffleExec>() {
         format!("UnresolvedShuffleExec [stage_id={}]", exec.stage_id)
-    } else if let Some(exec) = plan.as_any().downcast_ref::<ShuffleReaderExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<ShuffleReaderExec>() {
         format!("ShuffleReader [{} partitions]", exec.partition.len())
-    } else if let Some(exec) = plan.as_any().downcast_ref::<ShuffleWriterExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<ShuffleWriterExec>() {
         format!(
             "ShuffleWriter [{} partitions]",
             exec.input_partition_count()
         )
-    } else if let Some(exec) = plan.as_any().downcast_ref::<SortShuffleWriterExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<SortShuffleWriterExec>() {
         format!(
             "SortShuffleWriter [{} partitions]",
             exec.input_partition_count()
         )
-    } else if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>() {
-        let config = if let Some(config) =
-            exec.data_source().as_any().downcast_ref::<FileScanConfig>()
-        {
-            get_file_scan(config)
-        } else if let Some(_config) = exec
-            .data_source()
-            .as_any()
-            .downcast_ref::<MemorySourceConfig>()
-        {
-            "Memory".to_string()
-        } else {
-            "Unknown".to_string()
-        };
+    } else if let Some(exec) = plan.downcast_ref::<DataSourceExec>() {
+        let config =
+            if let Some(config) = exec.data_source().downcast_ref::<FileScanConfig>() {
+                get_file_scan(config)
+            } else if let Some(_config) =
+                exec.data_source().downcast_ref::<MemorySourceConfig>()
+            {
+                "Memory".to_string()
+            } else {
+                "Unknown".to_string()
+            };
 
         let parts = exec.properties().output_partitioning().partition_count();
 
         format!("DataSourceExec: ({config}) [{parts} partitions]")
-    } else if let Some(exec) = plan.as_any().downcast_ref::<GlobalLimitExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<GlobalLimitExec>() {
         format!(
             "GlobalLimit(skip={}, fetch={:?})",
             exec.skip(),
             exec.fetch()
         )
-    } else if let Some(exec) = plan.as_any().downcast_ref::<LocalLimitExec>() {
+    } else if let Some(exec) = plan.downcast_ref::<LocalLimitExec>() {
         format!("LocalLimit({})", exec.fetch())
     } else {
         debug!("Unknown physical operator when producing DOT graph: {plan:?}");
@@ -589,6 +586,14 @@ filter_expr="]
             .options_mut()
             .optimizer
             .enable_round_robin_repartition = false;
+        config
+            .options_mut()
+            .optimizer
+            .hash_join_single_partition_threshold = 0;
+        config
+            .options_mut()
+            .optimizer
+            .hash_join_single_partition_threshold_rows = 0;
         let ctx = SessionContext::new_with_config(config);
         let schema = Arc::new(Schema::new(vec![
             Field::new("a", DataType::UInt32, false),
@@ -627,6 +632,14 @@ filter_expr="]
             .options_mut()
             .optimizer
             .enable_round_robin_repartition = false;
+        config
+            .options_mut()
+            .optimizer
+            .hash_join_single_partition_threshold = 0;
+        config
+            .options_mut()
+            .optimizer
+            .hash_join_single_partition_threshold_rows = 0;
         let ctx = SessionContext::new_with_config(config);
         let schema =
             Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, false)]));
diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs
index a9695d3b8b..2b17a1e16d 100644
--- a/ballista/scheduler/src/state/execution_stage.rs
+++ b/ballista/scheduler/src/state/execution_stage.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::convert::TryInto;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
@@ -35,7 +35,7 @@ use ballista_core::error::{BallistaError, Result};
 use ballista_core::execution_plans::{ShuffleWriterExec, SortShuffleWriterExec};
 use ballista_core::serde::protobuf::failed_task::FailedReason;
 use ballista_core::serde::protobuf::{
-    FailedTask, OperatorMetricsSet, ResultLost, SuccessfulTask, TaskStatus,
+    FailedTask, OperatorMetricsSet, ResultLost, SuccessfulTask, TaskKilled, TaskStatus,
 };
 use ballista_core::serde::protobuf::{RunningTask, task_status};
 use ballista_core::serde::scheduler::PartitionLocation;
@@ -107,6 +107,30 @@ impl ExecutionStage {
             ExecutionStage::Failed(stage) => stage.plan.as_ref(),
         }
     }
+
+    /// Get the output links for this stage. An empty slice means this is a
+    /// final stage in the `ExecutionGraph`.
+    pub fn output_links(&self) -> &[usize] {
+        match self {
+            ExecutionStage::UnResolved(stage) => &stage.output_links,
+            ExecutionStage::Resolved(stage) => &stage.output_links,
+            ExecutionStage::Running(stage) => &stage.output_links,
+            ExecutionStage::Successful(stage) => &stage.output_links,
+            ExecutionStage::Failed(stage) => &stage.output_links,
+        }
+    }
+
+    /// TaskInfos indexed by task_id (the append-order slot) for stages
+    /// that have started binding tasks. Returns `None` for
+    /// UnResolved/Resolved stages that haven't bound anything yet.
+    pub fn task_infos(&self) -> Option<&[TaskInfo]> {
+        match self {
+            ExecutionStage::Running(stage) => Some(&stage.task_infos),
+            ExecutionStage::Successful(stage) => Some(&stage.task_infos),
+            ExecutionStage::Failed(stage) => Some(&stage.task_infos),
+            ExecutionStage::UnResolved(_) | ExecutionStage::Resolved(_) => None,
+        }
+    }
 }
 
 /// For a stage whose input stages are not all completed, we say it's a unresolved stage
@@ -167,8 +191,8 @@ pub struct RunningStage {
     pub stage_attempt_num: usize,
     /// Stage activation time (when was stage become running) in millis
     pub stage_running_time: u128,
-    /// Total number of partitions for this stage.
-    /// This stage will produce on task for partition.
+    /// Total plan input partitions for this stage (frozen at resolve).
+    /// Num tasks is emergent from `pending` / `task_infos.len()`, not this.
     pub partitions: usize,
     /// Stage ID of the stage that will take this stages outputs as inputs.
     /// If `output_links` is empty then this the final stage in the `ExecutionGraph`
@@ -177,11 +201,18 @@ pub struct RunningStage {
     pub inputs: HashMap<usize, StageOutput>,
     /// `ExecutionPlan` for this stage
     pub plan: Arc<dyn ExecutionPlan>,
-    /// TaskInfo of each already scheduled task. If info is None, the partition has not yet been scheduled.
-    /// The index of the Vec is the task's partition id
-    pub task_infos: Vec<Option<TaskInfo>>,
-    /// Track the number of failures for each partition's task attempts.
-    /// The index of the Vec is the task's partition id.
+    /// Cursor over partitions still waiting to be scheduled. Bind time
+    /// pulls a chunk sized to the executor's free vcores; failed
+    /// partitions are pushed back to the front for re-attempt.
+    pub pending: PendingPartitions,
+    /// TaskInfo of every task ever started for this stage (append-only).
+    /// Indexed by task_id (the bind-order slot). Retries append new
+    /// entries with fresh task_ids rather than reusing slots.
+    pub task_infos: Vec<TaskInfo>,
+    /// Number of times each plan input partition has been tried and
+    /// failed. Indexed by partition id (real plan input index), not by
+    /// task_id. When any partition exceeds `stage_max_failures`, the
+    /// stage is failed.
     pub task_failure_numbers: Vec<usize>,
     /// Combined metrics of the already finished tasks in the stage, If it is None, no task is finished yet.
     pub stage_metrics: Option<Vec<MetricsSet>>,
@@ -231,9 +262,9 @@ pub struct FailedStage {
     pub output_links: Vec<usize>,
     /// `ExecutionPlan` for this stage
     pub plan: Arc<dyn ExecutionPlan>,
-    /// TaskInfo of each already scheduled tasks. If info is None, the partition has not yet been scheduled
-    /// The index of the Vec is the task's partition id
-    pub task_infos: Vec<Option<TaskInfo>>,
+    /// TaskInfo of every task ever started before the stage failed.
+    /// Append-only, indexed by task_id (bind order).
+    pub task_infos: Vec<TaskInfo>,
     /// Combined metrics of the already finished tasks in the stage, If it is None, no task is finished yet.
     #[allow(dead_code)] // not used at the moment, will be used later
     pub stage_metrics: Option<Vec<MetricsSet>>,
@@ -241,6 +272,72 @@ pub struct FailedStage {
     pub error_message: String,
 }
 
+/// Cursor over the partitions remaining to be scheduled for a stage.
+///
+/// One task processes a slice of partitions (up to `exec.vcores`). Rather
+/// than pre-computing num_tasks at resolve time and pre-allocating
+/// `Vec<Option<TaskInfo>>`, the number of tasks is emergent: at bind time
+/// each executor's free-vcore budget pulls a chunk of partitions off the
+/// queue. Retries push failed partitions to the front so they get
+/// re-attempted before any fresh partition.
+#[derive(Clone, Debug)]
+pub struct PendingPartitions {
+    /// Total plan input partitions this stage must process (frozen).
+    total: usize,
+    /// Partitions waiting to be scheduled. Front = next out.
+    queue: VecDeque<usize>,
+}
+
+impl PendingPartitions {
+    /// Create a cursor covering partitions `[0..total)`.
+    pub fn new(total: usize) -> Self {
+        Self {
+            total,
+            queue: (0..total).collect(),
+        }
+    }
+
+    /// Create an empty cursor sized to `total` partitions. Callers populate
+    /// via `reschedule` (used by `SuccessfulStage::to_running` where only
+    /// the failed/lost subset needs re-processing).
+    pub fn empty(total: usize) -> Self {
+        Self {
+            total,
+            queue: VecDeque::new(),
+        }
+    }
+
+    /// Take up to `max` partitions for the next task. Returns an empty
+    /// vec when the stage has no more work.
+    pub fn next_slice(&mut self, max: usize) -> Vec<usize> {
+        let take = max.min(self.queue.len());
+        self.queue.drain(..take).collect()
+    }
+
+    /// Push failed partitions to the front of the queue so the next bind
+    /// picks them up before any fresh partition.
+    pub fn reschedule(&mut self, partitions: impl IntoIterator<Item = usize>) {
+        for p in partitions.into_iter().collect::<Vec<_>>().into_iter().rev() {
+            self.queue.push_front(p);
+        }
+    }
+
+    /// True when there are no more partitions to hand out.
+    pub fn is_empty(&self) -> bool {
+        self.queue.is_empty()
+    }
+
+    /// Total partitions still unassigned.
+    pub fn remaining(&self) -> usize {
+        self.queue.len()
+    }
+
+    /// Total plan input partition count (frozen at construction).
+    pub fn total_partitions(&self) -> usize {
+        self.total
+    }
+}
+
 /// Information about a task's execution lifecycle and current status.
 #[derive(Clone)]
 #[allow(dead_code)] // we may use the fields later
@@ -259,6 +356,19 @@ pub struct TaskInfo {
     pub finish_time: u128,
     /// Current status of the task (Running, Successful, Failed).
     pub task_status: task_status::Status,
+    /// Plan input partitions this task is (or was) processing. Mirrors the
+    /// `global_input_partition_ids` on `TaskDescription`; needed here so executor-loss
+    /// and retry paths know which partitions to push back to `pending`,
+    /// and per-partition failure bookkeeping can locate entries in
+    /// `task_failure_numbers`.
+    pub global_input_partition_ids: Vec<usize>,
+    /// Vcores this task consumed from the executor's budget at bind time.
+    /// Usually equals `global_input_partition_ids.len()` (one vcore per
+    /// packed partition), but for collapse-input tasks that consume the
+    /// executor's entire remaining budget it is `min(slice_len, budget)`.
+    /// Used to refund the exact amount when the task completes so the
+    /// executor's vcore budget stays consistent across bind/refund.
+    pub vcores_consumed: u32,
 }
 
 impl UnresolvedStage {
@@ -437,7 +547,7 @@ impl ResolvedStage {
         last_attempt_failure_reasons: HashSet<String>,
         session_config: Arc<SessionConfig>,
     ) -> Self {
-        let partitions = get_stage_partitions(plan.clone());
+        let partitions = stage_input_partitions(&plan);
 
         Self {
             stage_id,
@@ -515,7 +625,8 @@ impl RunningStage {
             output_links,
             inputs,
             plan,
-            task_infos: vec![None; partitions],
+            pending: PendingPartitions::new(partitions),
+            task_infos: Vec::new(),
             task_failure_numbers: vec![0; partitions],
             stage_metrics: None,
             session_config,
@@ -524,19 +635,7 @@ impl RunningStage {
 
     /// Converts this running stage to a successful stage after all tasks complete.
     pub fn to_successful(&self) -> SuccessfulStage {
-        let task_infos = self
-            .task_infos
-            .iter()
-            .enumerate()
-            .map(|(partition_id, info)| {
-                info.clone().unwrap_or_else(|| {
-                    panic!(
-                        "TaskInfo for task {}.{}/{} should not be none",
-                        self.stage_id, self.stage_attempt_num, partition_id
-                    )
-                })
-            })
-            .collect();
+        let task_infos = self.task_infos.clone();
         let stage_metrics = self.stage_metrics.clone().unwrap_or_else(|| {
             warn!("The metrics for stage {} should not be none", self.stage_id);
             vec![]
@@ -554,15 +653,36 @@ impl RunningStage {
         }
     }
 
-    /// Converts this running stage to a failed stage with the given error message.
+    /// Converts this running stage to a failed stage. Still-running tasks are recorded
+    /// as cancelled (`Failed(TaskKilled)`) since failing the stage cancels them.
     pub fn to_failed(&self, error_message: String) -> FailedStage {
+        let task_infos = self
+            .task_infos
+            .iter()
+            .map(|info| {
+                if matches!(info.task_status, task_status::Status::Running(_)) {
+                    TaskInfo {
+                        task_status: task_status::Status::Failed(FailedTask {
+                            error: "killed".to_string(),
+                            retryable: false,
+                            count_to_failures: false,
+                            failed_reason: Some(FailedReason::TaskKilled(TaskKilled {})),
+                        }),
+                        ..info.clone()
+                    }
+                } else {
+                    info.clone()
+                }
+            })
+            .collect();
+
         FailedStage {
             stage_id: self.stage_id,
             stage_attempt_num: self.stage_attempt_num,
             partitions: self.partitions,
             output_links: self.output_links.clone(),
             plan: self.plan.clone(),
-            task_infos: self.task_infos.clone(),
+            task_infos,
             stage_metrics: self.stage_metrics.clone(),
             error_message,
         }
@@ -587,80 +707,91 @@ impl RunningStage {
         Ok(unresolved)
     }
 
-    /// Returns `true` if all tasks for this stage are successful
+    /// Returns `true` if every plan input partition has been processed
+    /// successfully at least once and no partitions remain in `pending`.
+    /// Retried tasks may leave Failed entries in `task_infos`; a partition
+    /// counts as covered if any Successful task's slice includes it.
     pub fn is_successful(&self) -> bool {
-        self.task_infos.iter().all(|info| {
-            matches!(
-                info,
-                Some(TaskInfo {
-                    task_status: task_status::Status::Successful(_),
-                    ..
-                })
-            )
-        })
+        if !self.pending.is_empty() {
+            return false;
+        }
+        let mut covered = vec![false; self.partitions];
+        for info in &self.task_infos {
+            if matches!(info.task_status, task_status::Status::Successful(_)) {
+                for p in &info.global_input_partition_ids {
+                    covered[*p] = true;
+                }
+            }
+        }
+        covered.iter().all(|c| *c)
     }
 
-    /// Returns the number of successful tasks
+    /// Returns the number of task attempts that reached Successful status
+    /// (multiple attempts may exist for the same partition after retries).
     pub fn successful_tasks(&self) -> usize {
         self.task_infos
             .iter()
-            .filter(|info| {
-                matches!(
-                    info,
-                    Some(TaskInfo {
-                        task_status: task_status::Status::Successful(_),
-                        ..
-                    })
-                )
-            })
+            .filter(|info| matches!(info.task_status, task_status::Status::Successful(_)))
             .count()
     }
 
-    /// Returns the number of scheduled tasks
+    /// Returns the number of tasks that have been scheduled (started) so far
+    /// — every entry in `task_infos` counts, including retries and losses.
     pub fn scheduled_tasks(&self) -> usize {
-        self.task_infos.iter().filter(|s| s.is_some()).count()
+        self.task_infos.len()
     }
 
-    /// Returns a vector of currently running tasks in this stage
-    pub fn running_tasks(&self) -> Vec<(usize, usize, usize, String)> {
+    /// Returns a vector of currently running tasks in this stage: tuples of
+    /// `(task_id, stage_id, executor_id)`. `task_id` is the task's append
+    /// slot in `task_infos`.
+    pub fn running_tasks(&self) -> Vec<(usize, usize, String)> {
         self.task_infos
             .iter()
             .enumerate()
-            .filter_map(|(partition, info)| match info {
-                Some(TaskInfo {task_id,
-                         task_status: task_status::Status::Running(RunningTask { executor_id }), ..}) => {
-                    Some((*task_id, self.stage_id, partition, executor_id.clone()))
+            .filter_map(|(task_id, info)| match &info.task_status {
+                task_status::Status::Running(RunningTask { executor_id }) => {
+                    Some((task_id, self.stage_id, executor_id.clone()))
                 }
                 _ => None,
             })
             .collect()
     }
 
-    /// Returns the number of tasks in this stage which are available for scheduling.
-    /// If the stage is not yet resolved, then this will return `0`, otherwise it will
-    /// return the number of tasks where the task info is not yet set.
+    /// Returns the number of plan input partitions still waiting to be
+    /// handed to a task. This is what scheduling decisions draw from: as
+    /// executors free vcores, `pending.next_slice(exec.vcores)` drains
+    /// this pool into fresh tasks.
     pub fn available_tasks(&self) -> usize {
-        self.task_infos.iter().filter(|s| s.is_none()).count()
+        self.pending.remaining()
     }
 
-    /// Update the TaskInfo for task partition
-    pub fn update_task_info(&mut self, partition_id: usize, status: TaskStatus) -> bool {
-        debug!("Updating TaskInfo for partition {partition_id}");
-        let Some(task_info) = self.task_infos[partition_id].as_ref() else {
-            warn!(
-                "Ignore TaskStatus update for partition {partition_id} because the task was already reset (executor lost)"
-            );
-            return false;
-        };
-        let task_id = task_info.task_id;
-        if (status.task_id as usize) < task_id {
+    /// Apply a status update for the task at `task_id` (its append slot in
+    /// `task_infos`).
+    ///
+    /// Rejects (returns false) if the task's status is already terminal
+    /// Failed with a lost/killed reason (i.e., `reset_tasks` moved its
+    /// partitions back to pending because the executor died) — a late
+    /// status from that attempt should be ignored.
+    ///
+    /// On success, updates the task's status and adjusts per-partition
+    /// failure counters using the task's `global_input_partition_ids`.
+    pub fn update_task_info(&mut self, task_id: usize, status: TaskStatus) -> bool {
+        debug!("Updating TaskInfo for task_id {task_id}");
+        let task_info = &self.task_infos[task_id];
+        if let task_status::Status::Failed(FailedTask {
+            failed_reason:
+                Some(FailedReason::TaskKilled(_)) | Some(FailedReason::ResultLost(_)),
+            ..
+        }) = &task_info.task_status
+        {
             warn!(
-                "Ignore TaskStatus update with TID {} because there is more recent task attempt with TID {} running for partition {}",
-                status.task_id, task_id, partition_id
+                "Ignore TaskStatus update for task_id {task_id} because it was already reset (executor lost)"
             );
             return false;
         }
         let scheduled_time = task_info.scheduled_time;
+        let global_input_partition_ids = task_info.global_input_partition_ids.clone();
+        let vcores_consumed = task_info.vcores_consumed;
         let task_status = status.status.unwrap();
         let updated_task_info = TaskInfo {
             task_id,
@@ -673,16 +804,23 @@ impl RunningStage {
                 .unwrap()
                 .as_millis(),
             task_status: task_status.clone(),
+            global_input_partition_ids: global_input_partition_ids.clone(),
+            vcores_consumed,
         };
-        self.task_infos[partition_id] = Some(updated_task_info);
+        self.task_infos[task_id] = updated_task_info;
 
-        if let task_status::Status::Failed(failed_task) = task_status {
-            // if the failed task is retryable, increase the task failure count for this partition
-            if failed_task.retryable {
-                self.task_failure_numbers[partition_id] += 1;
+        match task_status {
+            task_status::Status::Failed(failed_task) if failed_task.retryable => {
+                for p in &global_input_partition_ids {
+                    self.task_failure_numbers[*p] += 1;
+                }
             }
-        } else {
-            self.task_failure_numbers[partition_id] = 0;
+            task_status::Status::Successful(_) => {
+                for p in &global_input_partition_ids {
+                    self.task_failure_numbers[*p] = 0;
+                }
+            }
+            _ => {}
         }
         true
     }
@@ -690,7 +828,7 @@ impl RunningStage {
     /// update and upsert the task metrics to the stage metrics
     pub fn update_task_metrics(
         &mut self,
-        partition: usize,
+        task_id: usize,
         metrics: Vec<OperatorMetricsSet>,
     ) -> Result<()> {
         // For some cases, task metrics not set, especially for testings.
@@ -698,6 +836,9 @@ impl RunningStage {
             return Ok(());
         }
 
+        let global_partitions =
+            self.task_infos[task_id].global_input_partition_ids.clone();
+
         let new_metrics_set = if let Some(combined_metrics) = &mut self.stage_metrics {
             if metrics.len() != combined_metrics.len() {
                 return Err(BallistaError::Internal(format!(
@@ -706,29 +847,29 @@ impl RunningStage {
                     self.stage_id,
                     metrics.len(),
                     combined_metrics.len(),
-                    partition
+                    task_id
                 )));
             }
             let metrics_values_array = metrics
                 .into_iter()
-                .map(|ms| Self::metrics_set_from_task_metrics(ms, partition))
+                .map(|ms| Self::metrics_set_from_task_metrics(ms, &global_partitions))
                 .collect::<Result<Vec<_>>>()?;
 
             combined_metrics
                 .iter_mut()
                 .zip(metrics_values_array)
-                .map(|(existing_metrics, new_partition_metrics)| {
-                    Self::upsert_metrics_set_for_partition(
+                .map(|(existing_metrics, new_task_metrics)| {
+                    Self::upsert_metrics_set_for_task(
                         existing_metrics,
-                        new_partition_metrics,
-                        partition,
+                        new_task_metrics,
+                        &global_partitions,
                     )
                 })
                 .collect()
         } else {
             metrics
                 .into_iter()
-                .map(|ms| Self::metrics_set_from_task_metrics(ms, partition))
+                .map(|ms| Self::metrics_set_from_task_metrics(ms, &global_partitions))
                 .collect::<Result<Vec<_>>>()?
         };
         self.stage_metrics = Some(new_metrics_set);
@@ -736,76 +877,158 @@ impl RunningStage {
         Ok(())
     }
 
-    /// Converts task metrics into a metrics set for a specific partition
+    /// Convert a task's operator-metrics snapshot into a `MetricsSet`, mapping
+    /// each metric's local partition index to a stage-global partition id via
+    /// `global_partitions` (the task's slice of stage-global partition ids).
+    ///
+    /// Three cases, keyed on the local partition value the executor put on
+    /// the wire:
+    ///
+    /// 1. `local < global_partitions.len()` — restricted-arm metric (see
+    ///    [`crate::state::task_builder`]). Local index maps 1:1 to
+    ///    `global_partitions[local]`. Honest per-input-partition detail.
+    /// 2. `global_partitions.len() == 1` — collapse task; all metrics
+    ///    belong to the single output partition.
+    /// 3. `local >= global_partitions.len()` — under-collect arm. The
+    ///    plan's subtree below a `CoalescePartitionsExec` /
+    ///    `SortPreservingMergeExec` / SinglePartition join-build side
+    ///    keeps the full upstream partition count, so operators there
+    ///    stamp metrics with local indices unrelated to the task's slice
+    ///    (0..upstream_full_count). Cross-product the metric against every
+    ///    slice member so the aggregate mirrors what master reported
+    ///    (`N tasks × 1 task-partition each = N buckets, 1 bucket per
+    ///    stage-global partition`).
+    ///
+    /// TODO(#2038 metrics story): neither master nor this design is
+    /// actually *correct*. Master's per-partition attribution was a lie in
+    /// two ways:
+    ///   (a) it discarded DataFusion's real per-input-partition tag and
+    ///       re-labelled every metric with the task's own partition id,
+    ///       and
+    ///   (b) it counted under-collect work in each task's bucket, so the
+    ///       cross-stage aggregate over-reported the physical cost by a
+    ///       factor of the stage's task count.
+    /// This design preserves (a)'s per-partition-tag behaviour for wire
+    /// compatibility with the REST API and TUI, and cross-products
+    /// under-collect metrics to keep the same UI shape master had — but it
+    /// *hides* multi-partition-tasks' real efficiency win (fewer tasks =
+    /// fewer redundant under-collect executions) behind the same inflated
+    /// aggregate. The honest representation is probably: one bucket per
+    /// arm per task, with the arm identity (which shuffle input? which
+    /// side of a join? which scope?) attached as labels — but the REST
+    /// API / TUI don't yet have a way to display arm-scoped metrics, and
+    /// that redesign is out of scope for this PR. Open question: what does
+    /// the per-partition/per-arm view actually *mean* under
+    /// multi-partition tasks, and how should the API surface it?
+    /// [[project-prs-2038-2088]]
     fn metrics_set_from_task_metrics(
         metrics: OperatorMetricsSet,
-        partition: usize,
+        global_partitions: &[usize],
     ) -> Result<MetricsSet> {
         let mut metrics_set = MetricsSet::new();
-        for metric in metrics.metrics {
-            let metric_value: MetricValue = metric.try_into()?;
-            metrics_set.push(Arc::new(Metric::new(metric_value, Some(partition))));
+        for proto_metric in metrics.metrics {
+            let local_partition = proto_metric.partition.map(|p| p as usize);
+            let inner = proto_metric.metric.ok_or_else(|| {
+                BallistaError::Internal(
+                    "OperatorMetric.metric is None while folding task metrics".into(),
+                )
+            })?;
+            let metric_value: MetricValue = inner.try_into()?;
+            let tags: Vec<Option<usize>> = match local_partition {
+                Some(local) if local < global_partitions.len() => {
+                    vec![Some(global_partitions[local])]
+                }
+                Some(_) if global_partitions.len() == 1 => {
+                    vec![Some(global_partitions[0])]
+                }
+                Some(_) => global_partitions.iter().map(|&p| Some(p)).collect(),
+                None => vec![None],
+            };
+            for tag in tags {
+                metrics_set.push(Arc::new(Metric::new(metric_value.clone(), tag)));
+            }
         }
         Ok(metrics_set)
     }
 
-    /// Upserts raw metrics from a completed task into the stage metrics
-    pub fn upsert_metrics_set_for_partition(
+    /// Upsert a task's raw metrics into the stage metrics. Task metrics are
+    /// snapshots, so any prior entry for one of the task's global partitions
+    /// is replaced by the new snapshot.
+    pub fn upsert_metrics_set_for_task(
         existing_metrics: &mut MetricsSet,
-        new_partition_metrics: MetricsSet,
-        partition: usize,
+        new_task_metrics: MetricsSet,
+        global_partitions: &[usize],
     ) -> MetricsSet {
         let mut updated_metrics = MetricsSet::new();
-        // Task metrics are snapshots, so replace any prior metrics for this partition.
         for metric in existing_metrics.iter() {
-            if metric.partition() != Some(partition) {
+            let owned_by_task = metric
+                .partition()
+                .map(|p| global_partitions.contains(&p))
+                .unwrap_or(false);
+            if !owned_by_task {
                 updated_metrics.push(metric.clone());
             }
         }
-        for metric in new_partition_metrics.iter() {
+        for metric in new_task_metrics.iter() {
             updated_metrics.push(metric.clone());
         }
         updated_metrics
     }
 
-    /// Returns the number of times the task for the given partition has failed.
-    pub fn task_failure_number(&self, partition_id: usize) -> usize {
-        self.task_failure_numbers[partition_id]
+    /// Returns the highest per-partition failure count across the
+    /// partitions in the given task's slice. Callers use this to decide
+    /// whether any partition in the task has exhausted its retry budget.
+    pub fn task_failure_number(&self, task_id: usize) -> usize {
+        self.task_infos[task_id]
+            .global_input_partition_ids
+            .iter()
+            .map(|p| self.task_failure_numbers[*p])
+            .max()
+            .unwrap_or(0)
     }
 
-    /// Reset the task info for the given task partition. This should be called when a task failed and need to be
-    /// re-scheduled.
-    pub fn reset_task_info(&mut self, partition_id: usize) {
-        self.task_infos[partition_id] = None;
+    /// Mark the task as lost/killed and push its partitions back to the
+    /// front of `pending` so they are retried on the next bind. Does not
+    /// touch failure counts — those are updated in `update_task_info`.
+    pub fn reset_task_info(&mut self, task_id: usize) {
+        let task = &mut self.task_infos[task_id];
+        let partitions = task.global_input_partition_ids.clone();
+        task.task_status = task_status::Status::Failed(FailedTask {
+            error: "task reset for retry".to_string(),
+            retryable: true,
+            count_to_failures: false,
+            failed_reason: Some(FailedReason::TaskKilled(TaskKilled {})),
+        });
+        self.pending.reschedule(partitions);
     }
 
-    /// Reset the running and completed tasks on a given executor
-    /// Returns the number of running tasks that were reset
+    /// Reset the running and completed tasks on a given executor by
+    /// marking their `TaskInfo` as `Failed(ResultLost)` and pushing their
+    /// partition slices back to `pending`. Returns the number of tasks
+    /// reset.
     pub fn reset_tasks(&mut self, executor: &str) -> usize {
         let mut reset = 0;
+        let mut to_reschedule: Vec<usize> = vec![];
         for task in self.task_infos.iter_mut() {
-            match task {
-                Some(TaskInfo {
-                    task_status: task_status::Status::Running(RunningTask { executor_id }),
-                    ..
-                }) if *executor == *executor_id => {
-                    *task = None;
-                    reset += 1;
-                }
-                Some(TaskInfo {
-                    task_status:
-                        task_status::Status::Successful(SuccessfulTask {
-                            executor_id,
-                            partitions: _,
-                        }),
-                    ..
-                }) if *executor == *executor_id => {
-                    *task = None;
-                    reset += 1;
-                }
-                _ => {}
+            let matches_exec = match &task.task_status {
+                task_status::Status::Running(RunningTask { executor_id })
+                | task_status::Status::Successful(SuccessfulTask {
+                    executor_id, ..
+                }) => executor == executor_id,
+                _ => false,
+            };
+            if matches_exec {
+                task.task_status = task_status::Status::Failed(FailedTask {
+                    error: format!("Task failure due to Executor {executor} lost"),
+                    retryable: true,
+                    count_to_failures: false,
+                    failed_reason: Some(FailedReason::ResultLost(ResultLost {})),
+                });
+                to_reschedule.extend(task.global_input_partition_ids.iter().copied());
+                reset += 1;
             }
         }
+        self.pending.reschedule(to_reschedule);
         reset
     }
 
@@ -861,18 +1084,32 @@ impl Debug for RunningStage {
 }
 
 impl SuccessfulStage {
-    /// Change to the running state and bump the stage attempt number
+    /// Change to the running state and bump the stage attempt number.
+    ///
+    /// task_infos is append-only across retries — the same partition may
+    /// appear in multiple entries (an old Failed one from a lost executor
+    /// plus a later Successful one from the retry). Walk newest-to-oldest
+    /// and, for each partition, only consider the *latest* attempt: if it's
+    /// Successful the partition is done; otherwise it goes back to pending.
+    /// Naive iteration would push the same partition multiple times.
     pub fn to_running(&self) -> RunningStage {
-        let mut task_infos: Vec<Option<TaskInfo>> = Vec::new();
-        for task in self.task_infos.iter() {
-            match task {
-                TaskInfo {
-                    task_status: task_status::Status::Successful(_),
-                    ..
-                } => task_infos.push(Some(task.clone())),
-                _ => task_infos.push(None),
+        let mut pending = PendingPartitions::empty(self.partitions);
+        let mut seen = vec![false; self.partitions];
+        let mut to_reschedule: Vec<usize> = vec![];
+        for task in self.task_infos.iter().rev() {
+            let needs_reschedule =
+                !matches!(task.task_status, task_status::Status::Successful(_));
+            for &p in &task.global_input_partition_ids {
+                if p < seen.len() && !seen[p] {
+                    seen[p] = true;
+                    if needs_reschedule {
+                        to_reschedule.push(p);
+                    }
+                }
             }
         }
+        to_reschedule.sort_unstable();
+        pending.reschedule(to_reschedule);
         let stage_metrics = if self.stage_metrics.is_empty() {
             None
         } else {
@@ -885,7 +1122,8 @@ impl SuccessfulStage {
             output_links: self.output_links.clone(),
             inputs: self.inputs.clone(),
             plan: self.plan.clone(),
-            task_infos,
+            pending,
+            task_infos: self.task_infos.clone(),
             // It is Ok to forget the previous task failure attempts
             task_failure_numbers: vec![0; self.partitions],
             stage_metrics,
@@ -897,39 +1135,30 @@ impl SuccessfulStage {
         }
     }
 
-    /// Reset the successful tasks on a given executor
-    /// Returns the number of running tasks that were reset
+    /// Mark successful tasks on a lost executor as `Failed(ResultLost)` so
+    /// `to_running` will reschedule their partitions on the next attempt.
+    /// Returns the number of tasks reset.
     pub fn reset_tasks(&mut self, executor: &str) -> usize {
         let mut reset = 0;
         let failure_reason = format!("Task failure due to Executor {executor} lost");
         for task in self.task_infos.iter_mut() {
-            match task {
-                TaskInfo {
-                    task_id,
-                    scheduled_time,
-                    task_status:
-                        task_status::Status::Successful(SuccessfulTask {
-                            executor_id, ..
-                        }),
-                    ..
-                } if *executor == *executor_id => {
-                    *task = TaskInfo {
-                        task_id: *task_id,
-                        scheduled_time: *scheduled_time,
-                        launch_time: 0,
-                        start_exec_time: 0,
-                        end_exec_time: 0,
-                        finish_time: 0,
-                        task_status: task_status::Status::Failed(FailedTask {
-                            error: failure_reason.clone(),
-                            retryable: true,
-                            count_to_failures: false,
-                            failed_reason: Some(FailedReason::ResultLost(ResultLost {})),
-                        }),
-                    };
-                    reset += 1;
-                }
-                _ => {}
+            let hit = matches!(
+                &task.task_status,
+                task_status::Status::Successful(SuccessfulTask { executor_id, .. })
+                    if executor == executor_id
+            );
+            if hit {
+                task.launch_time = 0;
+                task.start_exec_time = 0;
+                task.end_exec_time = 0;
+                task.finish_time = 0;
+                task.task_status = task_status::Status::Failed(FailedTask {
+                    error: failure_reason.clone(),
+                    retryable: true,
+                    count_to_failures: false,
+                    failed_reason: Some(FailedReason::ResultLost(ResultLost {})),
+                });
+                reset += 1;
             }
         }
         reset
@@ -953,31 +1182,22 @@ impl Debug for SuccessfulStage {
 }
 
 impl FailedStage {
-    /// Returns the number of successful tasks
+    /// Returns the number of task attempts that reached Successful status
+    /// before the stage failed.
     pub fn successful_tasks(&self) -> usize {
         self.task_infos
             .iter()
-            .filter(|info| {
-                matches!(
-                    info,
-                    Some(TaskInfo {
-                        task_status: task_status::Status::Successful(_),
-                        ..
-                    })
-                )
-            })
+            .filter(|info| matches!(info.task_status, task_status::Status::Successful(_)))
             .count()
     }
-    /// Returns the number of scheduled tasks
+    /// Returns the number of tasks scheduled (every task_infos entry).
     pub fn scheduled_tasks(&self) -> usize {
-        self.task_infos.iter().filter(|s| s.is_some()).count()
+        self.task_infos.len()
     }
 
-    /// Returns the number of tasks in this stage which are available for scheduling.
-    /// If the stage is not yet resolved, then this will return `0`, otherwise it will
-    /// return the number of tasks where the task status is not yet set.
+    /// A failed stage has no schedulable work remaining.
     pub fn available_tasks(&self) -> usize {
-        self.task_infos.iter().filter(|s| s.is_none()).count()
+        0
     }
 }
 
@@ -1000,20 +1220,32 @@ impl Debug for FailedStage {
     }
 }
 
-/// Get the total number of partitions for a stage with plan.
-/// Only for shuffle writers, the input partition count and the output partition count
-/// will be different. Here, we should use the input partition count.
-fn get_stage_partitions(plan: Arc<dyn ExecutionPlan>) -> usize {
-    // Try ShuffleWriterExec first
-    if let Some(shuffle_writer) = plan.as_any().downcast_ref::<ShuffleWriterExec>() {
-        return shuffle_writer.input_partition_count();
-    }
-    // Try SortShuffleWriterExec
-    if let Some(shuffle_writer) = plan.as_any().downcast_ref::<SortShuffleWriterExec>() {
-        return shuffle_writer.input_partition_count();
+/// Total plan input partitions a stage must process. Frozen at resolve.
+/// Number of TASKS is emergent — bind time draws slices from a cursor
+/// (`PendingPartitions`) sized to whichever executor's free vcores show
+/// up, so one 16-vcore exec covers 16 partitions in a single task while
+/// four 4-vcore execs cover the same 16 in four tasks.
+///
+/// The count comes from the shuffle writer's immediate child's
+/// `output_partitioning().partition_count()`. That is the number of
+/// independent output-partition polling contexts the writer needs to
+/// drive, which matches the "one tokio worker per vcore" invariant: one
+/// task = one `execute(i)` on the top plan = one primary tokio pipeline.
+/// Anything an internal `RepartitionExec` spawns is per-plan-instance
+/// machinery shared across those polls and does not become a separate
+/// scheduling unit.
+fn stage_input_partitions(plan: &Arc<dyn ExecutionPlan>) -> usize {
+    if plan.downcast_ref::<ShuffleWriterExec>().is_some()
+        || plan.downcast_ref::<SortShuffleWriterExec>().is_some()
+    {
+        plan.children()[0]
+            .properties()
+            .output_partitioning()
+            .partition_count()
+    } else {
+        plan.properties().output_partitioning().partition_count()
     }
-    // Fallback to output partitioning
-    plan.properties().output_partitioning().partition_count()
+    .max(1)
 }
 
 /// This data structure collects the partition locations for an `ExecutionStage`.
@@ -1107,13 +1339,12 @@ mod tests {
         )
     }
 
-    fn make_task_status(task_id: u32, partition_id: u32) -> TaskStatus {
+    fn make_task_status(task_id: u32) -> TaskStatus {
         TaskStatus {
             task_id,
             job_id: "test-job".to_string(),
             stage_id: 1,
             stage_attempt_num: 0,
-            partition_id,
             launch_time: 100,
             start_exec_time: 200,
             end_exec_time: 300,
@@ -1128,93 +1359,98 @@ mod tests {
     fn make_operator_metrics_set(
         output_rows: u64,
         elapsed_compute_nanos: u64,
+    ) -> OperatorMetricsSet {
+        make_operator_metrics_set_at(0, output_rows, elapsed_compute_nanos)
+    }
+
+    /// Builds an `OperatorMetricsSet` mimicking an executor-side snapshot
+    /// where each metric carries the local partition index (0..slice_len).
+    fn make_operator_metrics_set_at(
+        local_partition: u32,
+        output_rows: u64,
+        elapsed_compute_nanos: u64,
     ) -> OperatorMetricsSet {
         OperatorMetricsSet {
             metrics: vec![
                 OperatorMetric {
                     metric: Some(operator_metric::Metric::OutputRows(output_rows)),
+                    partition: Some(local_partition),
                 },
                 OperatorMetric {
                     metric: Some(operator_metric::Metric::ElapseTime(
                         elapsed_compute_nanos,
                     )),
+                    partition: Some(local_partition),
                 },
             ],
         }
     }
 
-    /// Regression test: `update_task_info` must not panic when the task slot
-    /// is `None` (task was reset after executor heartbeat timeout).
-    #[test]
-    fn test_update_task_info_after_reset_does_not_panic() {
-        let mut stage = make_running_stage(2);
-
-        // Both task slots start as None (not yet scheduled).
-        // Simulates receiving a status update for a task that was already
-        // reset (e.g., executor heartbeat timed out).
-        let status = make_task_status(0, 0);
-        let result = stage.update_task_info(0, status);
-
-        // Should return false (update rejected), not panic.
-        assert!(!result);
-    }
-
-    /// Verify that a normal update succeeds when the task slot is populated.
-    #[test]
-    fn test_update_task_info_normal_update_succeeds() {
-        let mut stage = make_running_stage(2);
-
-        // Simulate scheduling the task: populate the task slot.
-        stage.task_infos[0] = Some(TaskInfo {
-            task_id: 0,
+    /// Push one Running task_info covering the given partitions onto the
+    /// stage (test helper mirroring what a real bind would do).
+    fn append_running_task(
+        stage: &mut RunningStage,
+        task_id: usize,
+        executor: &str,
+        partitions: Vec<usize>,
+    ) {
+        let vcores_consumed = partitions.len() as u32;
+        stage.task_infos.push(TaskInfo {
+            task_id,
             scheduled_time: 50,
-            launch_time: 0,
-            start_exec_time: 0,
+            launch_time: 100,
+            start_exec_time: 200,
             end_exec_time: 0,
             finish_time: 0,
             task_status: task_status::Status::Running(RunningTask {
-                executor_id: "executor-1".to_string(),
+                executor_id: executor.to_string(),
             }),
+            global_input_partition_ids: partitions,
+            vcores_consumed,
         });
+    }
+
+    /// Verify that a normal status update transitions the task to Successful.
+    #[test]
+    fn test_update_task_info_normal_update_succeeds() {
+        let mut stage = make_running_stage(2);
+        append_running_task(&mut stage, 0, "executor-1", vec![0]);
 
-        let status = make_task_status(0, 0);
+        let status = make_task_status(0);
         let result = stage.update_task_info(0, status);
 
         assert!(result);
         assert!(matches!(
-            stage.task_infos[0].as_ref().unwrap().task_status,
+            stage.task_infos[0].task_status,
             task_status::Status::Successful(_)
         ));
     }
 
-    /// After reset_tasks sets a slot to None, update_task_info must not panic.
+    /// After `reset_tasks` marks a task as ResultLost, a late status update
+    /// from the (now lost) executor must be rejected without panicking.
     #[test]
     fn test_update_task_info_after_executor_lost() {
         let mut stage = make_running_stage(2);
+        append_running_task(&mut stage, 0, "executor-1", vec![0]);
+        append_running_task(&mut stage, 1, "executor-1", vec![1]);
+        // Both partitions were drained by binds; nothing left pending.
+        stage.pending.next_slice(2);
 
-        // Populate tasks as running on executor-1.
-        for i in 0..2 {
-            stage.task_infos[i] = Some(TaskInfo {
-                task_id: i,
-                scheduled_time: 50,
-                launch_time: 100,
-                start_exec_time: 200,
-                end_exec_time: 0,
-                finish_time: 0,
-                task_status: task_status::Status::Running(RunningTask {
-                    executor_id: "executor-1".to_string(),
-                }),
-            });
-        }
-
-        // Executor heartbeat times out - tasks are reset.
+        // Executor heartbeat times out - tasks are marked lost and
+        // their partitions get pushed back to pending.
         let reset_count = stage.reset_tasks("executor-1");
         assert_eq!(reset_count, 2);
-        assert!(stage.task_infos[0].is_none());
-        assert!(stage.task_infos[1].is_none());
+        assert_eq!(stage.pending.remaining(), 2);
+        assert!(matches!(
+            &stage.task_infos[0].task_status,
+            task_status::Status::Failed(FailedTask {
+                failed_reason: Some(FailedReason::ResultLost(_)),
+                ..
+            })
+        ));
 
-        // Executor sends a late status update for partition 0.
-        let status = make_task_status(0, 0);
+        // Late status update from the (now lost) executor.
+        let status = make_task_status(0);
         let result = stage.update_task_info(0, status);
 
         // Should gracefully reject the update, not panic.
@@ -1224,6 +1460,10 @@ mod tests {
     #[test]
     fn test_update_task_metrics_keeps_raw_partition_snapshots() {
         let mut stage = make_running_stage(3);
+        // One task per partition — the classic Spark-style layout.
+        append_running_task(&mut stage, 0, "executor-1", vec![0]);
+        append_running_task(&mut stage, 1, "executor-1", vec![1]);
+        append_running_task(&mut stage, 2, "executor-1", vec![2]);
 
         stage
             .update_task_metrics(0, vec![make_operator_metrics_set(100, 10)])
@@ -1293,4 +1533,158 @@ mod tests {
         assert_eq!(aggregated.output_rows(), Some(650));
         assert_eq!(aggregated.elapsed_compute().unwrap(), 65);
     }
+
+    /// A task that covers multiple global partitions must file each partition's
+    /// metric separately, using the task's `global_input_partition_ids` to map
+    /// the executor's local partition index onto the global partition id.
+    #[test]
+    fn test_update_task_metrics_multi_partition_task_preserves_per_partition_detail() {
+        let mut stage = make_running_stage(4);
+        // One task covering four global partitions [3, 7, 5, 1] (order matters:
+        // the plan sees them as local 0..3 in this order).
+        append_running_task(&mut stage, 0, "executor-1", vec![3, 7, 5, 1]);
+
+        stage
+            .update_task_metrics(
+                0,
+                vec![OperatorMetricsSet {
+                    metrics: vec![
+                        // Local 0 (global 3): 100 rows
+                        OperatorMetric {
+                            metric: Some(operator_metric::Metric::OutputRows(100)),
+                            partition: Some(0),
+                        },
+                        // Local 1 (global 7): 4000 rows — the skewed partition
+                        OperatorMetric {
+                            metric: Some(operator_metric::Metric::OutputRows(4000)),
+                            partition: Some(1),
+                        },
+                        // Local 2 (global 5): 200 rows
+                        OperatorMetric {
+                            metric: Some(operator_metric::Metric::OutputRows(200)),
+                            partition: Some(2),
+                        },
+                        // Local 3 (global 1): 150 rows
+                        OperatorMetric {
+                            metric: Some(operator_metric::Metric::OutputRows(150)),
+                            partition: Some(3),
+                        },
+                    ],
+                }],
+            )
+            .unwrap();
+
+        let metrics = stage.stage_metrics.as_ref().unwrap();
+        let operator_metrics = &metrics[0];
+
+        // Each global partition shows up under its own bucket — no collapsing.
+        let mut per_partition: Vec<(Option<usize>, usize)> = operator_metrics
+            .iter()
+            .filter_map(|metric| match metric.value() {
+                MetricValue::OutputRows(value) => {
+                    Some((metric.partition(), value.value()))
+                }
+                _ => None,
+            })
+            .collect();
+        per_partition.sort_by_key(|(partition, _)| *partition);
+        assert_eq!(
+            per_partition,
+            vec![
+                (Some(1), 150),
+                (Some(3), 100),
+                (Some(5), 200),
+                (Some(7), 4000),
+            ],
+        );
+
+        // Re-report from the same task — snapshots for its four partitions are
+        // replaced, not appended.
+        stage
+            .update_task_metrics(
+                0,
+                vec![OperatorMetricsSet {
+                    metrics: vec![OperatorMetric {
+                        metric: Some(operator_metric::Metric::OutputRows(4200)),
+                        partition: Some(1),
+                    }],
+                }],
+            )
+            .unwrap();
+
+        let metrics = stage.stage_metrics.as_ref().unwrap();
+        let operator_metrics = &metrics[0];
+        let output_rows: Vec<(Option<usize>, usize)> = operator_metrics
+            .iter()
+            .filter_map(|metric| match metric.value() {
+                MetricValue::OutputRows(value) => {
+                    Some((metric.partition(), value.value()))
+                }
+                _ => None,
+            })
+            .collect();
+        assert_eq!(output_rows, vec![(Some(7), 4200)]);
+    }
+
+    /// A metric whose local partition index doesn't fit the task's slice
+    /// comes from an under-collect subtree (below a `CoalescePartitionsExec`,
+    /// `SortPreservingMergeExec`, or a SinglePartition-requiring join build
+    /// side, per `task_builder::child_scopes`). Those operators keep the
+    /// full upstream partition count and stamp metrics with indices unrelated
+    /// to the task's slice. We cross-product each such metric against every
+    /// slice member, matching the pre-branch model where every task's under-
+    /// collect metric was re-labelled with the task's single partition id
+    /// (see the TODO on `metrics_set_from_task_metrics`).
+    #[test]
+    fn test_update_task_metrics_under_collect_cross_products_across_slice() {
+        let mut stage = make_running_stage(4);
+        // Task covers slice [3, 7, 5, 1] (four global partitions).
+        append_running_task(&mut stage, 0, "executor-1", vec![3, 7, 5, 1]);
+
+        // Single metric with local partition=8 — cannot fit slice of len 4.
+        // Represents e.g. a build-side ShuffleReader below a CoalescePartitions
+        // reading upstream partition 8 of 16.
+        stage
+            .update_task_metrics(
+                0,
+                vec![OperatorMetricsSet {
+                    metrics: vec![OperatorMetric {
+                        metric: Some(operator_metric::Metric::OutputRows(500)),
+                        partition: Some(8),
+                    }],
+                }],
+            )
+            .unwrap();
+
+        let metrics = stage.stage_metrics.as_ref().unwrap();
+        let operator_metrics = &metrics[0];
+
+        // The single incoming metric gets one entry per slice member.
+        let mut per_partition: Vec<(Option<usize>, usize)> = operator_metrics
+            .iter()
+            .filter_map(|metric| match metric.value() {
+                MetricValue::OutputRows(value) => {
+                    Some((metric.partition(), value.value()))
+                }
+                _ => None,
+            })
+            .collect();
+        per_partition.sort_by_key(|(p, _)| *p);
+        assert_eq!(
+            per_partition,
+            vec![
+                (Some(1), 500),
+                (Some(3), 500),
+                (Some(5), 500),
+                (Some(7), 500),
+            ],
+        );
+
+        // Aggregate: 500 × slice_len = 2000. Matches the pre-branch shape
+        // where each of the task's owned partitions would each report the
+        // full under-collect cost. Physical work was 1× 500 — the 4× is the
+        // per-partition lie called out in the TODO.
+        let aggregated = operator_metrics.aggregate_by_name();
+        assert_eq!(aggregated.output_rows(), Some(2000));
+    }
 }
diff --git a/ballista/scheduler/src/state/executor_manager.rs b/ballista/scheduler/src/state/executor_manager.rs
index 1871514a7b..d8a8331778 100644
--- a/ballista/scheduler/src/state/executor_manager.rs
+++ b/ballista/scheduler/src/state/executor_manager.rs
@@ -60,14 +60,15 @@ type ExecutorClients = Arc<DashMap<String, ExecutorGrpcClient<Channel>>>;
 /// - Cleaning up job data on executors
 #[derive(Clone)]
 pub struct ExecutorManager {
-    /// Cluster state for tracking executor registration and task slots.
+    /// Cluster state for tracking executor registration and vcores.
     cluster_state: Arc<dyn ClusterState>,
     /// Scheduler configuration.
     config: Arc<SchedulerConfig>,
     /// Cached gRPC clients for communicating with executors.
     clients: ExecutorClients,
-    /// Jobs pending cleanup on each executor.
-    pending_cleanup_jobs: Arc<DashMap<String, HashSet<JobId>>>,
+    /// Per-executor pending cleanups: job id -> stage ids to remove
+    /// (empty stage ids ⇒ remove the whole job dir).
+    pending_cleanup_jobs: Arc<DashMap<String, HashMap<JobId, Vec<u32>>>>,
     /// Configuration for gRPC client connections.
     grpc_client_config: GrpcClientConfig,
 }
@@ -127,7 +128,7 @@ impl ExecutorManager {
             .await
     }
 
-    /// Returns reserved task slots to the pool of available slots.
+    /// Returns reserved vcores to the pool of available slots.
     ///
     /// This operation is atomic: either all slots are returned or none are.
     pub async fn unbind_tasks(&self, executor_slots: Vec<ExecutorSlot>) -> Result<()> {
@@ -145,7 +146,6 @@ impl ExecutorManager {
                 task_id: task_info.task_id as u32,
                 job_id: task_info.job_id.into(),
                 stage_id: task_info.stage_id as u32,
-                partition_id: task_info.partition_id as u32,
             });
         }
 
@@ -191,7 +191,9 @@ impl ExecutorManager {
         let executor_manager = self.clone();
         tokio::spawn(async move {
             tokio::time::sleep(Duration::from_secs(clean_up_interval)).await;
-            executor_manager.clean_up_job_data_inner(job_id).await;
+            executor_manager
+                .clean_up_job_data_inner(job_id, vec![])
+                .await;
         });
     }
 
@@ -199,13 +201,49 @@ impl ExecutorManager {
     pub fn clean_up_job_data(&self, job_id: JobId) {
         let executor_manager = self.clone();
         tokio::spawn(async move {
-            executor_manager.clean_up_job_data_inner(job_id).await;
+            executor_manager
+                .clean_up_job_data_inner(job_id, vec![])
+                .await;
+        });
+    }
+
+    /// Immediately reclaim intermediate-stage shuffle data for a successful job,
+    /// retaining the rest of the job dir (e.g. the final-stage output) for the
+    /// existing delayed whole-job cleanup. No-op if `remove_stage_ids` is empty.
+    ///
+    /// Contract: call this AT MOST ONCE per job, passing the complete set of
+    /// intermediate stage ids. It is not designed for per-stage invocation.
+    /// In poll-based scheduling the pending-cleanup value is a `Vec<u32>` keyed
+    /// by job, and `HashMap::insert` REPLACES on key collision (it does not
+    /// merge). One intermediate call followed by the delayed whole-job cleanup
+    /// (empty `remove_stage_ids`) is correct: the empty/whole-job entry
+    /// supersedes and removes the entire job dir. But two DISTINCT partial calls
+    /// for the same job before the executor polls would drop the earlier stage
+    /// ids. Do not "fix" this by merging/extending: an empty vec means "remove
+    /// the whole job dir", so extending could never represent whole-job
+    /// supersession.
+    pub(crate) fn clean_up_intermediate_job_data(
+        &self,
+        job_id: JobId,
+        remove_stage_ids: Vec<u32>,
+    ) {
+        if remove_stage_ids.is_empty() {
+            return;
+        }
+        let executor_manager = self.clone();
+        tokio::spawn(async move {
+            executor_manager
+                .clean_up_job_data_inner(job_id, remove_stage_ids)
+                .await;
         });
     }
 
     /// 1. Push strategy: Send rpc to Executors to clean up the job data
     /// 2. Poll strategy: Save cleanup job ids and send them to executors
-    async fn clean_up_job_data_inner(&self, job_id: JobId) {
+    ///
+    /// `remove_stage_ids` empty ⇒ remove the whole job dir; non-empty ⇒ remove
+    /// only those stage subdirs.
+    async fn clean_up_job_data_inner(&self, job_id: JobId, remove_stage_ids: Vec<u32>) {
         let alive_executors = self.get_alive_executors();
 
         for executor in alive_executors {
@@ -215,10 +253,12 @@ impl ExecutorManager {
                 if let Ok(mut client) =
                     self.get_client(&executor, &self.grpc_client_config).await
                 {
+                    let remove_stage_ids = remove_stage_ids.clone();
                     tokio::spawn(async move {
                         if let Err(err) = client
                             .remove_job_data(RemoveJobDataParams {
                                 job_id: job_id_clone,
+                                remove_stage_ids,
                             })
                             .await
                         {
@@ -234,7 +274,7 @@ impl ExecutorManager {
                 self.pending_cleanup_jobs
                     .entry(executor)
                     .or_default()
-                    .insert(job_id.clone());
+                    .insert(job_id.clone(), remove_stage_ids.clone());
             }
         }
     }
@@ -291,15 +331,15 @@ impl ExecutorManager {
     /// For push-based scheduling, use [`Self::register_executor`] instead.
     pub async fn save_executor_metadata(&self, metadata: ExecutorMetadata) -> Result<()> {
         trace!(
-            "save executor metadata {} with {} task slots (pull-based registration)",
-            metadata.id, metadata.specification.task_slots
+            "save executor metadata {} with {} vcores (pull-based registration)",
+            metadata.id, metadata.specification.vcores
         );
         self.cluster_state.save_executor_metadata(metadata).await
     }
 
     /// Registers the executor with the scheduler for push-based task scheduling.
     ///
-    /// This saves both the executor metadata and available task slots to persistent state.
+    /// This saves both the executor metadata and vcore inventory to persistent state.
     /// For pull-based scheduling, use [`Self::save_executor_metadata`] instead.
     pub async fn register_executor(
         &self,
@@ -307,8 +347,8 @@ impl ExecutorManager {
         specification: ExecutorData,
     ) -> Result<()> {
         debug!(
-            "registering executor {} with {} task slots (push-based registration)",
-            metadata.id, specification.total_task_slots
+            "registering executor {} with {} vcores (push-based registration)",
+            metadata.id, specification.total_vcores
         );
 
         ExecutorManager::test_connectivity(&metadata).await?;
@@ -387,10 +427,13 @@ impl ExecutorManager {
         Ok(())
     }
 
-    pub(crate) fn drain_pending_cleanup_jobs(&self, executor_id: &str) -> HashSet<JobId> {
+    pub(crate) fn drain_pending_cleanup_jobs(
+        &self,
+        executor_id: &str,
+    ) -> Vec<(JobId, Vec<u32>)> {
         self.pending_cleanup_jobs
             .remove(executor_id)
-            .map(|(_, jobs)| jobs)
+            .map(|(_, jobs)| jobs.into_iter().collect())
             .unwrap_or_default()
     }
 
diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs
index 7268942356..d86488e8fb 100644
--- a/ballista/scheduler/src/state/mod.rs
+++ b/ballista/scheduler/src/state/mod.rs
@@ -17,7 +17,7 @@
 
 use crate::cluster::{BallistaCluster, BoundTask, ExecutorSlot};
 use crate::config::SchedulerConfig;
-use crate::scheduler_server::event::QueryStageSchedulerEvent;
+use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan};
 use crate::state::execution_graph::TaskDescription;
 use crate::state::executor_manager::ExecutorManager;
 use crate::state::session_manager::SessionManager;
@@ -28,7 +28,6 @@ use ballista_core::serde::BallistaCodec;
 use ballista_core::serde::protobuf::TaskStatus;
 use ballista_core::{JobId, JobStatusSubscriber};
 use datafusion::execution::context::SessionContext;
-use datafusion::logical_expr::LogicalPlan;
 use datafusion_proto::logical_plan::AsLogicalPlan;
 use datafusion_proto::physical_plan::AsExecutionPlan;
 use log::{debug, error, info, warn};
@@ -50,6 +49,8 @@ pub mod execution_stage;
 pub mod executor_manager;
 /// Session state management.
 pub mod session_manager;
+/// Per-task plan rewriter (restrict scan/shuffle-reader to task's slice).
+pub mod task_builder;
 /// Task scheduling and lifecycle management.
 pub mod task_manager;
 
@@ -267,7 +268,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerState<T,
             HashMap<(JobId, usize), Vec<TaskDescription>>,
         > = HashMap::new();
         for (executor_id, task) in bound_tasks.into_iter() {
-            let stage_key = (task.partition.job_id.clone(), task.partition.stage_id);
+            let stage_key = (task.key.job_id.clone(), task.key.stage_id);
             if let Some(tasks) = executor_stage_assignments.get_mut(&executor_id) {
                 if let Some(executor_stage_tasks) = tasks.get_mut(&stage_key) {
                     executor_stage_tasks.push(task);
@@ -366,21 +367,14 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerState<T,
         job_id: &JobId,
         job_name: &str,
         session_ctx: Arc<SessionContext>,
-        logical_plan: &LogicalPlan,
+        plan: &SubmitPlan,
         queued_at: u64,
         subscriber: Option<JobStatusSubscriber>,
     ) -> Result<()> {
         let start = Instant::now();
 
         self.task_manager
-            .submit_job(
-                job_id,
-                job_name,
-                session_ctx,
-                logical_plan,
-                queued_at,
-                subscriber,
-            )
+            .submit_plan(job_id, job_name, session_ctx, plan, queued_at, subscriber)
             .await?;
 
         let elapsed = start.elapsed();
@@ -390,8 +384,16 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerState<T,
         Ok(())
     }
 
-    /// Spawn a delayed future to clean up job data on both Scheduler and Executors
-    pub(crate) fn clean_up_successful_job(&self, job_id: JobId) {
+    /// Immediately reclaim intermediate shuffle data, then spawn a delayed
+    /// future to clean up the remaining job data (final-stage output + job
+    /// state) on both Scheduler and Executors.
+    pub(crate) fn clean_up_successful_job(
+        &self,
+        job_id: JobId,
+        intermediate_stage_ids: Vec<u32>,
+    ) {
+        self.executor_manager
+            .clean_up_intermediate_job_data(job_id.clone(), intermediate_stage_ids);
         self.executor_manager.clean_up_job_data_delayed(
             job_id.clone(),
             self.config.finished_job_data_clean_up_interval_seconds,
diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs
new file mode 100644
index 0000000000..37da2865c3
--- /dev/null
+++ b/ballista/scheduler/src/state/task_builder.rs
@@ -0,0 +1,591 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Per-task plan rewriter.
+//!
+//! Before shipping a stage's plan to an executor, the scheduler restricts the
+//! plan's leaves (Scan file groups, ShuffleReader partition locations) to the
+//! task's assigned partition slice. The executor then just runs whatever it's
+//! given — no operator needs to know its task's global identity, and any
+//! within-stage operator's `output_partitioning()` naturally reflects the
+//! restricted count.
+//!
+//! Restriction is *scoped*: a leaf below a collapse (`CoalescePartitionsExec`,
+//! `SortPreservingMergeExec`, or the `SinglePartition`-requiring build side
+//! of a join) must read the entire upstream, not the task's slice — otherwise
+//! each of N sibling tasks would collapse only 1/N of the input, producing
+//! partial results downstream tries to merge (e.g. the wrong scalar threshold
+//! from a HAVING subquery).
+//!
+//! Scoping is expressed as an `under_collect: bool` parameter threaded down
+//! the recursion. The call stack is the tree walk's natural stack; scope
+//! flows from parent to descendants via function arguments, so sibling
+//! subtrees never share state and there's no traversal-order dependency.
+
+use ballista_core::execution_plans::ShuffleReaderExec;
+use datafusion::datasource::memory::MemorySourceConfig;
+use datafusion::datasource::physical_plan::{
+    FileGroup, FileScanConfig, FileScanConfigBuilder,
+};
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::error::Result;
+use datafusion::physical_expr::Distribution;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::joins::{HashJoinExec, NestedLoopJoinExec};
+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
+use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
+use datafusion::physical_plan::union::UnionExec;
+use datafusion::physical_plan::with_new_children_if_necessary;
+use datafusion::physical_plan::{ExecutionPlan, Partitioning};
+use log::warn;
+use std::any::Any;
+use std::sync::Arc;
+
+/// Restrict `plan` so that its leaves only see the given `partitions`, unless
+/// a leaf sits below a collapse operator (`CoalescePartitionsExec`,
+/// `SortPreservingMergeExec`, or a `SinglePartition`-requiring join build
+/// side) — those leaves keep the full upstream so the collapse sees every
+/// partition.
+pub fn restrict_plan_to_partitions(
+    plan: Arc<dyn ExecutionPlan>,
+    partitions: &[usize],
+) -> Result<Arc<dyn ExecutionPlan>> {
+    restrict(plan, partitions, /* under_collect */ false)
+}
+
+/// Recursive worker. `under_collect` is the scope inherited from ancestors:
+/// once set, every descendant leaf reads the full upstream. Scope is passed
+/// by value, so sibling subtrees never leak state to each other.
+fn restrict(
+    plan: Arc<dyn ExecutionPlan>,
+    partitions: &[usize],
+    under_collect: bool,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    // Leaves: apply the algebraic partition-projection when we're not
+    // under a collapse. Under collapse, leaves keep their full upstream —
+    // the generic recursion below leaves them unchanged.
+    if !under_collect && let Some(rewritten) = select_output_partitions(&plan, partitions)
+    {
+        return Ok(rewritten);
+    }
+
+    // UnionExec: parent partition `p` maps to exactly one child's local
+    // partition (`p` minus the sum of preceding children's counts). Split
+    // `partitions` into disjoint per-child sub-slices and recurse; a child
+    // that gets `[]` becomes a 0-partition subplan, and UnionExec's
+    // partition-index math routes `execute(i)` past it correctly. Under
+    // `under_collect`, every descendant reads the full upstream anyway,
+    // so the generic recursion below applies instead.
+    if !under_collect && plan.is::<UnionExec>() {
+        let per_child = union_child_partitions(&plan, partitions);
+        let new_children: Vec<Arc<dyn ExecutionPlan>> = plan
+            .children()
+            .into_iter()
+            .cloned()
+            .zip(per_child)
+            .map(|(child, sub)| restrict(child, &sub, false))
+            .collect::<Result<Vec<_>>>()?;
+        return with_new_children_if_necessary(plan, new_children);
+    }
+
+    // Interior: compute per-child scope, recurse, rebuild the node.
+    let per_child = child_scopes(&plan, under_collect);
+    let new_children: Vec<Arc<dyn ExecutionPlan>> = plan
+        .children()
+        .into_iter()
+        .cloned()
+        .zip(per_child)
+        .map(|(child, collect)| restrict(child, partitions, collect))
+        .collect::<Result<Vec<_>>>()?;
+    with_new_children_if_necessary(plan, new_children)
+}
+
+/// Bucket each parent partition of a `UnionExec` into its owning child's
+/// local index. Iterates over `partitions` and, for each, subtracts each
+/// child's count until the remainder lands inside a child. Returns one
+/// sub-slice per child (empty for children this task never polls).
+fn union_child_partitions(
+    plan: &Arc<dyn ExecutionPlan>,
+    partitions: &[usize],
+) -> Vec<Vec<usize>> {
+    let child_counts: Vec<usize> = plan
+        .children()
+        .iter()
+        .map(|c| c.properties().output_partitioning().partition_count())
+        .collect();
+    let mut per_child: Vec<Vec<usize>> = vec![vec![]; child_counts.len()];
+    for &p in partitions {
+        let mut remaining = p;
+        for (i, &count) in child_counts.iter().enumerate() {
+            if remaining < count {
+                per_child[i].push(remaining);
+                break;
+            }
+            remaining -= count;
+        }
+    }
+    per_child
+}
+
+/// The `under_collect` value to pass to each child of `plan`.
+///
+/// - Once `under_collect` is set on the parent, it propagates unchanged to
+///   every child (it's sticky — descendants of a collapse all read
+///   everything).
+/// - `CoalescePartitionsExec` / `SortPreservingMergeExec`: set for all
+///   children.
+/// - `HashJoinExec` / `NestedLoopJoinExec`: set per child based on
+///   `required_input_distribution()`. Typically only the build side is
+///   `SinglePartition`, so the probe side inherits `false` and remains
+///   partition-aligned.
+///
+/// `UnionExec` is handled by the caller before reaching this function —
+/// its children need per-child *partition* sub-slices, not just per-child
+/// scope, so a `Vec<bool>` can't express what it needs.
+fn child_scopes(plan: &Arc<dyn ExecutionPlan>, under_collect: bool) -> Vec<bool> {
+    let children = plan.children();
+    if under_collect {
+        return vec![true; children.len()];
+    }
+    if plan.is::<CoalescePartitionsExec>() || plan.is::<SortPreservingMergeExec>() {
+        return vec![true; children.len()];
+    }
+    if plan.is::<HashJoinExec>() || plan.is::<NestedLoopJoinExec>() {
+        return plan
+            .required_input_distribution()
+            .into_iter()
+            .map(|d| matches!(d, Distribution::SinglePartition))
+            .collect();
+    }
+    vec![false; children.len()]
+}
+
+/// Restrict a plan node to a subset of its output partitions, re-indexed
+/// so position `i` of the result corresponds to input `indices[i]`.
+///
+/// Contract when `Some(new)` is returned:
+///
+/// - `new.output_partitioning().partition_count() == indices.len()`
+/// - `new.execute(j)` ≡ `plan.execute(indices[j])`
+/// - `Partitioning` kind is preserved (`Hash(exprs, N) → Hash(exprs, |S|)`,
+///   `RoundRobinBatch(N) → RoundRobinBatch(|S|)`, `Unknown(N) → Unknown(|S|)`).
+/// - `new.schema() == plan.schema()`
+///
+/// `None` means "unrecognised plan type — no idea how." The caller decides
+/// whether to fall through to a generic children walk, bail, or panic.
+/// `None` is NOT "no restriction needed."
+///
+/// # Why a helper and not a trait method
+///
+/// The operation is algebraically a projection on the partition-index
+/// axis and belongs on `ExecutionPlan` as `select_partitions(&[usize])`
+/// with default impls for partition-preserving operators. DataFusion's
+/// trait doesn't provide it — the closest is `repartitioned(N)`, which
+/// reshapes count but doesn't preserve partition identity, and isn't
+/// implemented on the self-generating leaves anyway. So we downcast here.
+///
+/// **When a new leaf type surfaces, add a branch below.** Currently
+/// covered: `ShuffleReaderExec`, `DataSourceExec` (files, in-memory),
+/// `PlaceholderRowExec`, `EmptyExec`. `ValuesExec` and other future
+/// self-generating leaves will silently fall through as `None`; if one
+/// sits under a partition-splitting fan-in, the walker leaves it at its
+/// full partition count and the executor duplicates output (see the
+/// PlaceholderRow-under-UnionExec bug for the failure mode).
+///
+/// # Scope
+///
+/// This helper is scope-agnostic. The `under_collect` decision (whether
+/// a leaf below a `CoalescePartitionsExec` / `SortPreservingMergeExec` /
+/// join build side must read its full upstream) lives in the walker —
+/// the walker simply doesn't call this function under collapse.
+fn select_output_partitions(
+    plan: &Arc<dyn ExecutionPlan>,
+    indices: &[usize],
+) -> Option<Arc<dyn ExecutionPlan>> {
+    // ShuffleReaderExec: cross-stage inputs.
+    if let Some(reader) = plan.downcast_ref::<ShuffleReaderExec>() {
+        // Broadcast readers serve everything from partition[0] for every
+        // consumer — slicing would drop the payload. Leave intact.
+        if reader.broadcast {
+            return Some(plan.clone());
+        }
+        let kept: Vec<Vec<_>> = indices
+            .iter()
+            .filter_map(|&p| reader.partition.get(p).cloned())
+            .collect();
+        let partitioning = match reader.properties().output_partitioning() {
+            Partitioning::Hash(exprs, _) => Partitioning::Hash(exprs.clone(), kept.len()),
+            Partitioning::RoundRobinBatch(_) => Partitioning::RoundRobinBatch(kept.len()),
+            Partitioning::UnknownPartitioning(_) => {
+                Partitioning::UnknownPartitioning(kept.len())
+            }
+        };
+        let restricted = ShuffleReaderExec::try_new(
+            reader.stage_id,
+            kept,
+            reader.schema(),
+            partitioning,
+        )
+        .ok()?;
+        return Some(Arc::new(restricted));
+    }
+
+    // DataSourceExec: file-backed or in-memory scans.
+    if let Some(exec) = plan.downcast_ref::<DataSourceExec>() {
+        let source: &dyn Any = exec.data_source().as_ref();
+        if let Some(config) = source.downcast_ref::<FileScanConfig>() {
+            let file_groups: Vec<FileGroup> = indices
+                .iter()
+                .filter_map(|&i| config.file_groups.get(i).cloned())
+                .collect();
+            let restricted = FileScanConfigBuilder::from(config.clone())
+                .with_file_groups(file_groups)
+                .build();
+            return Some(DataSourceExec::from_data_source(restricted));
+        }
+        if let Some(config) = source.downcast_ref::<MemorySourceConfig>() {
+            let kept: Vec<Vec<_>> = indices
+                .iter()
+                .filter_map(|&i| config.partitions().get(i).cloned())
+                .collect();
+            let restricted = MemorySourceConfig::try_new(
+                &kept,
+                config.original_schema(),
+                config.projection().clone(),
+            )
+            .ok()?;
+            return Some(DataSourceExec::from_data_source(restricted));
+        }
+        warn!(
+            "select_output_partitions: unrecognised DataSourceExec source \
+             left unrestricted; if it distributes work from a shared queue, \
+             tasks would over-read"
+        );
+        return None;
+    }
+
+    // Self-generating leaves: PlaceholderRow (1 all-null row per partition)
+    // and EmptyExec (0 rows per partition). Every partition of one of these
+    // is semantically identical, so `with_partitions(indices.len())` is the
+    // algebraically correct restriction.
+    //
+    // Caveat: DataFusion's proto codec for these two doesn't roundtrip the
+    // `partitions` field — encode preserves only schema; decode always
+    // reconstructs with `::new(schema)` (partitions = 1). So a plan we
+    // restrict to `with_partitions(0)` here would arrive at the executor as
+    // a 1-partition leaf, undoing the restriction and reproducing the
+    // duplicated-output bug this whole function is meant to prevent.
+    //
+    // Workaround for the empty-slice case: swap in a 0-partition
+    // `MemorySourceConfig` (which DOES roundtrip its partition list).
+    // For non-empty slices we return `None` and let the generic recursion
+    // leave the leaf as-is — currently safe because the only shape emitted
+    // is `partitions == 1` and UnionExec routing gives sub-slices of `[0]`
+    // (identity, no restriction needed) or `[]` (handled by this branch).
+    if plan.is::<PlaceholderRowExec>() || plan.is::<EmptyExec>() {
+        if indices.is_empty() {
+            let stub = MemorySourceConfig::try_new(&[], plan.schema(), None).ok()?;
+            return Some(DataSourceExec::from_data_source(stub));
+        }
+        return None;
+    }
+
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ballista_core::JobId;
+    use ballista_core::execution_plans::{
+        CoalescePlan, PartitionGroup, ShuffleReaderExec,
+    };
+    use ballista_core::serde::scheduler::{
+        ExecutorMetadata, PartitionId, PartitionLocation,
+    };
+    use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::physical_expr::Partitioning;
+
+    fn create_partition(partition: usize) -> PartitionLocation {
+        PartitionLocation {
+            map_partition_id: 0,
+            partition_id: PartitionId {
+                job_id: JobId::new("demo".to_string()),
+                stage_id: 0,
+                partition_id: partition,
+            },
+            executor_meta: ExecutorMetadata {
+                id: "1".to_string(),
+                host: "1.1.1.1".to_string(),
+                port: 0,
+                grpc_port: 0,
+                specification: Default::default(),
+                os_info: Default::default(),
+            },
+            partition_stats: Default::default(),
+            file_id: None,
+            is_sort_shuffle: false,
+        }
+    }
+
+    #[test]
+    fn keeps_only_the_wanted_partition() {
+        let partitions = (0..4).map(|i| vec![create_partition(i)]).collect();
+        let reader = ShuffleReaderExec::try_new(
+            1,
+            partitions,
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            Partitioning::UnknownPartitioning(4),
+        )
+        .unwrap();
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(reader);
+        let pruned = restrict_plan_to_partitions(plan, &[1]).unwrap();
+        let r = pruned
+            .downcast_ref::<ShuffleReaderExec>()
+            .expect("expected a ShuffleReaderExec");
+        // Kept partitions get positions 0..kept.len(); position 0 now holds
+        // what used to live at upstream partition 1.
+        assert_eq!(r.partition.len(), 1);
+        assert_eq!(r.partition[0].len(), 1);
+        assert_eq!(r.partition[0][0].partition_id.partition_id, 1);
+    }
+
+    #[test]
+    fn keeps_multiple_wanted_partitions_in_request_order() {
+        let partitions = (0..4).map(|i| vec![create_partition(i)]).collect();
+        let reader = ShuffleReaderExec::try_new(
+            1,
+            partitions,
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            Partitioning::UnknownPartitioning(4),
+        )
+        .unwrap();
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(reader);
+        let pruned = restrict_plan_to_partitions(plan, &[0, 3]).unwrap();
+        let r = pruned
+            .downcast_ref::<ShuffleReaderExec>()
+            .expect("expected a ShuffleReaderExec");
+        assert_eq!(r.partition.len(), 2);
+        assert_eq!(r.partition[0][0].partition_id.partition_id, 0);
+        assert_eq!(r.partition[1][0].partition_id.partition_id, 3);
+    }
+
+    #[test]
+    fn broadcast_reader_is_not_pruned() {
+        // Broadcast readers serve partition[0] for every consumer index, so
+        // restriction must be a no-op regardless of the assigned slice.
+        let reader = ShuffleReaderExec::try_new_broadcast(
+            1,
+            (0..4).map(create_partition).collect(),
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            4,
+        )
+        .unwrap();
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(reader);
+        let pruned = restrict_plan_to_partitions(plan, &[1]).unwrap();
+        let r = pruned
+            .downcast_ref::<ShuffleReaderExec>()
+            .expect("expected a ShuffleReaderExec");
+        assert!(r.broadcast);
+        assert_eq!(r.partition.len(), 1);
+        assert_eq!(r.partition[0].len(), 4);
+    }
+
+    // TODO: rewrite_shuffle_reader currently drops the CoalescePlan when
+    // restricting a coalesced reader (it goes through ShuffleReaderExec::try_new,
+    // not try_new_coalesced). Coalesced readers ARE constructed at runtime in
+    // aqe/adapter.rs, so a slice-restricted coalesced reader would silently
+    // become non-coalesced. Once that's fixed, this test should assert the
+    // coalesce metadata survives and the reindexed partition[0] holds the
+    // second coalesce group's 2 locations.
+    #[test]
+    #[ignore = "coalesce metadata is stripped during restriction — see fn comment"]
+    fn coalesced_reader_stays_coalesced_when_restricted() {
+        let coalesce = CoalescePlan {
+            upstream_partition_count: 4,
+            groups: vec![
+                PartitionGroup {
+                    upstream_indices: vec![0, 1],
+                },
+                PartitionGroup {
+                    upstream_indices: vec![2, 3],
+                },
+            ],
+        };
+        let reader = ShuffleReaderExec::try_new_coalesced(
+            1,
+            vec![
+                vec![create_partition(0), create_partition(1)],
+                vec![create_partition(2), create_partition(3)],
+            ],
+            coalesce,
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            Partitioning::UnknownPartitioning(2),
+        )
+        .unwrap();
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(reader);
+        let pruned = restrict_plan_to_partitions(plan, &[1]).unwrap();
+        let r = pruned
+            .downcast_ref::<ShuffleReaderExec>()
+            .expect("expected a ShuffleReaderExec");
+        assert!(r.coalesce.is_some());
+        assert_eq!(r.partition.len(), 1);
+        assert_eq!(r.partition[0].len(), 2);
+    }
+
+    #[test]
+    fn restricting_to_every_partition_is_a_no_op() {
+        let partitions = (0..3).map(|i| vec![create_partition(i)]).collect();
+        let reader = ShuffleReaderExec::try_new(
+            1,
+            partitions,
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            Partitioning::UnknownPartitioning(3),
+        )
+        .unwrap();
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(reader);
+        let pruned = restrict_plan_to_partitions(plan, &[0, 1, 2]).unwrap();
+        let r = pruned
+            .downcast_ref::<ShuffleReaderExec>()
+            .expect("expected a ShuffleReaderExec");
+        assert_eq!(r.partition.len(), 3);
+        for (i, loc) in r.partition.iter().enumerate() {
+            assert_eq!(loc.len(), 1);
+            assert_eq!(loc[0].partition_id.partition_id, i);
+        }
+    }
+
+    // --- UnionExec routing ---
+    //
+    // These tests exercise the walker's UnionExec branch. Under a union, the
+    // parent's partition indices are disjointly allocated across children by
+    // subtracting each preceding child's partition count. A child that owns
+    // none of the requested partitions is restricted to `[]` and produces a
+    // 0-partition subplan; UnionExec's `execute(i)` math routes past it. The
+    // upstream fix used a per-single-partition helper on the executor side;
+    // this suite ports the same invariants to the scheduler-side rewriter.
+
+    use datafusion::arrow::datatypes::SchemaRef;
+    use datafusion::datasource::listing::PartitionedFile;
+    use datafusion::datasource::physical_plan::ParquetSource;
+    use datafusion::execution::object_store::ObjectStoreUrl;
+    use datafusion::physical_plan::union::UnionExec;
+
+    /// Build a `DataSourceExec` over `n` file groups (one file each), so
+    /// every group is exactly one partition. The scan's output partition
+    /// count equals `n`.
+    fn scan_with_file_groups(n: usize) -> Arc<dyn ExecutionPlan> {
+        let schema: SchemaRef =
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+        let source = Arc::new(ParquetSource::new(schema));
+        let mut builder =
+            FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source);
+        for i in 0..n {
+            builder =
+                builder.with_file_group(FileGroup::new(vec![PartitionedFile::new(
+                    format!("file{i}.parquet"),
+                    100,
+                )]));
+        }
+        DataSourceExec::from_data_source(builder.build())
+    }
+
+    /// File counts per file group of a `DataSourceExec` — the shape a union
+    /// test cares about after restriction.
+    fn group_file_counts(plan: &Arc<dyn ExecutionPlan>) -> Vec<usize> {
+        let exec = plan.downcast_ref::<DataSourceExec>().unwrap();
+        let source: &dyn Any = exec.data_source().as_ref();
+        let config = source.downcast_ref::<FileScanConfig>().unwrap();
+        config.file_groups.iter().map(|g| g.len()).collect()
+    }
+
+    fn union_child_file_counts(plan: &Arc<dyn ExecutionPlan>) -> Vec<Vec<usize>> {
+        plan.children().into_iter().map(group_file_counts).collect()
+    }
+
+    /// A `UnionExec` concatenates its children's partitions, so stage
+    /// partition `left_count + k` is the right child's local partition
+    /// `k`. Applying the parent's index directly to every child (the pre-
+    /// port behaviour) would route every partition into every child and
+    /// most would land out of range, emptying every scan.
+    #[test]
+    fn union_partition_maps_to_the_right_childs_local_group() {
+        let union: Arc<dyn ExecutionPlan> =
+            UnionExec::try_new(vec![scan_with_file_groups(4), scan_with_file_groups(4)])
+                .unwrap();
+        assert_eq!(
+            union.properties().output_partitioning().partition_count(),
+            8
+        );
+
+        // Parent partition 1 is served by the left child's local partition 1.
+        let restricted = restrict_plan_to_partitions(union.clone(), &[1]).unwrap();
+        assert_eq!(
+            union_child_file_counts(&restricted),
+            vec![vec![1], vec![]],
+            "left scan keeps only its local group 1; right scan is not polled \
+             and becomes 0-partition"
+        );
+
+        // Parent partition 6 is served by the right child's local partition 2
+        // (6 - 4). Before the mapping existed this would have addressed no
+        // group at all in either child.
+        let restricted = restrict_plan_to_partitions(union, &[6]).unwrap();
+        assert_eq!(
+            union_child_file_counts(&restricted),
+            vec![vec![], vec![1]],
+            "right scan keeps only its local group 2; left scan is not polled"
+        );
+    }
+
+    /// Every partition of a union must pin exactly one group in exactly one
+    /// child, so across the whole stage each file group is read exactly once.
+    #[test]
+    fn every_union_partition_pins_exactly_one_group() {
+        for partition in 0..6 {
+            let union: Arc<dyn ExecutionPlan> = UnionExec::try_new(vec![
+                scan_with_file_groups(3),
+                scan_with_file_groups(3),
+            ])
+            .unwrap();
+            let restricted = restrict_plan_to_partitions(union, &[partition]).unwrap();
+            let counts = union_child_file_counts(&restricted);
+            let (serving, idle) = if partition < 3 { (0, 1) } else { (1, 0) };
+            assert_eq!(
+                counts[serving],
+                vec![1],
+                "partition {partition}: serving child must keep exactly one group"
+            );
+            assert!(
+                counts[idle].is_empty(),
+                "partition {partition}: idle child must be a 0-partition subplan, \
+                 got {:?}",
+                counts[idle]
+            );
+        }
+    }
+
+    /// Without a union a partition passes straight through, so the behaviour
+    /// is unchanged from restricting the scan directly.
+    #[test]
+    fn scan_without_union_is_pinned_to_its_own_partition() {
+        let plan = scan_with_file_groups(4);
+        let restricted = restrict_plan_to_partitions(plan, &[2]).unwrap();
+        assert_eq!(group_file_counts(&restricted), vec![1]);
+    }
+}
diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs
index 7bcbd68534..fe84ef252b 100644
--- a/ballista/scheduler/src/state/task_manager.rs
+++ b/ballista/scheduler/src/state/task_manager.rs
@@ -18,15 +18,17 @@
 use crate::cluster::JobState;
 use crate::config::SchedulerConfig;
 use crate::planner::DefaultDistributedPlanner;
-use crate::scheduler_server::event::QueryStageSchedulerEvent;
+use crate::scheduler_server::event::{QueryStageSchedulerEvent, SubmitPlan};
 use crate::state::aqe::AdaptiveExecutionGraph;
 use crate::state::distributed_explain::handle_explain_plan;
 use crate::state::execution_graph::{
     ExecutionGraphBox, RunningTaskInfo, StaticExecutionGraph, TaskDescription,
 };
 use crate::state::executor_manager::ExecutorManager;
+use crate::state::task_builder::restrict_plan_to_partitions;
 use ballista_core::error::BallistaError;
 use ballista_core::error::Result;
+use ballista_core::execution_plans::compute_global_output_partition_ids;
 use ballista_core::extension::{SessionConfigExt, SessionConfigHelperExt};
 use ballista_core::serde::BallistaCodec;
 use ballista_core::serde::protobuf::{
@@ -40,12 +42,13 @@ use datafusion::execution::context::SessionContext;
 use datafusion::logical_expr::LogicalPlan;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_proto::logical_plan::AsLogicalPlan;
-use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec};
+use datafusion_proto::physical_plan::AsExecutionPlan;
 use datafusion_proto::protobuf::PhysicalPlanNode;
 use log::{debug, error, info, trace, warn};
 use rand::distr::Alphanumeric;
 use rand::{RngExt, rng};
 use std::collections::{HashMap, HashSet};
+use std::future::Future;
 use std::ops::Deref;
 use std::sync::Arc;
 use std::time::Duration;
@@ -93,7 +96,7 @@ impl TaskLauncher for DefaultTaskLauncher {
                     let task_ids: Vec<u32> = task
                         .task_ids
                         .iter()
-                        .map(|task_id| task_id.partition_id)
+                        .map(|task_id| task_id.task_id)
                         .collect();
                     format!("{}/{}/{:?}", task.job_id, task.stage_id, task_ids)
                 })
@@ -136,8 +139,6 @@ pub struct TaskManager<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
     stage_max_failures: usize,
 }
 
-/// Cache for active job information managed by this scheduler.
-///
 /// Contains the execution graph and cached data to improve performance
 /// when scheduling tasks for the job.
 #[derive(Clone)]
@@ -146,9 +147,6 @@ pub struct JobInfoCache {
     pub execution_graph: Arc<RwLock<ExecutionGraphBox>>,
     /// Cached job status for quick access.
     pub status: Option<job_status::Status>,
-    #[cfg(not(feature = "disable-stage-plan-cache"))]
-    /// Cache for encoded execution stage plans to avoid redundant serialization.
-    encoded_stage_plans: HashMap<usize, Vec<u8>>,
 }
 
 impl JobInfoCache {
@@ -159,42 +157,8 @@ impl JobInfoCache {
         Self {
             execution_graph: Arc::new(RwLock::new(graph)),
             status,
-            #[cfg(not(feature = "disable-stage-plan-cache"))]
-            encoded_stage_plans: HashMap::new(),
-        }
-    }
-    #[cfg(not(feature = "disable-stage-plan-cache"))]
-    fn encode_stage_plan<U: AsExecutionPlan>(
-        &mut self,
-        stage_id: usize,
-        plan: &Arc<dyn ExecutionPlan>,
-        codec: &dyn PhysicalExtensionCodec,
-    ) -> Result<Vec<u8>> {
-        if let Some(plan) = self.encoded_stage_plans.get(&stage_id) {
-            Ok(plan.clone())
-        } else {
-            let mut plan_buf: Vec<u8> = vec![];
-            let plan_proto = U::try_from_physical_plan(plan.clone(), codec)?;
-            plan_proto.try_encode(&mut plan_buf)?;
-            self.encoded_stage_plans.insert(stage_id, plan_buf.clone());
-
-            Ok(plan_buf)
         }
     }
-
-    #[cfg(feature = "disable-stage-plan-cache")]
-    fn encode_stage_plan<U: AsExecutionPlan>(
-        &mut self,
-        _stage_id: usize,
-        plan: &Arc<dyn ExecutionPlan>,
-        codec: &dyn PhysicalExtensionCodec,
-    ) -> Result<Vec<u8>> {
-        let mut plan_buf: Vec<u8> = vec![];
-        let plan_proto = U::try_from_physical_plan(plan.clone(), codec)?;
-        plan_proto.try_encode(&mut plan_buf)?;
-
-        Ok(plan_buf)
-    }
 }
 
 /// Tracks stage state changes during task status updates.
@@ -271,59 +235,95 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
 
     /// Get the number of running jobs.
     pub fn running_job_number(&self) -> usize {
-        self.active_job_cache.len()
+        self.active_job_cache
+            .iter()
+            .filter(|job_info| {
+                matches!(job_info.status, Some(job_status::Status::Running(_)))
+            })
+            .count()
     }
 
     /// Generate an ExecutionGraph for the job and save it to the persistent state.
     /// By default, this job will be curated by the scheduler which receives it.
     /// Then we will also save it to the active execution graph
+    ///
+    /// Shared entry point for logical and physical submissions.
     #[allow(clippy::too_many_arguments)]
-    pub async fn submit_job(
+    pub async fn submit_plan(
         &self,
         job_id: &JobId,
         job_name: &str,
         ctx: Arc<SessionContext>,
-        logical_plan: &LogicalPlan,
+        plan: &SubmitPlan,
         queued_at: u64,
         subscriber: Option<JobStatusSubscriber>,
     ) -> Result<()> {
         let mut planner = DefaultDistributedPlanner::new();
         let session_state = ctx.state();
         let session_config = session_state.config();
-        let mut graph = if session_config.ballista_adaptive_query_planner_enabled() {
-            debug!("Using adaptive query planner (AQE) for job planning");
-            warn!(
-                "Adaptive Query Planning is EXPERIMENTAL, should be used for testing purposes only!"
-            );
-            Box::new(
-                AdaptiveExecutionGraph::try_new(
+
+        let mut graph = match plan {
+            SubmitPlan::Logical(logical_plan) => {
+                if session_config.ballista_adaptive_query_planner_enabled() {
+                    debug!("Using adaptive query planner (AQE) for job planning");
+                    warn!(
+                        "Adaptive Query Planning is EXPERIMENTAL, should be used for testing purposes only!"
+                    );
+                    Box::new(
+                        AdaptiveExecutionGraph::try_new(
+                            &self.scheduler_id,
+                            job_id,
+                            job_name,
+                            &ctx,
+                            logical_plan,
+                            queued_at,
+                        )
+                        .await?,
+                    ) as ExecutionGraphBox
+                } else {
+                    debug!("Using static query planner for job planning");
+                    let session_config = Arc::new(ctx.copied_config());
+
+                    let physical_plan =
+                        ctx.state().create_physical_plan(logical_plan).await?;
+                    let physical_plan =
+                        handle_explain_plan(job_id, &ctx, logical_plan, physical_plan)
+                            .await?;
+
+                    Box::new(StaticExecutionGraph::new(
+                        &self.scheduler_id,
+                        job_id,
+                        job_name,
+                        &ctx.session_id(),
+                        physical_plan,
+                        queued_at,
+                        session_config,
+                        &mut planner,
+                        Some(logical_plan.display_indent().to_string()),
+                    )?) as ExecutionGraphBox
+                }
+            }
+            SubmitPlan::Physical(physical_plan) => {
+                if session_config.ballista_adaptive_query_planner_enabled() {
+                    return Err(BallistaError::NotImplemented(
+                        "Adaptive query planning (AQE) does not support jobs submitted as an already-built physical plan; disable AQE for this session or submit a logical plan instead.".to_string(),
+                    ));
+                }
+                debug!("Using static query planner for physical-plan job submission");
+                let session_config = Arc::new(ctx.copied_config());
+
+                Box::new(StaticExecutionGraph::new(
                     &self.scheduler_id,
                     job_id,
                     job_name,
-                    &ctx,
-                    logical_plan,
+                    &ctx.session_id(),
+                    physical_plan.clone(),
                     queued_at,
-                )
-                .await?,
-            ) as ExecutionGraphBox
-        } else {
-            debug!("Using static query planner for job planning");
-            let session_config = Arc::new(ctx.copied_config());
-
-            let plan = ctx.state().create_physical_plan(logical_plan).await?;
-            let plan = handle_explain_plan(job_id, &ctx, logical_plan, plan).await?;
-
-            Box::new(StaticExecutionGraph::new(
-                &self.scheduler_id,
-                job_id,
-                job_name,
-                &ctx.session_id(),
-                plan,
-                queued_at,
-                session_config,
-                &mut planner,
-                Some(logical_plan.display_indent().to_string()),
-            )?) as ExecutionGraphBox
+                    session_config,
+                    &mut planner,
+                    None,
+                )?) as ExecutionGraphBox
+            }
         };
         let string_plan =
             datafusion::physical_plan::displayable(graph.physical_plan().as_ref())
@@ -342,6 +342,52 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
         Ok(())
     }
 
+    /// Submit a logical plan for distributed execution.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn submit_job(
+        &self,
+        job_id: &JobId,
+        job_name: &str,
+        ctx: Arc<SessionContext>,
+        plan: &LogicalPlan,
+        queued_at: u64,
+        subscriber: Option<JobStatusSubscriber>,
+    ) -> Result<()> {
+        self.submit_plan(
+            job_id,
+            job_name,
+            ctx,
+            &SubmitPlan::Logical(plan.clone()),
+            queued_at,
+            subscriber,
+        )
+        .await
+    }
+
+    /// Submit an already-built physical plan for distributed execution. The physical
+    /// plan is planned with the static distributed planner; adaptive query planning
+    /// (AQE) is not available for this path.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn submit_physical_plan(
+        &self,
+        job_id: &JobId,
+        job_name: &str,
+        ctx: Arc<SessionContext>,
+        plan: Arc<dyn ExecutionPlan>,
+        queued_at: u64,
+        subscriber: Option<JobStatusSubscriber>,
+    ) -> Result<()> {
+        self.submit_plan(
+            job_id,
+            job_name,
+            ctx,
+            &SubmitPlan::Physical(plan),
+            queued_at,
+            subscriber,
+        )
+        .await
+    }
+
     /// Returns a snapshot of currently running jobs from the cache.
     pub fn get_running_job_cache(&self) -> Arc<HashMap<JobId, JobInfoCache>> {
         let ret = self
@@ -460,6 +506,39 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
         Ok(graph.session_config())
     }
 
+    /// Sum the vcores consumed by the given task statuses at their original
+    /// bind time. Used to refund the executor's vcore budget when tasks
+    /// complete: refunding `statuses.len()` (one per task) would drop
+    /// leftover vcores forever, because `bind_one` consumes `slice.len()`
+    /// vcores per task under the multi-partition-task model.
+    ///
+    /// Statuses whose (job, stage, task_id) can no longer be resolved
+    /// (e.g. the job's graph has been evicted) contribute 0.
+    pub(crate) async fn sum_vcores_for_statuses(&self, statuses: &[TaskStatus]) -> u32 {
+        let mut statuses_by_job: HashMap<String, Vec<&TaskStatus>> = HashMap::new();
+        for status in statuses {
+            statuses_by_job
+                .entry(status.job_id.clone())
+                .or_default()
+                .push(status);
+        }
+        let mut total_vcores: u32 = 0;
+        for (job_id, job_statuses) in statuses_by_job {
+            let Some(graph_arc) = self.get_active_execution_graph(&job_id.into()) else {
+                continue;
+            };
+            let graph = graph_arc.read().await;
+            for status in job_statuses {
+                if let Some(vcores) =
+                    graph.task_vcores(status.stage_id as usize, status.task_id as usize)
+                {
+                    total_vcores += vcores;
+                }
+            }
+        }
+        total_vcores
+    }
+
     /// Update given task statuses in the respective job and return a tuple containing:
     /// 1. A list of QueryStageSchedulerEvent to publish.
     /// 2. A list of reservations that can now be offered.
@@ -508,68 +587,101 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
         Ok(events)
     }
 
-    /// Mark a job to success. This will create a key under the CompletedJobs keyspace
-    /// and remove the job from ActiveJobs
-    pub(crate) async fn succeed_job(&self, job_id: &JobId) -> Result<()> {
+    /// Save a terminal job snapshot and remove it from the active cache.
+    pub(crate) async fn persist_terminal_and_evict(
+        &self,
+        job_id: &JobId,
+        snapshot: &ExecutionGraphBox,
+    ) -> Result<()> {
+        if let Some(mut job_info) = self.active_job_cache.get_mut(job_id) {
+            job_info.status = snapshot.status().status.clone();
+        }
+
+        let save_result = self.state.save_job(job_id, snapshot).await;
+        if let Err(error) = &save_result {
+            warn!(
+                "Failed to persist terminal state for job {job_id}; evicting the local cache entry: {error}"
+            );
+        }
+        let _ = self.remove_active_execution_graph(job_id);
+        save_result
+    }
+
+    /// Move a job from Active to Success and return the ids of its intermediate
+    /// (non-final) stages so their shuffle data can be reclaimed immediately.
+    /// Returns an empty vec if the job is not found or not successful.
+    pub(crate) async fn succeed_job(&self, job_id: &JobId) -> Result<Vec<u32>> {
         debug!("Moving job {job_id} from Active to Success");
 
-        if let Some(graph) = self.remove_active_execution_graph(job_id) {
-            let graph = graph.read().await;
-            if graph.is_successful() {
-                self.state.save_job(job_id, &graph).await?;
-            } else {
-                error!("Job {job_id} has not finished and cannot be completed");
-                return Ok(());
-            }
+        if let Some(graph) = self.get_active_execution_graph(job_id) {
+            let (snapshot, intermediate_stage_ids) = {
+                let graph = graph.read().await;
+                if !graph.is_successful() {
+                    error!("Job {job_id} has not finished and cannot be completed");
+                    return Ok(vec![]);
+                }
+
+                (graph.cloned(), graph.intermediate_stage_ids())
+            };
+
+            self.persist_terminal_and_evict(job_id, &snapshot).await?;
+            Ok(intermediate_stage_ids)
         } else {
             warn!("Fail to find job {job_id} in the cache");
+            Ok(vec![])
         }
-
-        Ok(())
     }
 
-    /// Cancel the job and return a Vec of running tasks need to cancel
-    pub(crate) async fn cancel_job(
-        &self,
-        job_id: &JobId,
-    ) -> Result<(Vec<RunningTaskInfo>, usize)> {
-        self.abort_job(job_id, "Cancelled".to_owned()).await
-    }
-
-    /// Abort the job and return a Vec of running tasks need to cancel
-    pub(crate) async fn abort_job(
+    /// Mark a job as failed, cancel its running tasks, and persist terminal state.
+    ///
+    /// Task cancellation is invoked before persistence so executor work is
+    /// stopped even when all terminal-state save attempts fail.
+    pub(crate) async fn abort_job<F, Fut>(
         &self,
         job_id: &JobId,
         failure_reason: String,
-    ) -> Result<(Vec<RunningTaskInfo>, usize)> {
-        let (tasks_to_cancel, pending_tasks) = if let Some(graph) =
-            self.remove_active_execution_graph(job_id)
-        {
-            let mut guard = graph.write().await;
-
-            let pending_tasks = guard.available_tasks();
-            let running_tasks = guard.running_tasks();
-
-            info!(
-                "Cancelling {} running tasks for job {}",
-                running_tasks.len(),
-                job_id
-            );
-
-            guard.fail_job(failure_reason);
-
-            self.state.save_job(job_id, &guard).await?;
-
-            (running_tasks, pending_tasks)
-        } else {
+        cancel_tasks: F,
+    ) -> Result<usize>
+    where
+        F: FnOnce(Vec<RunningTaskInfo>) -> Fut,
+        Fut: Future<Output = Result<()>>,
+    {
+        let Some(graph) = self.get_active_execution_graph(job_id) else {
             // TODO listen the job state update event and fix task cancelling
             warn!(
                 "Fail to find job {job_id} in the cache, unable to cancel tasks for job, fail the job state only."
             );
-            (vec![], 0)
+            return Ok(0);
+        };
+
+        let (running_tasks, pending_tasks, snapshot) = {
+            let mut guard = graph.write().await;
+            let pending_tasks = guard.available_tasks();
+            let running_tasks = guard.abort_running(failure_reason);
+            let snapshot = guard.cloned();
+            if let Some(mut job_info) = self.active_job_cache.get_mut(job_id) {
+                job_info.status = snapshot.status().status.clone();
+            }
+            (running_tasks, pending_tasks, snapshot)
         };
 
-        Ok((tasks_to_cancel, pending_tasks))
+        info!(
+            "Cancelling {} running tasks for job {}",
+            running_tasks.len(),
+            job_id
+        );
+
+        let cancel_result = cancel_tasks(running_tasks).await;
+        let persist_result = self.persist_terminal_and_evict(job_id, &snapshot).await;
+        match (cancel_result, persist_result) {
+            (Ok(()), Ok(())) => Ok(pending_tasks),
+            (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
+            (Err(cancel_error), Err(persist_error)) => {
+                Err(BallistaError::General(format!(
+                    "Failed to cancel tasks for job {job_id}: {cancel_error}; failed to persist terminal state: {persist_error}"
+                )))
+            }
+        }
     }
 
     /// Mark a unscheduled job as failed. This will create a key under the FailedJobs keyspace
@@ -646,6 +758,11 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
     }
 
     /// Prepares a task definition for a single task to be sent to an executor.
+    ///
+    /// The task's plan is rewritten to restrict leaves (Scan file_groups,
+    /// ShuffleReader partitions) to this task's assigned slice, then encoded
+    /// fresh. No shared-plan cache — each task gets its own bytes because
+    /// each task's plan is different.
     #[allow(dead_code)]
     pub fn prepare_task_definition(
         &self,
@@ -653,30 +770,42 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
     ) -> Result<TaskDefinition> {
         debug!("Preparing task definition for {task:?}");
 
-        let job_id = task.partition.job_id.clone();
-        let stage_id = task.partition.stage_id;
+        let job_id = task.key.job_id.clone();
+        let stage_id = task.key.stage_id;
 
-        if let Some(mut job_info) = self.active_job_cache.get_mut(&job_id) {
-            let plan = job_info.encode_stage_plan::<PhysicalPlanNode>(
-                stage_id,
-                &task.plan,
+        if self.active_job_cache.get(&job_id).is_some() {
+            let restricted = restrict_plan_to_partitions(
+                task.plan.clone(),
+                &task.global_input_partition_ids,
+            )?;
+            let mut plan_buf: Vec<u8> = vec![];
+            let plan_proto = PhysicalPlanNode::try_from_physical_plan(
+                restricted,
                 self.codec.physical_extension_codec(),
             )?;
+            plan_proto.try_encode(&mut plan_buf)?;
 
             let task_definition = TaskDefinition {
-                task_id: task.task_id as u32,
+                task_id: task.key.task_id as u32,
                 task_attempt_num: task.task_attempt as u32,
                 job_id: job_id.into(),
                 stage_id: stage_id as u32,
                 stage_attempt_num: task.stage_attempt_num as u32,
-                partition_id: task.partition.partition_id as u32,
-                plan,
+                plan: plan_buf,
                 session_id: task.session_id,
                 launch_time: SystemTime::now()
                     .duration_since(UNIX_EPOCH)
                     .unwrap()
                     .as_millis() as u64,
                 props: task.session_config.to_key_value_pairs(),
+                global_output_partition_ids: compute_global_output_partition_ids(
+                    &task.plan,
+                    &task.global_input_partition_ids,
+                )
+                .into_iter()
+                .map(|pid| pid as u32)
+                .collect(),
+                vcores_consumed: task.vcores_consumed,
             };
             Ok(task_definition)
         } else {
@@ -711,72 +840,78 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
     }
 
     #[allow(dead_code)]
-    /// Prepare a MultiTaskDefinition with multiple tasks belonging to the same job stage
+    /// Emit one `MultiTaskDefinition` per task. Previously grouped multiple
+    /// tasks under one shared plan; now each task's plan is uniquely restricted
+    /// to its assigned partition slice so we can't share encoding.
     fn prepare_multi_task_definition(
         &self,
         tasks: Vec<TaskDescription>,
     ) -> Result<Vec<MultiTaskDefinition>> {
-        if let Some(task) = tasks.first() {
-            let session_id = task.session_id.clone();
-            let job_id = task.partition.job_id.clone();
-            let stage_id = task.partition.stage_id;
-            let stage_attempt_num = task.stage_attempt_num;
-
-            if log::max_level() >= log::Level::Debug {
-                let task_ids: Vec<usize> = tasks
-                    .iter()
-                    .map(|task| task.partition.partition_id)
-                    .collect();
-                debug!(
-                    "Preparing multi task definition for tasks {task_ids:?} belonging to job stage {job_id}/{stage_id}"
-                );
-                trace!("With task details {tasks:?}");
-            }
-
-            if let Some(mut job_info) = self.active_job_cache.get_mut(&job_id) {
-                let plan = job_info.encode_stage_plan::<PhysicalPlanNode>(
-                    stage_id,
-                    &task.plan,
-                    self.codec.physical_extension_codec(),
-                )?;
+        let [first_task, ..] = tasks.as_slice() else {
+            return Err(BallistaError::General(
+                "Cannot prepare multi task definition for an empty vec".to_string(),
+            ));
+        };
+        let session_id = first_task.session_id.clone();
+        let job_id = first_task.key.job_id.clone();
+        let stage_id = first_task.key.stage_id;
+        let stage_attempt_num = first_task.stage_attempt_num;
+
+        if log::max_level() >= log::Level::Debug {
+            let task_ids: Vec<usize> =
+                tasks.iter().map(|task| task.key.task_id).collect();
+            debug!(
+                "Preparing multi task definition for tasks {task_ids:?} belonging to job stage {job_id}/{stage_id}"
+            );
+            trace!("With task details {tasks:?}");
+        }
 
-                let launch_time = SystemTime::now()
-                    .duration_since(UNIX_EPOCH)
-                    .unwrap()
-                    .as_millis() as u64;
-
-                let mut multi_tasks = vec![];
-                let props = task.session_config.to_key_value_pairs();
-                let task_ids = tasks
-                    .into_iter()
-                    .map(|task| TaskId {
-                        task_id: task.task_id as u32,
-                        task_attempt_num: task.task_attempt as u32,
-                        partition_id: task.partition.partition_id as u32,
-                    })
-                    .collect();
-                multi_tasks.push(MultiTaskDefinition {
-                    task_ids,
-                    job_id: job_id.into(),
-                    stage_id: stage_id as u32,
-                    stage_attempt_num: stage_attempt_num as u32,
-                    plan,
-                    session_id,
-                    launch_time,
-                    props,
-                });
+        if self.active_job_cache.get(&job_id).is_none() {
+            return Err(BallistaError::General(format!(
+                "Cannot prepare multi task definition for job {job_id} which is not in active cache"
+            )));
+        }
 
-                Ok(multi_tasks)
-            } else {
-                Err(BallistaError::General(format!(
-                    "Cannot prepare multi task definition for job {job_id} which is not in active cache"
-                )))
-            }
-        } else {
-            Err(BallistaError::General(
-                "Cannot prepare multi task definition for an empty vec".to_string(),
-            ))
+        let launch_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as u64;
+        let codec = self.codec.physical_extension_codec();
+
+        let mut multi_tasks = Vec::with_capacity(tasks.len());
+        for task in tasks {
+            let restricted = restrict_plan_to_partitions(
+                task.plan.clone(),
+                &task.global_input_partition_ids,
+            )?;
+            let mut plan_buf: Vec<u8> = vec![];
+            let plan_proto = PhysicalPlanNode::try_from_physical_plan(restricted, codec)?;
+            plan_proto.try_encode(&mut plan_buf)?;
+            let props = task.session_config.to_key_value_pairs();
+            let task_ids = vec![TaskId {
+                task_id: task.key.task_id as u32,
+                task_attempt_num: task.task_attempt as u32,
+                global_output_partition_ids: compute_global_output_partition_ids(
+                    &task.plan,
+                    &task.global_input_partition_ids,
+                )
+                .into_iter()
+                .map(|p| p as u32)
+                .collect(),
+                vcores_consumed: task.vcores_consumed,
+            }];
+            multi_tasks.push(MultiTaskDefinition {
+                task_ids,
+                job_id: job_id.clone().into(),
+                stage_id: stage_id as u32,
+                stage_attempt_num: stage_attempt_num as u32,
+                plan: plan_buf,
+                session_id: session_id.clone(),
+                launch_time,
+                props,
+            });
         }
+        Ok(multi_tasks)
     }
 
     /// Get the `ExecutionGraph` for the given job ID from cache
@@ -863,3 +998,375 @@ impl From<&ExecutionGraphBox> for JobOverview {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cluster::JobStateEventStream;
+    use crate::cluster::memory::InMemoryJobState;
+    use crate::test_utils::{mock_completed_task, mock_executor, test_aggregation_plan};
+    use ballista_core::serde::protobuf::job_status::Status;
+    use ballista_core::utils::{default_config_producer, default_session_builder};
+    use datafusion_proto::protobuf::LogicalPlanNode;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use tokio::sync::Notify;
+    use tokio::time::timeout;
+
+    struct BlockingJobState {
+        inner: InMemoryJobState,
+        save_started: Notify,
+        allow_save: Notify,
+        failures_remaining: AtomicUsize,
+        save_attempts: AtomicUsize,
+    }
+
+    impl BlockingJobState {
+        fn new() -> Self {
+            Self {
+                inner: InMemoryJobState::new(
+                    "test-scheduler",
+                    Arc::new(default_session_builder),
+                    Arc::new(default_config_producer),
+                ),
+                save_started: Notify::new(),
+                allow_save: Notify::new(),
+                failures_remaining: AtomicUsize::new(0),
+                save_attempts: AtomicUsize::new(0),
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl JobState for BlockingJobState {
+        fn accept_job(
+            &self,
+            job_id: &JobId,
+            job_name: &str,
+            queued_at: u64,
+        ) -> Result<()> {
+            self.inner.accept_job(job_id, job_name, queued_at)
+        }
+
+        fn pending_job_number(&self) -> usize {
+            self.inner.pending_job_number()
+        }
+
+        async fn submit_job(
+            &self,
+            job_id: JobId,
+            graph: &ExecutionGraphBox,
+            subscriber: Option<JobStatusSubscriber>,
+        ) -> Result<()> {
+            self.inner.submit_job(job_id, graph, subscriber).await
+        }
+
+        async fn get_jobs(&self) -> Result<HashSet<JobId>> {
+            self.inner.get_jobs().await
+        }
+
+        async fn get_all_jobs(&self) -> Result<HashSet<JobId>> {
+            self.inner.get_all_jobs().await
+        }
+
+        async fn get_job_status(&self, job_id: &JobId) -> Result<Option<JobStatus>> {
+            self.inner.get_job_status(job_id).await
+        }
+
+        async fn get_execution_graph(
+            &self,
+            job_id: &JobId,
+        ) -> Result<Option<ExecutionGraphBox>> {
+            self.inner.get_execution_graph(job_id).await
+        }
+
+        async fn save_job(
+            &self,
+            job_id: &JobId,
+            graph: &ExecutionGraphBox,
+        ) -> Result<()> {
+            let attempt = self.save_attempts.fetch_add(1, Ordering::SeqCst);
+            if attempt == 0 {
+                self.save_started.notify_one();
+                self.allow_save.notified().await;
+            }
+            if self
+                .failures_remaining
+                .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| {
+                    remaining.checked_sub(1)
+                })
+                .is_ok()
+            {
+                return Err(BallistaError::General("injected save failure".to_string()));
+            }
+            self.inner.save_job(job_id, graph).await
+        }
+
+        async fn fail_unscheduled_job(
+            &self,
+            job_id: &JobId,
+            reason: String,
+        ) -> Result<()> {
+            self.inner.fail_unscheduled_job(job_id, reason).await
+        }
+
+        async fn remove_job(&self, job_id: &JobId) -> Result<()> {
+            self.inner.remove_job(job_id).await
+        }
+
+        async fn try_acquire_job(
+            &self,
+            job_id: &JobId,
+        ) -> Result<Option<ExecutionGraphBox>> {
+            self.inner.try_acquire_job(job_id).await
+        }
+
+        async fn job_state_events(&self) -> Result<JobStateEventStream> {
+            self.inner.job_state_events().await
+        }
+
+        async fn create_or_update_session(
+            &self,
+            session_id: &str,
+            config: &SessionConfig,
+        ) -> Result<Arc<SessionContext>> {
+            self.inner
+                .create_or_update_session(session_id, config)
+                .await
+        }
+
+        async fn remove_session(&self, session_id: &str) -> Result<()> {
+            self.inner.remove_session(session_id).await
+        }
+
+        fn produce_config(&self) -> SessionConfig {
+            self.inner.produce_config()
+        }
+    }
+
+    type TestTaskManager = TaskManager<LogicalPlanNode, PhysicalPlanNode>;
+
+    async fn setup_job(
+        successful: bool,
+    ) -> Result<(
+        TestTaskManager,
+        Arc<BlockingJobState>,
+        JobId,
+        Arc<RwLock<ExecutionGraphBox>>,
+    )> {
+        let state = Arc::new(BlockingJobState::new());
+        let job_state: Arc<dyn JobState> = state.clone();
+        let manager = TaskManager::new(
+            job_state,
+            BallistaCodec::default(),
+            "test-scheduler".to_string(),
+            Arc::new(SchedulerConfig::default()),
+        );
+
+        let graph: ExecutionGraphBox = Box::new(test_aggregation_plan(2).await);
+        let job_id = graph.job_id().to_owned();
+        state.accept_job(&job_id, graph.job_name(), 0)?;
+        state.submit_job(job_id.clone(), &graph, None).await?;
+
+        manager
+            .active_job_cache
+            .insert(job_id.clone(), JobInfoCache::new(graph));
+        let active_graph = manager
+            .get_active_execution_graph(&job_id)
+            .expect("job should be active");
+
+        if successful {
+            // Complete the graph after it is cached. `JobInfoCache::status` still
+            // contains Running until terminal persistence updates it.
+            let mut graph = active_graph.write().await;
+            let executor = mock_executor("executor-1".to_string());
+            while let Some(task) = graph.pop_next_task(&executor.id)? {
+                graph.update_task_status(
+                    &executor,
+                    vec![mock_completed_task(task, &executor.id)],
+                    1,
+                    1,
+                )?;
+            }
+            graph.succeed_job()?;
+        }
+
+        Ok((manager, state, job_id, active_graph))
+    }
+
+    #[tokio::test]
+    async fn successful_job_stays_visible_and_unlocked_until_persisted() -> Result<()> {
+        let (manager, state, job_id, active_graph) = setup_job(true).await?;
+        let manager_for_save = manager.clone();
+        let job_id_for_save = job_id.clone();
+        let save =
+            tokio::spawn(
+                async move { manager_for_save.succeed_job(&job_id_for_save).await },
+            );
+
+        timeout(Duration::from_secs(1), state.save_started.notified())
+            .await
+            .expect("terminal save should start");
+
+        let status = manager
+            .get_job_status(&job_id)
+            .await?
+            .expect("successful job should remain visible during persistence");
+        assert!(matches!(status.status, Some(Status::Successful(_))));
+        assert!(!manager.get_running_job_cache().contains_key(&job_id));
+        assert_eq!(manager.running_job_number(), 0);
+
+        let graph_guard = timeout(Duration::from_secs(1), active_graph.write())
+            .await
+            .expect("graph lock must not be held across persistence");
+        drop(graph_guard);
+
+        state.allow_save.notify_one();
+        save.await.expect("succeed_job task should not panic")?;
+
+        assert!(manager.get_active_execution_graph(&job_id).is_none());
+        let persisted = state
+            .get_job_status(&job_id)
+            .await?
+            .expect("successful status should be persisted");
+        assert!(matches!(persisted.status, Some(Status::Successful(_))));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn failed_terminal_save_is_not_retried_and_evicts_local_cache() -> Result<()> {
+        let (manager, state, job_id, active_graph) = setup_job(true).await?;
+        state.failures_remaining.store(1, Ordering::SeqCst);
+        let manager_for_save = manager.clone();
+        let job_id_for_save = job_id.clone();
+        let save =
+            tokio::spawn(
+                async move { manager_for_save.succeed_job(&job_id_for_save).await },
+            );
+
+        timeout(Duration::from_secs(1), state.save_started.notified())
+            .await
+            .expect("terminal save should start");
+        assert!(manager.get_active_execution_graph(&job_id).is_some());
+        assert_eq!(manager.running_job_number(), 0);
+        state.allow_save.notify_one();
+        assert!(
+            save.await
+                .expect("succeed_job task should not panic")
+                .is_err()
+        );
+
+        assert_eq!(state.save_attempts.load(Ordering::SeqCst), 1);
+        assert!(manager.get_active_execution_graph(&job_id).is_none());
+        assert_eq!(manager.running_job_number(), 0);
+        let authoritative_status = manager
+            .get_job_status(&job_id)
+            .await?
+            .expect("shared status should remain available after local eviction");
+        assert!(matches!(
+            authoritative_status.status,
+            Some(Status::Running(_))
+        ));
+
+        let graph_guard = timeout(Duration::from_secs(1), active_graph.write())
+            .await
+            .expect("graph lock must be released after a failed save");
+        drop(graph_guard);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn aborted_tasks_are_cancelled_before_terminal_persistence() -> Result<()> {
+        let (manager, state, job_id, active_graph) = setup_job(false).await?;
+        let executor = mock_executor("executor-1".to_string());
+        active_graph
+            .write()
+            .await
+            .pop_next_task(&executor.id)?
+            .expect("job should have a task to assign");
+
+        let cancelled_tasks = Arc::new(AtomicUsize::new(0));
+        let cancelled_tasks_for_abort = cancelled_tasks.clone();
+        let manager_for_abort = manager.clone();
+        let job_id_for_abort = job_id.clone();
+        let abort = tokio::spawn(async move {
+            manager_for_abort
+                .abort_job(
+                    &job_id_for_abort,
+                    "test failure".to_string(),
+                    move |tasks| async move {
+                        cancelled_tasks_for_abort.store(tasks.len(), Ordering::SeqCst);
+                        Ok(())
+                    },
+                )
+                .await
+        });
+
+        timeout(Duration::from_secs(1), state.save_started.notified())
+            .await
+            .expect("terminal save should start");
+        assert_eq!(cancelled_tasks.load(Ordering::SeqCst), 1);
+        assert_eq!(state.save_attempts.load(Ordering::SeqCst), 1);
+        assert_eq!(manager.running_job_number(), 0);
+
+        let status = manager
+            .get_job_status(&job_id)
+            .await?
+            .expect("failed job should remain visible during persistence");
+        assert!(matches!(status.status, Some(Status::Failed(_))));
+        let graph_guard = timeout(Duration::from_secs(1), active_graph.write())
+            .await
+            .expect("graph lock must not be held across persistence");
+        drop(graph_guard);
+
+        state.allow_save.notify_one();
+        abort.await.expect("abort_job task should not panic")?;
+        assert!(manager.get_active_execution_graph(&job_id).is_none());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn aborted_tasks_are_cancelled_when_terminal_save_fails() -> Result<()> {
+        let (manager, state, job_id, active_graph) = setup_job(false).await?;
+        let executor = mock_executor("executor-1".to_string());
+        active_graph
+            .write()
+            .await
+            .pop_next_task(&executor.id)?
+            .expect("job should have a task to assign");
+        state.failures_remaining.store(1, Ordering::SeqCst);
+
+        let cancelled_tasks = Arc::new(AtomicUsize::new(0));
+        let cancelled_tasks_for_abort = cancelled_tasks.clone();
+        let manager_for_abort = manager.clone();
+        let job_id_for_abort = job_id.clone();
+        let abort = tokio::spawn(async move {
+            manager_for_abort
+                .abort_job(
+                    &job_id_for_abort,
+                    "test failure".to_string(),
+                    move |tasks| async move {
+                        cancelled_tasks_for_abort.store(tasks.len(), Ordering::SeqCst);
+                        Ok(())
+                    },
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), state.save_started.notified())
+            .await
+            .expect("terminal save should start");
+        assert_eq!(cancelled_tasks.load(Ordering::SeqCst), 1);
+        state.allow_save.notify_one();
+        assert!(
+            abort
+                .await
+                .expect("abort_job task should not panic")
+                .is_err()
+        );
+
+        assert_eq!(cancelled_tasks.load(Ordering::SeqCst), 1);
+        assert_eq!(state.save_attempts.load(Ordering::SeqCst), 1);
+        assert!(manager.get_active_execution_graph(&job_id).is_none());
+        assert_eq!(manager.running_job_number(), 0);
+        Ok(())
+    }
+}
diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs
index 810983ef2b..be184cd6e6 100644
--- a/ballista/scheduler/src/test_utils.rs
+++ b/ballista/scheduler/src/test_utils.rs
@@ -19,7 +19,6 @@ use ballista_core::error::{BallistaError, Result};
 use ballista_core::extension::SessionConfigExt;
 use ballista_core::{JobId, JobStatusSubscriber};
 use datafusion::catalog::Session;
-use std::any::Any;
 use std::collections::HashMap;
 use std::future::Future;
 use std::sync::Arc;
@@ -80,10 +79,6 @@ pub struct ExplodingTableProvider;
 
 #[async_trait::async_trait]
 impl TableProvider for ExplodingTableProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         Arc::new(Schema::empty())
     }
@@ -299,19 +294,13 @@ pub fn default_task_runner() -> impl TaskRunner {
             })
             .collect();
 
-        for TaskId {
-            task_id,
-            partition_id,
-            ..
-        } in task.task_ids
-        {
+        for TaskId { task_id, .. } in task.task_ids {
             let timestamp = timestamp_millis();
             statuses.push(TaskStatus {
                 task_id,
                 job_id: task.job_id.clone(),
                 stage_id: task.stage_id,
                 stage_attempt_num: task.stage_attempt_num,
-                partition_id,
                 launch_time: timestamp,
                 start_exec_time: timestamp,
                 end_exec_time: timestamp,
@@ -330,7 +319,7 @@ pub fn default_task_runner() -> impl TaskRunner {
 #[derive(Clone)]
 struct VirtualExecutor {
     executor_id: String,
-    task_slots: usize,
+    vcores: usize,
     runner: Arc<dyn TaskRunner>,
 }
 
@@ -404,14 +393,14 @@ impl SchedulerTest {
         config: SchedulerConfig,
         metrics_collector: Arc<dyn SchedulerMetricsCollector>,
         num_executors: usize,
-        task_slots_per_executor: usize,
+        vcores_per_executor: usize,
         runner: Option<Arc<dyn TaskRunner>>,
     ) -> Result<Self> {
         let cluster = BallistaCluster::new_from_config(&config).await?;
 
-        let session_config = if num_executors > 0 && task_slots_per_executor > 0 {
+        let session_config = if num_executors > 0 && vcores_per_executor > 0 {
             SessionConfig::new_with_ballista()
-                .with_target_partitions(num_executors * task_slots_per_executor)
+                .with_target_partitions(num_executors * vcores_per_executor)
         } else {
             SessionConfig::new_with_ballista()
         };
@@ -423,7 +412,7 @@ impl SchedulerTest {
                 let id = format!("virtual-executor-{i}");
                 let executor = VirtualExecutor {
                     executor_id: id.clone(),
-                    task_slots: task_slots_per_executor,
+                    vcores: vcores_per_executor,
                     runner: runner.clone(),
                 };
                 (id, executor)
@@ -448,21 +437,21 @@ impl SchedulerTest {
             );
         scheduler.init().await?;
 
-        for (executor_id, VirtualExecutor { task_slots, .. }) in executors {
+        for (executor_id, VirtualExecutor { vcores, .. }) in executors {
             let metadata = ExecutorMetadata {
                 id: executor_id.clone(),
                 host: String::default(),
                 port: 0,
                 grpc_port: 0,
                 specification: ExecutorSpecification::default()
-                    .with_task_slots(task_slots as u32),
+                    .with_vcores(vcores as u32),
                 os_info: ExecutorOperatingSystemSpecification::default(),
             };
 
             let executor_data = ExecutorData {
                 executor_id,
-                total_task_slots: task_slots as u32,
-                available_task_slots: task_slots as u32,
+                total_vcores: vcores as u32,
+                available_vcores: vcores as u32,
             };
 
             scheduler
@@ -847,11 +836,7 @@ pub fn revive_graph_and_complete_next_stage_with_executor(
         .values()
         .map(|stage| {
             if let ExecutionStage::Running(stage) = stage {
-                stage
-                    .task_infos
-                    .iter()
-                    .filter(|info| info.is_none())
-                    .count()
+                stage.available_tasks()
             } else {
                 0
             }
@@ -880,6 +865,23 @@ pub async fn test_aggregation_plan(partition: usize) -> StaticExecutionGraph {
 pub async fn test_aggregation_plan_with_job_id(
     partition: usize,
     job_id: &JobId,
+) -> StaticExecutionGraph {
+    test_aggregation_plan_with_config(
+        partition,
+        job_id,
+        Arc::new(SessionConfig::new_with_ballista()),
+    )
+    .await
+}
+
+/// Same as `test_aggregation_plan_with_job_id`, but the caller supplies the
+/// Ballista `SessionConfig` used by the resulting graph. Use this when a test
+/// needs to override a scheduler-side knob (e.g. `max_partitions_per_task`)
+/// that changes how `bind_one` shapes tasks.
+pub async fn test_aggregation_plan_with_config(
+    partition: usize,
+    job_id: &JobId,
+    session_config: Arc<SessionConfig>,
 ) -> StaticExecutionGraph {
     let config = SessionConfig::new().with_target_partitions(partition);
     let ctx = Arc::new(SessionContext::new_with_config(config));
@@ -917,7 +919,7 @@ pub async fn test_aggregation_plan_with_job_id(
         "session",
         plan,
         0,
-        Arc::new(SessionConfig::new_with_ballista()),
+        session_config,
         &mut planner,
         None,
     )
@@ -1016,7 +1018,7 @@ pub async fn test_coalesce_plan(partition: usize) -> StaticExecutionGraph {
 
 /// Creates a test execution graph with a join operation.
 pub async fn test_join_plan(partition: usize) -> StaticExecutionGraph {
-    let mut config = SessionConfig::new().with_target_partitions(partition);
+    let mut config = SessionConfig::new_with_ballista().with_target_partitions(partition);
     config
         .options_mut()
         .optimizer
@@ -1172,7 +1174,7 @@ pub fn mock_executor(executor_id: String) -> ExecutorMetadata {
         host: "localhost2".to_string(),
         port: 8080,
         grpc_port: 9090,
-        specification: ExecutorSpecification::default().with_task_slots(1),
+        specification: ExecutorSpecification::default().with_vcores(1),
         os_info: ExecutorOperatingSystemSpecification::default(),
     }
 }
@@ -1196,11 +1198,10 @@ pub fn mock_completed_task(task: TaskDescription, executor_id: &str) -> TaskStat
 
     // Complete the task
     protobuf::TaskStatus {
-        task_id: task.task_id as u32,
-        job_id: task.partition.job_id.clone().into(),
-        stage_id: task.partition.stage_id as u32,
+        task_id: task.key.task_id as u32,
+        job_id: task.key.job_id.clone().into(),
+        stage_id: task.key.stage_id as u32,
         stage_attempt_num: task.stage_attempt_num as u32,
-        partition_id: task.partition.partition_id as u32,
         launch_time: 0,
         start_exec_time: 0,
         end_exec_time: 0,
@@ -1231,11 +1232,10 @@ pub fn mock_failed_task(task: TaskDescription, failed_task: FailedTask) -> TaskS
 
     // Fail the task
     protobuf::TaskStatus {
-        task_id: task.task_id as u32,
-        job_id: task.partition.job_id.clone().into(),
-        stage_id: task.partition.stage_id as u32,
+        task_id: task.key.task_id as u32,
+        job_id: task.key.job_id.clone().into(),
+        stage_id: task.key.stage_id as u32,
         stage_attempt_num: task.stage_attempt_num as u32,
-        partition_id: task.partition.partition_id as u32,
         launch_time: 0,
         start_exec_time: 0,
         end_exec_time: 0,
diff --git a/ballista/scheduler/tests/tpch_plan_stability/README.md b/ballista/scheduler/tests/tpch_plan_stability/README.md
new file mode 100644
index 0000000000..5b62c7f9af
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/README.md
@@ -0,0 +1,59 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# TPC-H plan-stability suite
+
+Freezes each TPC-H query's **distributed staged plan** (static planner, SF100
+table statistics, `target_partitions=16`) as an approved text file under
+`approved/`. The suite fails if a code change alters plan shape — join strategy,
+shuffle/stage boundaries, or broadcast decisions.
+
+- Run: `cargo test -p ballista-scheduler --test tpch_plan_stability`
+- Regenerate after an intended change: `dev/update-tpch-plan-stability.sh`
+  (or `BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability`),
+  then review the diff under `approved/`.
+
+Scope: TPC-H only, static planner, Ballista default config (SortMergeJoin).
+Tables are dataless providers with injected SF100 cardinalities (`fixtures.rs`).
+
+Query SQL is read directly from the canonical `benchmarks/queries/` at test time
+(not copied), so a change to a benchmark query flows into the planned plan and
+surfaces as a golden diff, prompting a deliberate regeneration.
+
+## CI coverage
+
+This suite is registered as a `[[test]]` target in `ballista/scheduler/Cargo.toml`
+(`tpch_plan_stability`), so it already runs wherever CI exercises the workspace's
+default cargo tests — no dedicated job was added:
+
+- `.github/workflows/rust.yml` → `linux-test` (`cargo test --profile ci --features=testcontainers`) and `macos-test` (`cargo test --profile ci --locked`) both run from the workspace root without `-p`/`--workspace`
+  scoping. Since the root `Cargo.toml` sets no `default-members`, this tests
+  every workspace member, including `ballista-scheduler`, which picks up this
+  target automatically.
+- `.github/workflows/rust.yml` → `clippy` already runs `cargo clippy --all-targets --package ballista-scheduler --all-features -- -D warnings`,
+  which lints this test target too.
+- `.github/workflows/rust.yml` → `lint` runs `cargo fmt --all -- --check`,
+  which covers these files as well.
+
+The generated `approved/*.txt` golden plans carry no license header (the test
+compares their exact bytes), so they are excluded from the Apache RAT license
+check in `.github/workflows/dev.yml` via
+`ballista/scheduler/tests/tpch_plan_stability/approved/*` in
+`dev/release/rat_exclude_files.txt` — mirroring the existing
+`ballista/scheduler/testdata/*` exclusion.
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt
new file mode 100644
index 0000000000..b01d7f2aff
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt
@@ -0,0 +1,18 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 16)
+  AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))]
+    ProjectionExec: expr=[l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus]
+      FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_extendedprice@1, l_discount@2, l_quantity@0, l_tax@3, l_returnflag@4, l_linestatus@5]
+        StatsExec: rows=600037902
+
+=== Stage 2 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order]
+      AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([l_returnflag@0, l_linestatus@1], 16)
+
+=== Stage 3 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=2, partitioning: Hash([l_returnflag@0, l_linestatus@1], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt
new file mode 100644
index 0000000000..7b12a86d23
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt
@@ -0,0 +1,57 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  StatsExec: rows=15000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16)
+  FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1]
+    StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@7], 16)
+  ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_nationkey@3 as c_nationkey, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, o_orderkey@7 as o_orderkey]
+    SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)]
+      SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16)
+      SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2]
+    StatsExec: rows=600037902
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([c_nationkey@3], 16)
+  ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_nationkey@3 as c_nationkey, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@9 as l_extendedprice, l_discount@10 as l_discount]
+    SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)]
+      SortExec: expr=[o_orderkey@7 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderkey@7], 16)
+      SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16)
+  AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+    ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_address@2 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@10 as n_name]
+      SortMergeJoinExec: join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)]
+        SortExec: expr=[c_nationkey@3 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=5, partitioning: Hash([c_nationkey@3], 16)
+        SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=6, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 8 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: TopK(fetch=20), expr=[revenue@2 DESC], preserve_partitioning=[true]
+    ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment]
+      AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 16)
+
+=== Stage 9 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [revenue@2 DESC], fetch=20
+    UnresolvedShuffleExec: stage=8, partitioning: Hash([c_custkey@0, c_name@1, c_acctbal@3, c_phone@6, n_name@4, c_address@5, c_comment@7], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt
new file mode 100644
index 0000000000..736c88e26c
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt
@@ -0,0 +1,81 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16)
+  StatsExec: rows=80000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16)
+  ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@4 as s_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)]
+      SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([ps_suppkey@0], 16)
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]
+    StatsExec: rows=25
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
+    ProjectionExec: expr=[ps_availqty@0 as ps_availqty, ps_supplycost@1 as ps_supplycost]
+      SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)]
+        SortExec: expr=[s_nationkey@2 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=3, partitioning: Hash([s_nationkey@2], 16)
+        SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=4, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1], 16)
+  StatsExec: rows=80000000
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16)
+  ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@5 as s_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)]
+      SortExec: expr=[ps_suppkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([ps_suppkey@1], 16)
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]
+    StatsExec: rows=25
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16)
+  AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
+    ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost]
+      SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)]
+        SortExec: expr=[s_nationkey@3 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=8, partitioning: Hash([s_nationkey@3], 16)
+        SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=9, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 11 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[value@1 DESC], preserve_partitioning=[true]
+    ProjectionExec: expr=[ps_partkey@1 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2 as value]
+      NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2]
+        ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]
+          AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
+            CoalescePartitionsExec
+              UnresolvedShuffleExec: stage=5, partitioning: Hash([n_nationkey@3], 16)
+        ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1]
+          AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
+            UnresolvedShuffleExec: stage=10, partitioning: Hash([ps_partkey@0], 16)
+
+=== Stage 12 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [value@1 DESC]
+    UnresolvedShuffleExec: stage=11, partitioning: Hash([ps_partkey@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt
new file mode 100644
index 0000000000..3b86494e39
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt
@@ -0,0 +1,30 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4]
+    StatsExec: rows=600037902
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([l_shipmode@0], 16)
+  AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)]
+    ProjectionExec: expr=[l_shipmode@1 as l_shipmode, o_orderpriority@3 as o_orderpriority]
+      SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)]
+        SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=1, partitioning: Hash([l_orderkey@0], 16)
+        SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=2, partitioning: Hash([o_orderkey@0], 16)
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count]
+      AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([l_shipmode@0], 16)
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=4, partitioning: Hash([l_shipmode@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt
new file mode 100644
index 0000000000..10bc255aeb
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt
@@ -0,0 +1,32 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  StatsExec: rows=15000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16)
+  FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1]
+    StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([c_count@0], 16)
+  AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))]
+    ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count]
+      AggregateExec: mode=SinglePartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], ordering_mode=Sorted
+        ProjectionExec: expr=[c_custkey@0 as c_custkey, o_orderkey@1 as o_orderkey]
+          SortMergeJoinExec: join_type=Left, on=[(c_custkey@0, o_custkey@1)]
+            SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16)
+            SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16)
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true]
+    ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist]
+      AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([c_count@0], 16)
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC]
+    UnresolvedShuffleExec: stage=4, partitioning: Hash([c_count@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt
new file mode 100644
index 0000000000..5f8721f421
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt
@@ -0,0 +1,25 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16)
+  FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2]
+    StatsExec: rows=600037902
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  StatsExec: rows=20000000
+
+=== Stage 3 ===
+ShuffleWriterExec: partitioning: None
+  AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+    ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, p_type@4 as p_type]
+      SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)]
+        SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16)
+        SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16)
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue]
+    AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+      CoalescePartitionsExec
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@3], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt
new file mode 100644
index 0000000000..caf8d0d9dd
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt
@@ -0,0 +1,53 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16)
+  AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+    FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]
+      StatsExec: rows=600037902
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([total_revenue@4], 16)
+  ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address, s_phone@3 as s_phone, total_revenue@5 as total_revenue]
+    SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, supplier_no@0)]
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([s_suppkey@0], 16)
+      SortExec: expr=[supplier_no@0 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]
+          AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+            UnresolvedShuffleExec: stage=2, partitioning: Hash([l_suppkey@0], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@0], 16)
+  AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+    FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]
+      StatsExec: rows=600037902
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)]
+    ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]
+      AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([l_suppkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([max(revenue0.total_revenue)@0], 16)
+  AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)]
+    CoalescePartitionsExec
+      UnresolvedShuffleExec: stage=5, partitioning: Hash([l_suppkey@0], 16)
+
+=== Stage 7 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address, s_phone@3 as s_phone, total_revenue@4 as total_revenue]
+      SortMergeJoinExec: join_type=Inner, on=[(total_revenue@4, max(revenue0.total_revenue)@0)]
+        SortExec: expr=[total_revenue@4 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=3, partitioning: Hash([total_revenue@4], 16)
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([max(revenue0.total_revenue)@0], 16)
+
+=== Stage 8 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=7, partitioning: Hash([total_revenue@4], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt
new file mode 100644
index 0000000000..2a62656215
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt
@@ -0,0 +1,44 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16)
+  StatsExec: rows=80000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9])
+    StatsExec: rows=20000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16)
+  ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey, p_brand@3 as p_brand, p_type@4 as p_type, p_size@5 as p_size]
+    SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, p_partkey@0)]
+      SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([ps_partkey@0], 16)
+      SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0]
+    StatsExec: rows=1000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 16)
+  AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)]
+    AggregateExec: mode=SinglePartitioned, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[], ordering_mode=PartiallySorted([3])
+      SortMergeJoinExec: join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)]
+        SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=3, partitioning: Hash([ps_suppkey@0], 16)
+        SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=4, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 6 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt]
+      AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16)
+
+=== Stage 7 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=6, partitioning: Hash([p_brand@0, p_type@1, p_size@2], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt
new file mode 100644
index 0000000000..a74197f02d
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt
@@ -0,0 +1,36 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16)
+  StatsExec: rows=600037902
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0]
+    StatsExec: rows=20000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16)
+  AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)]
+    StatsExec: rows=600037902
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)]
+    ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice]
+      SortMergeJoinExec: join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1
+        ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@3 as p_partkey]
+          SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)]
+            SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16)
+            SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16)
+        SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey]
+            AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)]
+              UnresolvedShuffleExec: stage=3, partitioning: Hash([l_partkey@0], 16)
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly]
+    AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)]
+      CoalescePartitionsExec
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([l_partkey@4], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt
new file mode 100644
index 0000000000..7cfabb07ef
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt
@@ -0,0 +1,46 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  StatsExec: rows=15000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16)
+  StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@2], 16)
+  ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, o_orderkey@2 as o_orderkey, o_totalprice@4 as o_totalprice, o_orderdate@5 as o_orderdate]
+    SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)]
+      SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16)
+      SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  StatsExec: rows=600037902
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)]
+    StatsExec: rows=600037902
+
+=== Stage 6 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: TopK(fetch=100), expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true]
+    AggregateExec: mode=SinglePartitioned, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)], ordering_mode=PartiallySorted([2])
+      SortMergeJoinExec: join_type=LeftSemi, on=[(o_orderkey@2, l_orderkey@0)]
+        ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, o_orderkey@2 as o_orderkey, o_totalprice@3 as o_totalprice, o_orderdate@4 as o_orderdate, l_quantity@6 as l_quantity]
+          SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)]
+            SortExec: expr=[o_orderkey@2 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderkey@2], 16)
+            SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16)
+        SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+          FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0]
+            AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)]
+              UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@0], 16)
+
+=== Stage 7 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], fetch=100
+    UnresolvedShuffleExec: stage=6, partitioning: Hash([o_orderkey@2], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt
new file mode 100644
index 0000000000..acd9d8cf14
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt
@@ -0,0 +1,26 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@0], 16)
+  FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3]
+    StatsExec: rows=600037902
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_size@2 >= 1 AND (p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15)
+    StatsExec: rows=20000000
+
+=== Stage 3 ===
+ShuffleWriterExec: partitioning: None
+  AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+    ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount]
+      SortMergeJoinExec: join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15
+        SortExec: expr=[l_partkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=1, partitioning: Hash([l_partkey@0], 16)
+        SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=2, partitioning: Hash([p_partkey@0], 16)
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue]
+    AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+      CoalescePartitionsExec
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([p_partkey@4], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt
new file mode 100644
index 0000000000..ee10151895
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt
@@ -0,0 +1,123 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1]
+    StatsExec: rows=20000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16)
+  StatsExec: rows=80000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@2], 16)
+  ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, ps_suppkey@3 as ps_suppkey, ps_supplycost@4 as ps_supplycost]
+    SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, ps_partkey@0)]
+      SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([p_partkey@0], 16)
+      SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([ps_partkey@0], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@4], 16)
+  ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@5 as s_name, s_address@6 as s_address, s_nationkey@7 as s_nationkey, s_phone@8 as s_phone, s_acctbal@9 as s_acctbal, s_comment@10 as s_comment, ps_supplycost@3 as ps_supplycost]
+    SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@2, s_suppkey@0)]
+      SortExec: expr=[ps_suppkey@2 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([ps_suppkey@2], 16)
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([n_regionkey@9], 16)
+  ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@2 as s_name, s_address@3 as s_address, s_phone@5 as s_phone, s_acctbal@6 as s_acctbal, s_comment@7 as s_comment, ps_supplycost@8 as ps_supplycost, n_name@10 as n_name, n_regionkey@11 as n_regionkey]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@4, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@4 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([s_nationkey@4], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16)
+  FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0]
+    StatsExec: rows=5
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 16)
+  ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@2 as s_name, s_address@3 as s_address, s_phone@4 as s_phone, s_acctbal@5 as s_acctbal, s_comment@6 as s_comment, ps_supplycost@7 as ps_supplycost, n_name@8 as n_name]
+    SortMergeJoinExec: join_type=Inner, on=[(n_regionkey@9, r_regionkey@0)]
+      SortExec: expr=[n_regionkey@9 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([n_regionkey@9], 16)
+      SortExec: expr=[r_regionkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=8, partitioning: Hash([r_regionkey@0], 16)
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1], 16)
+  StatsExec: rows=80000000
+
+=== Stage 11 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 12 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16)
+  ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@4 as s_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)]
+      SortExec: expr=[ps_suppkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=10, partitioning: Hash([ps_suppkey@1], 16)
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=11, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 13 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 14 ===
+SortShuffleWriterExec: partitioning=Hash([n_regionkey@2], 16)
+  ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@1 as ps_supplycost, n_regionkey@4 as n_regionkey]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@2 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=12, partitioning: Hash([s_nationkey@2], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=13, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 15 ===
+SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16)
+  FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0]
+    StatsExec: rows=5
+
+=== Stage 16 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16)
+  AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]
+    ProjectionExec: expr=[ps_partkey@0 as ps_partkey, ps_supplycost@1 as ps_supplycost]
+      SortMergeJoinExec: join_type=Inner, on=[(n_regionkey@2, r_regionkey@0)]
+        SortExec: expr=[n_regionkey@2 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=14, partitioning: Hash([n_regionkey@2], 16)
+        SortExec: expr=[r_regionkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=15, partitioning: Hash([r_regionkey@0], 16)
+
+=== Stage 17 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16)
+  ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey]
+    AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]
+      UnresolvedShuffleExec: stage=16, partitioning: Hash([ps_partkey@0], 16)
+
+=== Stage 18 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: TopK(fetch=100), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@8 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment]
+      SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)]
+        SortExec: expr=[p_partkey@0 ASC, ps_supplycost@7 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=9, partitioning: Hash([p_partkey@0, ps_supplycost@7], 16)
+        SortExec: expr=[ps_partkey@1 ASC, min(partsupp.ps_supplycost)@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=17, partitioning: Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 16)
+
+=== Stage 19 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=100
+    UnresolvedShuffleExec: stage=18, partitioning: Hash([p_partkey@3, min(partsupp.ps_supplycost)@9], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt
new file mode 100644
index 0000000000..de78d20fb1
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt
@@ -0,0 +1,66 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16)
+  StatsExec: rows=1000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0]
+    StatsExec: rows=25
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  ProjectionExec: expr=[s_suppkey@0 as s_suppkey, s_name@1 as s_name, s_address@2 as s_address]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@3 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([s_nationkey@3], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@0], 16)
+  StatsExec: rows=80000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0]
+    StatsExec: rows=20000000
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([ps_partkey@0, ps_suppkey@1], 16)
+  SortMergeJoinExec: join_type=LeftSemi, on=[(ps_partkey@0, p_partkey@0)]
+    SortExec: expr=[ps_partkey@0 ASC], preserve_partitioning=[true]
+      UnresolvedShuffleExec: stage=4, partitioning: Hash([ps_partkey@0], 16)
+    SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+      UnresolvedShuffleExec: stage=5, partitioning: Hash([p_partkey@0], 16)
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 16)
+  AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)]
+    FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2]
+      StatsExec: rows=600037902
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@0], 16)
+  ProjectionExec: expr=[ps_suppkey@1 as ps_suppkey]
+    SortMergeJoinExec: join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1
+      SortExec: expr=[ps_partkey@0 ASC, ps_suppkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([ps_partkey@0, ps_suppkey@1], 16)
+      SortExec: expr=[l_partkey@1 ASC, l_suppkey@2 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey]
+          AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)]
+            UnresolvedShuffleExec: stage=7, partitioning: Hash([l_partkey@0, l_suppkey@1], 16)
+
+=== Stage 9 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[s_name@1 as s_name, s_address@2 as s_address]
+      SortMergeJoinExec: join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)]
+        SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=3, partitioning: Hash([s_suppkey@0], 16)
+        SortExec: expr=[ps_suppkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=8, partitioning: Hash([ps_suppkey@0], 16)
+
+=== Stage 10 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [s_name@0 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=9, partitioning: Hash([s_suppkey@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt
new file mode 100644
index 0000000000..003c45c938
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt
@@ -0,0 +1,79 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16)
+  FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1]
+    StatsExec: rows=600037902
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@2], 16)
+  ProjectionExec: expr=[s_name@1 as s_name, s_nationkey@2 as s_nationkey, l_orderkey@3 as l_orderkey, l_suppkey@4 as l_suppkey]
+    SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)]
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([s_suppkey@0], 16)
+      SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([l_suppkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0]
+    StatsExec: rows=150000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@1], 16)
+  ProjectionExec: expr=[s_name@0 as s_name, s_nationkey@1 as s_nationkey, l_orderkey@2 as l_orderkey, l_suppkey@3 as l_suppkey]
+    SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@2, o_orderkey@0)]
+      SortExec: expr=[l_orderkey@2 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([l_orderkey@2], 16)
+      SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([o_orderkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0]
+    StatsExec: rows=25
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@1], 16)
+  ProjectionExec: expr=[s_name@0 as s_name, l_orderkey@2 as l_orderkey, l_suppkey@3 as l_suppkey]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@1, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([s_nationkey@1], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  StatsExec: rows=600037902
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1]
+    StatsExec: rows=600037902
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([s_name@0], 16)
+  AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))]
+    ProjectionExec: expr=[s_name@0 as s_name]
+      SortMergeJoinExec: join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0
+        SortMergeJoinExec: join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0
+          SortExec: expr=[l_orderkey@1 ASC], preserve_partitioning=[true]
+            UnresolvedShuffleExec: stage=7, partitioning: Hash([l_orderkey@1], 16)
+          SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+            UnresolvedShuffleExec: stage=8, partitioning: Hash([l_orderkey@0], 16)
+        SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=9, partitioning: Hash([l_orderkey@0], 16)
+
+=== Stage 11 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: TopK(fetch=100), expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait]
+      AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))]
+        UnresolvedShuffleExec: stage=10, partitioning: Hash([s_name@0], 16)
+
+=== Stage 12 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], fetch=100
+    UnresolvedShuffleExec: stage=11, partitioning: Hash([s_name@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt
new file mode 100644
index 0000000000..8255afee9a
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt
@@ -0,0 +1,35 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17])
+    StatsExec: rows=15000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@0], 16)
+  StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([cntrycode@0], 16)
+  AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]
+    ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal]
+      NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2]
+        AggregateExec: mode=Single, gby=[], aggr=[avg(customer.c_acctbal)]
+          FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1]
+            StatsExec: rows=15000000
+        ProjectionExec: expr=[c_phone@1 as c_phone, c_acctbal@2 as c_acctbal, CAST(c_acctbal@2 AS Decimal128(19, 6)) as join_proj_push_down_1]
+          SortMergeJoinExec: join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)]
+            SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16)
+            SortExec: expr=[o_custkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@0], 16)
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]
+      AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([cntrycode@0], 16)
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=4, partitioning: Hash([cntrycode@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt
new file mode 100644
index 0000000000..ea275d184b
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt
@@ -0,0 +1,40 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0]
+    StatsExec: rows=15000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16)
+  FilterExec: o_orderdate@2 < 1995-03-15
+    StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  ProjectionExec: expr=[o_orderkey@1 as o_orderkey, o_orderdate@3 as o_orderdate, o_shippriority@4 as o_shippriority]
+    SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)]
+      SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16)
+      SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2]
+    StatsExec: rows=600037902
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: TopK(fetch=10), expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority]
+      AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], ordering_mode=PartiallySorted([0])
+        ProjectionExec: expr=[o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority, l_orderkey@3 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount]
+          SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)]
+            SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderkey@0], 16)
+            SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+              UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16)
+
+=== Stage 6 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], fetch=10
+    UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt
new file mode 100644
index 0000000000..5e05011fef
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt
@@ -0,0 +1,31 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2]
+    StatsExec: rows=150000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0]
+    StatsExec: rows=600037902
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderpriority@0], 16)
+  AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))]
+    ProjectionExec: expr=[o_orderpriority@1 as o_orderpriority]
+      SortMergeJoinExec: join_type=LeftSemi, on=[(o_orderkey@0, l_orderkey@0)]
+        SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=1, partitioning: Hash([o_orderkey@0], 16)
+        SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=2, partitioning: Hash([l_orderkey@0], 16)
+
+=== Stage 4 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count]
+      AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderpriority@0], 16)
+
+=== Stage 5 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=4, partitioning: Hash([o_orderpriority@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt
new file mode 100644
index 0000000000..8849e6c0bb
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt
@@ -0,0 +1,83 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  StatsExec: rows=15000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@1], 16)
+  FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1]
+    StatsExec: rows=150000000
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@1], 16)
+  ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@2 as o_orderkey]
+    SortMergeJoinExec: join_type=Inner, on=[(c_custkey@0, o_custkey@1)]
+      SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([c_custkey@0], 16)
+      SortExec: expr=[o_custkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([o_custkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  StatsExec: rows=600037902
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@1, c_nationkey@0], 16)
+  ProjectionExec: expr=[c_nationkey@0 as c_nationkey, l_suppkey@3 as l_suppkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount]
+    SortMergeJoinExec: join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)]
+      SortExec: expr=[o_orderkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([o_orderkey@1], 16)
+      SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([l_orderkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0, s_nationkey@1], 16)
+  StatsExec: rows=1000000
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16)
+  ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@5 as s_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@1, s_suppkey@0), (c_nationkey@0, s_nationkey@1)]
+      SortExec: expr=[l_suppkey@1 ASC, c_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([l_suppkey@1, c_nationkey@0], 16)
+      SortExec: expr=[s_suppkey@0 ASC, s_nationkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([s_suppkey@0, s_nationkey@1], 16)
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([n_regionkey@3], 16)
+  ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@4 as n_name, n_regionkey@5 as n_regionkey]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@2 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([s_nationkey@2], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=8, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16)
+  FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0]
+    StatsExec: rows=5
+
+=== Stage 11 ===
+SortShuffleWriterExec: partitioning=Hash([n_name@0], 16)
+  AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+    ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, n_name@2 as n_name]
+      SortMergeJoinExec: join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)]
+        SortExec: expr=[n_regionkey@3 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=9, partitioning: Hash([n_regionkey@3], 16)
+        SortExec: expr=[r_regionkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=10, partitioning: Hash([r_regionkey@0], 16)
+
+=== Stage 12 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true]
+    ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue]
+      AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
+        UnresolvedShuffleExec: stage=11, partitioning: Hash([n_name@0], 16)
+
+=== Stage 13 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [revenue@1 DESC]
+    UnresolvedShuffleExec: stage=12, partitioning: Hash([n_name@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt
new file mode 100644
index 0000000000..1da12a0cb1
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt
@@ -0,0 +1,6 @@
+=== Stage 1 ===
+ShuffleWriterExec: partitioning: None
+  ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue]
+    AggregateExec: mode=Single, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)]
+      FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2]
+        StatsExec: rows=600037902
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt
new file mode 100644
index 0000000000..e2fbadd62c
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt
@@ -0,0 +1,84 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16)
+  FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31
+    StatsExec: rows=600037902
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@1], 16)
+  ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_orderkey@2 as l_orderkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, l_shipdate@6 as l_shipdate]
+    SortMergeJoinExec: join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)]
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([s_suppkey@0], 16)
+      SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([l_suppkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  StatsExec: rows=150000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@4], 16)
+  ProjectionExec: expr=[s_nationkey@0 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, o_custkey@6 as o_custkey]
+    SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)]
+      SortExec: expr=[l_orderkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([l_orderkey@1], 16)
+      SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([o_orderkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  StatsExec: rows=15000000
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@0], 16)
+  ProjectionExec: expr=[s_nationkey@0 as s_nationkey, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@6 as c_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(o_custkey@4, c_custkey@0)]
+      SortExec: expr=[o_custkey@4 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([o_custkey@4], 16)
+      SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([c_custkey@0], 16)
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY
+    StatsExec: rows=25
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([c_nationkey@3], 16)
+  ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@6 as n_name]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@0, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([s_nationkey@0], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=8, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE
+    StatsExec: rows=25
+
+=== Stage 11 ===
+SortShuffleWriterExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 16)
+  AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)]
+    ProjectionExec: expr=[n_name@4 as supp_nation, n_name@6 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume]
+      SortMergeJoinExec: join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE
+        SortExec: expr=[c_nationkey@3 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=9, partitioning: Hash([c_nationkey@3], 16)
+        SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=10, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 12 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue]
+      AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)]
+        UnresolvedShuffleExec: stage=11, partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16)
+
+=== Stage 13 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=12, partitioning: Hash([supp_nation@0, cust_nation@1, l_year@2], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt
new file mode 100644
index 0000000000..77ca93cd28
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt
@@ -0,0 +1,110 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0]
+    StatsExec: rows=20000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@1], 16)
+  StatsExec: rows=600037902
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@1], 16)
+  ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_suppkey@3 as l_suppkey, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount]
+    SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, l_partkey@1)]
+      SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([p_partkey@0], 16)
+      SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([l_partkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@5 as s_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@1, s_suppkey@0)]
+      SortExec: expr=[l_suppkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([l_suppkey@1], 16)
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31
+    StatsExec: rows=150000000
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([o_custkey@3], 16)
+  ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_custkey@5 as o_custkey, o_orderdate@6 as o_orderdate]
+    SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)]
+      SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([l_orderkey@0], 16)
+      SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([o_orderkey@0], 16)
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([c_custkey@0], 16)
+  StatsExec: rows=15000000
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([c_nationkey@4], 16)
+  ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, s_nationkey@2 as s_nationkey, o_orderdate@4 as o_orderdate, c_nationkey@6 as c_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(o_custkey@3, c_custkey@0)]
+      SortExec: expr=[o_custkey@3 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([o_custkey@3], 16)
+      SortExec: expr=[c_custkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=8, partitioning: Hash([c_custkey@0], 16)
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 11 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@2], 16)
+  ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, s_nationkey@2 as s_nationkey, o_orderdate@3 as o_orderdate, n_regionkey@6 as n_regionkey]
+    SortMergeJoinExec: join_type=Inner, on=[(c_nationkey@4, n_nationkey@0)]
+      SortExec: expr=[c_nationkey@4 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=9, partitioning: Hash([c_nationkey@4], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=10, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 12 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 13 ===
+SortShuffleWriterExec: partitioning=Hash([n_regionkey@3], 16)
+  ProjectionExec: expr=[l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@6 as n_name]
+    SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)]
+      SortExec: expr=[s_nationkey@2 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=11, partitioning: Hash([s_nationkey@2], 16)
+      SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=12, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 14 ===
+SortShuffleWriterExec: partitioning=Hash([r_regionkey@0], 16)
+  FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0]
+    StatsExec: rows=5
+
+=== Stage 15 ===
+SortShuffleWriterExec: partitioning=Hash([o_year@0], 16)
+  AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]
+    ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@4 as nation]
+      SortMergeJoinExec: join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)]
+        SortExec: expr=[n_regionkey@3 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=13, partitioning: Hash([n_regionkey@3], 16)
+        SortExec: expr=[r_regionkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=14, partitioning: Hash([r_regionkey@0], 16)
+
+=== Stage 16 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true]
+    ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share]
+      AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]
+        UnresolvedShuffleExec: stage=15, partitioning: Hash([o_year@0], 16)
+
+=== Stage 17 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [o_year@0 ASC NULLS LAST]
+    UnresolvedShuffleExec: stage=16, partitioning: Hash([o_year@0], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt
new file mode 100644
index 0000000000..48b15bca3a
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt
@@ -0,0 +1,82 @@
+=== Stage 1 ===
+SortShuffleWriterExec: partitioning=Hash([p_partkey@0], 16)
+  FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0]
+    StatsExec: rows=20000000
+
+=== Stage 2 ===
+SortShuffleWriterExec: partitioning=Hash([l_partkey@1], 16)
+  StatsExec: rows=600037902
+
+=== Stage 3 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@2], 16)
+  ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount]
+    SortMergeJoinExec: join_type=Inner, on=[(p_partkey@0, l_partkey@1)]
+      SortExec: expr=[p_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=1, partitioning: Hash([p_partkey@0], 16)
+      SortExec: expr=[l_partkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=2, partitioning: Hash([l_partkey@1], 16)
+
+=== Stage 4 ===
+SortShuffleWriterExec: partitioning=Hash([s_suppkey@0], 16)
+  StatsExec: rows=1000000
+
+=== Stage 5 ===
+SortShuffleWriterExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 16)
+  ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_partkey@1 as l_partkey, l_suppkey@2 as l_suppkey, l_quantity@3 as l_quantity, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, s_nationkey@7 as s_nationkey]
+    SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@2, s_suppkey@0)]
+      SortExec: expr=[l_suppkey@2 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=3, partitioning: Hash([l_suppkey@2], 16)
+      SortExec: expr=[s_suppkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=4, partitioning: Hash([s_suppkey@0], 16)
+
+=== Stage 6 ===
+SortShuffleWriterExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 16)
+  StatsExec: rows=80000000
+
+=== Stage 7 ===
+SortShuffleWriterExec: partitioning=Hash([l_orderkey@0], 16)
+  ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_quantity@3 as l_quantity, l_extendedprice@4 as l_extendedprice, l_discount@5 as l_discount, s_nationkey@6 as s_nationkey, ps_supplycost@9 as ps_supplycost]
+    SortMergeJoinExec: join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)]
+      SortExec: expr=[l_suppkey@2 ASC, l_partkey@1 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=5, partitioning: Hash([l_suppkey@2, l_partkey@1], 16)
+      SortExec: expr=[ps_suppkey@1 ASC, ps_partkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=6, partitioning: Hash([ps_suppkey@1, ps_partkey@0], 16)
+
+=== Stage 8 ===
+SortShuffleWriterExec: partitioning=Hash([o_orderkey@0], 16)
+  StatsExec: rows=150000000
+
+=== Stage 9 ===
+SortShuffleWriterExec: partitioning=Hash([s_nationkey@3], 16)
+  ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@7 as o_orderdate]
+    SortMergeJoinExec: join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)]
+      SortExec: expr=[l_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=7, partitioning: Hash([l_orderkey@0], 16)
+      SortExec: expr=[o_orderkey@0 ASC], preserve_partitioning=[true]
+        UnresolvedShuffleExec: stage=8, partitioning: Hash([o_orderkey@0], 16)
+
+=== Stage 10 ===
+SortShuffleWriterExec: partitioning=Hash([n_nationkey@0], 16)
+  StatsExec: rows=25
+
+=== Stage 11 ===
+SortShuffleWriterExec: partitioning=Hash([nation@0, o_year@1], 16)
+  AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)]
+    ProjectionExec: expr=[n_name@7 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) - ps_supplycost@4 * l_quantity@0 as amount]
+      SortMergeJoinExec: join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)]
+        SortExec: expr=[s_nationkey@3 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=9, partitioning: Hash([s_nationkey@3], 16)
+        SortExec: expr=[n_nationkey@0 ASC], preserve_partitioning=[true]
+          UnresolvedShuffleExec: stage=10, partitioning: Hash([n_nationkey@0], 16)
+
+=== Stage 12 ===
+ShuffleWriterExec: partitioning: None
+  SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true]
+    ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit]
+      AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)]
+        UnresolvedShuffleExec: stage=11, partitioning: Hash([nation@0, o_year@1], 16)
+
+=== Stage 13 ===
+ShuffleWriterExec: partitioning: None
+  SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC]
+    UnresolvedShuffleExec: stage=12, partitioning: Hash([nation@0, o_year@1], 16)
diff --git a/ballista/scheduler/tests/tpch_plan_stability/fixtures.rs b/ballista/scheduler/tests/tpch_plan_stability/fixtures.rs
new file mode 100644
index 0000000000..811370875e
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/fixtures.rs
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! TPC-H schema/fixture helpers and the distributed staged-plan text helper.
+//!
+//! `staged_plan_text` loads a TPC-H query's SQL, registers the 8 TPC-H tables
+//! (via [`TpchStatsTable`]) with SF100 row-count statistics, builds the
+//! physical plan with the static (Ballista) planner, breaks it into
+//! distributed query stages with [`DefaultDistributedPlanner`], and renders
+//! the stages to a normalized text representation suitable for snapshotting.
+
+use std::sync::Arc;
+
+use ballista_core::JobId;
+use ballista_core::extension::SessionConfigExt;
+use ballista_scheduler::planner::{DefaultDistributedPlanner, DistributedPlanner};
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::physical_plan::display::DisplayableExecutionPlan;
+use datafusion::prelude::{SessionConfig, SessionContext};
+
+use crate::stats_table::TpchStatsTable;
+
+const TARGET_PARTITIONS: usize = 16;
+const JOB_ID: &str = "plan_stability";
+
+/// The 8 TPC-H table names and their SF100 row counts.
+pub const SF100_ROWS: &[(&str, usize)] = &[
+    ("region", 5),
+    ("nation", 25),
+    ("supplier", 1_000_000),
+    ("customer", 15_000_000),
+    ("part", 20_000_000),
+    ("partsupp", 80_000_000),
+    ("orders", 150_000_000),
+    ("lineitem", 600_037_902),
+];
+
+/// Returns the schema for a TPC-H table, copied verbatim from
+/// `benchmarks/src/bin/tpch.rs::get_schema` (that function lives in a binary
+/// and cannot be imported directly).
+pub fn tpch_schema(table: &str) -> Schema {
+    match table {
+        "part" => Schema::new(vec![
+            Field::new("p_partkey", DataType::Int64, false),
+            Field::new("p_name", DataType::Utf8, false),
+            Field::new("p_mfgr", DataType::Utf8, false),
+            Field::new("p_brand", DataType::Utf8, false),
+            Field::new("p_type", DataType::Utf8, false),
+            Field::new("p_size", DataType::Int32, false),
+            Field::new("p_container", DataType::Utf8, false),
+            Field::new("p_retailprice", DataType::Decimal128(15, 2), false),
+            Field::new("p_comment", DataType::Utf8, false),
+        ]),
+
+        "supplier" => Schema::new(vec![
+            Field::new("s_suppkey", DataType::Int64, false),
+            Field::new("s_name", DataType::Utf8, false),
+            Field::new("s_address", DataType::Utf8, false),
+            Field::new("s_nationkey", DataType::Int64, false),
+            Field::new("s_phone", DataType::Utf8, false),
+            Field::new("s_acctbal", DataType::Decimal128(15, 2), false),
+            Field::new("s_comment", DataType::Utf8, false),
+        ]),
+
+        "partsupp" => Schema::new(vec![
+            Field::new("ps_partkey", DataType::Int64, false),
+            Field::new("ps_suppkey", DataType::Int64, false),
+            Field::new("ps_availqty", DataType::Int32, false),
+            Field::new("ps_supplycost", DataType::Decimal128(15, 2), false),
+            Field::new("ps_comment", DataType::Utf8, false),
+        ]),
+
+        "customer" => Schema::new(vec![
+            Field::new("c_custkey", DataType::Int64, false),
+            Field::new("c_name", DataType::Utf8, false),
+            Field::new("c_address", DataType::Utf8, false),
+            Field::new("c_nationkey", DataType::Int64, false),
+            Field::new("c_phone", DataType::Utf8, false),
+            Field::new("c_acctbal", DataType::Decimal128(15, 2), false),
+            Field::new("c_mktsegment", DataType::Utf8, false),
+            Field::new("c_comment", DataType::Utf8, false),
+        ]),
+
+        "orders" => Schema::new(vec![
+            Field::new("o_orderkey", DataType::Int64, false),
+            Field::new("o_custkey", DataType::Int64, false),
+            Field::new("o_orderstatus", DataType::Utf8, false),
+            Field::new("o_totalprice", DataType::Decimal128(15, 2), false),
+            Field::new("o_orderdate", DataType::Date32, false),
+            Field::new("o_orderpriority", DataType::Utf8, false),
+            Field::new("o_clerk", DataType::Utf8, false),
+            Field::new("o_shippriority", DataType::Int32, false),
+            Field::new("o_comment", DataType::Utf8, false),
+        ]),
+
+        "lineitem" => Schema::new(vec![
+            Field::new("l_orderkey", DataType::Int64, false),
+            Field::new("l_partkey", DataType::Int64, false),
+            Field::new("l_suppkey", DataType::Int64, false),
+            Field::new("l_linenumber", DataType::Int32, false),
+            Field::new("l_quantity", DataType::Decimal128(15, 2), false),
+            Field::new("l_extendedprice", DataType::Decimal128(15, 2), false),
+            Field::new("l_discount", DataType::Decimal128(15, 2), false),
+            Field::new("l_tax", DataType::Decimal128(15, 2), false),
+            Field::new("l_returnflag", DataType::Utf8, false),
+            Field::new("l_linestatus", DataType::Utf8, false),
+            Field::new("l_shipdate", DataType::Date32, false),
+            Field::new("l_commitdate", DataType::Date32, false),
+            Field::new("l_receiptdate", DataType::Date32, false),
+            Field::new("l_shipinstruct", DataType::Utf8, false),
+            Field::new("l_shipmode", DataType::Utf8, false),
+            Field::new("l_comment", DataType::Utf8, false),
+        ]),
+
+        "nation" => Schema::new(vec![
+            Field::new("n_nationkey", DataType::Int64, false),
+            Field::new("n_name", DataType::Utf8, false),
+            Field::new("n_regionkey", DataType::Int64, false),
+            Field::new("n_comment", DataType::Utf8, false),
+        ]),
+
+        "region" => Schema::new(vec![
+            Field::new("r_regionkey", DataType::Int64, false),
+            Field::new("r_name", DataType::Utf8, false),
+            Field::new("r_comment", DataType::Utf8, false),
+        ]),
+
+        other => panic!("unknown tpch table {other}"),
+    }
+}
+
+fn make_ctx() -> SessionContext {
+    let config =
+        SessionConfig::new_with_ballista().with_target_partitions(TARGET_PARTITIONS);
+    let ctx = SessionContext::new_with_config(config);
+    for (table, rows) in SF100_ROWS {
+        let schema = Arc::new(tpch_schema(table));
+        ctx.register_table(*table, Arc::new(TpchStatsTable::new(schema, *rows)))
+            .unwrap();
+    }
+    ctx
+}
+
+fn is_query_stmt(stmt: &str) -> bool {
+    let u = stmt.trim_start().to_uppercase();
+    u.starts_with("SELECT") || u.starts_with("WITH")
+}
+
+/// Produce the normalized distributed staged-plan text for a TPC-H query.
+pub async fn staged_plan_text(query_name: &str) -> String {
+    // Read the query SQL directly from the canonical benchmark location rather
+    // than a copy, so a change to a benchmark query surfaces as a golden diff.
+    let sql_path = format!(
+        "{}/../../benchmarks/queries/{query_name}.sql",
+        env!("CARGO_MANIFEST_DIR")
+    );
+    let sql = std::fs::read_to_string(&sql_path)
+        .unwrap_or_else(|e| panic!("read {sql_path}: {e}"));
+
+    let ctx = make_ctx();
+
+    // Split into statements; execute DDL, capture the physical plan of the answer
+    // (last SELECT/WITH) statement. Single-statement queries take the one statement.
+    let stmts: Vec<&str> = sql
+        .split(';')
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+        .collect();
+    let answer_idx = stmts
+        .iter()
+        .rposition(|s| is_query_stmt(s))
+        .expect("no SELECT/WITH statement in query");
+
+    let mut physical = None;
+    for (i, stmt) in stmts.iter().enumerate() {
+        if i == answer_idx {
+            physical = Some(
+                ctx.sql(stmt)
+                    .await
+                    .unwrap()
+                    .create_physical_plan()
+                    .await
+                    .unwrap(),
+            );
+        } else {
+            // DDL such as CREATE VIEW / DROP VIEW (q15) — apply it.
+            ctx.sql(stmt).await.unwrap().collect().await.unwrap();
+        }
+    }
+    let physical = physical.unwrap();
+
+    let mut planner = DefaultDistributedPlanner::new();
+    let state = ctx.state();
+    let job_id: JobId = JOB_ID.into();
+    let stages = planner
+        .plan_query_stages(&job_id, physical, state.config().options())
+        .unwrap();
+
+    let mut out = String::new();
+    for stage in &stages {
+        out.push_str(&format!("=== Stage {} ===\n", stage.stage_id()));
+        let ep: &dyn datafusion::physical_plan::ExecutionPlan = stage.as_ref();
+        out.push_str(&DisplayableExecutionPlan::new(ep).indent(false).to_string());
+        out.push('\n');
+    }
+    normalize(&out)
+}
+
+fn normalize(plan: &str) -> String {
+    // Strip the fixed job id and any hex addresses so output is byte-stable.
+    let s = plan.replace(JOB_ID, "<job_id>");
+    // remove 0x… addresses if any appear
+    let re = regex::Regex::new(r"0x[0-9a-fA-F]+").unwrap();
+    re.replace_all(&s, "0x<addr>").into_owned()
+}
diff --git a/ballista/scheduler/tests/tpch_plan_stability/main.rs b/ballista/scheduler/tests/tpch_plan_stability/main.rs
new file mode 100644
index 0000000000..38d6d5f2a5
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/main.rs
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod fixtures;
+mod stats_table;
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::common::stats::Precision;
+use datafusion::prelude::SessionContext;
+
+use stats_table::TpchStatsTable;
+
+#[tokio::test]
+async fn stats_table_reports_injected_rows() {
+    let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+    let ctx = SessionContext::new();
+    ctx.register_table(
+        "t",
+        Arc::new(TpchStatsTable::new(Arc::clone(&schema), 12_345)),
+    )
+    .unwrap();
+
+    let plan = ctx
+        .sql("SELECT a FROM t")
+        .await
+        .unwrap()
+        .create_physical_plan()
+        .await
+        .unwrap();
+
+    let stats = plan.partition_statistics(None).unwrap();
+    assert_eq!(stats.num_rows, Precision::Inexact(12_345));
+}
+
+#[tokio::test]
+async fn staged_plan_text_is_nonempty_and_shuffled() {
+    let text = fixtures::staged_plan_text("q1").await;
+    assert!(
+        text.contains("ShuffleWriterExec"),
+        "q1 plan should contain shuffle stages:\n{text}"
+    );
+    assert!(
+        !text.contains("plan_stability"),
+        "job id should be normalized out"
+    );
+    assert!(text.contains("=== Stage"), "stage banners present");
+}
+
+#[tokio::test]
+async fn multi_statement_q15_plans() {
+    let text = fixtures::staged_plan_text("q15").await;
+    assert!(
+        text.contains("=== Stage"),
+        "q15 (create/select/drop view) should plan:\n{text}"
+    );
+}
+
+fn golden_path(name: &str) -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/tpch_plan_stability/approved")
+        .join(format!("{name}.txt"))
+}
+
+async fn check_query(name: &str) {
+    let actual = fixtures::staged_plan_text(name).await;
+    let path = golden_path(name);
+    if std::env::var("BALLISTA_GENERATE_GOLDEN").is_ok() {
+        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+        std::fs::write(&path, actual.trim_end().to_string() + "\n").unwrap();
+        return;
+    }
+    let expected = std::fs::read_to_string(&path).unwrap_or_else(|_| {
+        panic!("missing golden {path:?}; regenerate with BALLISTA_GENERATE_GOLDEN=1")
+    });
+    assert_eq!(
+        actual.trim_end(),
+        expected.trim_end(),
+        "distributed plan drift for {name}. Review the change; if intended, regenerate with \
+         BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability"
+    );
+}
+
+macro_rules! plan_stability_test {
+    ($($name:ident),+ $(,)?) => {
+        $(
+            #[tokio::test]
+            async fn $name() { check_query(stringify!($name)).await; }
+        )+
+    };
+}
+
+plan_stability_test!(
+    q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, q15, q16, q17, q18, q19,
+    q20, q21, q22
+);
diff --git a/ballista/scheduler/tests/tpch_plan_stability/stats_table.rs b/ballista/scheduler/tests/tpch_plan_stability/stats_table.rs
new file mode 100644
index 0000000000..2d8d7b1b5d
--- /dev/null
+++ b/ballista/scheduler/tests/tpch_plan_stability/stats_table.rs
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt;
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::catalog::{Session, TableProvider};
+use datafusion::common::stats::Precision;
+use datafusion::common::{Result, Statistics};
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::{Expr, TableType};
+use datafusion::physical_expr::EquivalenceProperties;
+use datafusion::physical_plan::execution_plan::{
+    Boundedness, EmissionType, SchedulingType,
+};
+use datafusion::physical_plan::memory::MemoryStream;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
+    SendableRecordBatchStream,
+};
+
+use async_trait::async_trait;
+
+/// A leaf execution plan that produces no rows but reports a fixed row-count
+/// statistic, so the physical optimizer makes realistic join/broadcast choices
+/// without any data on disk.
+#[derive(Debug)]
+pub struct StatsExec {
+    schema: SchemaRef,
+    num_rows: usize,
+    cache: Arc<PlanProperties>,
+}
+
+impl StatsExec {
+    pub fn new(schema: SchemaRef, num_rows: usize) -> Self {
+        let cache = PlanProperties::new(
+            EquivalenceProperties::new(Arc::clone(&schema)),
+            Partitioning::UnknownPartitioning(1),
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        )
+        .with_scheduling_type(SchedulingType::Cooperative);
+        Self {
+            schema,
+            num_rows,
+            cache: Arc::new(cache),
+        }
+    }
+}
+
+impl DisplayAs for StatsExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+        match t {
+            DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                write!(f, "StatsExec: rows={}", self.num_rows)
+            }
+            DisplayFormatType::TreeRender => write!(f, ""),
+        }
+    }
+}
+
+impl ExecutionPlan for StatsExec {
+    fn name(&self) -> &'static str {
+        "StatsExec"
+    }
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.cache
+    }
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+    fn with_new_children(
+        self: Arc<Self>,
+        _: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(self)
+    }
+    fn execute(
+        &self,
+        _partition: usize,
+        _context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        Ok(Box::pin(MemoryStream::try_new(
+            vec![],
+            Arc::clone(&self.schema),
+            None,
+        )?))
+    }
+    fn partition_statistics(&self, _partition: Option<usize>) -> Result<Arc<Statistics>> {
+        Ok(Arc::new(
+            Statistics::new_unknown(&self.schema)
+                .with_num_rows(Precision::Inexact(self.num_rows)),
+        ))
+    }
+}
+
+/// A dataless `TableProvider` with a real schema and a fixed row-count statistic.
+#[derive(Debug)]
+pub struct TpchStatsTable {
+    schema: SchemaRef,
+    num_rows: usize,
+}
+
+impl TpchStatsTable {
+    pub fn new(schema: SchemaRef, num_rows: usize) -> Self {
+        Self { schema, num_rows }
+    }
+}
+
+#[async_trait]
+impl TableProvider for TpchStatsTable {
+    fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+    fn table_type(&self) -> TableType {
+        TableType::Base
+    }
+    async fn scan(
+        &self,
+        _state: &dyn Session,
+        projection: Option<&Vec<usize>>,
+        _filters: &[Expr],
+        _limit: Option<usize>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let schema = match projection {
+            Some(p) => Arc::new(self.schema.project(p)?),
+            None => Arc::clone(&self.schema),
+        };
+        Ok(Arc::new(StatsExec::new(schema, self.num_rows)))
+    }
+    fn statistics(&self) -> Option<Statistics> {
+        Some(
+            Statistics::new_unknown(&self.schema)
+                .with_num_rows(Precision::Inexact(self.num_rows)),
+        )
+    }
+}
diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore
index 6320cd248d..b1cc226609 100644
--- a/benchmarks/.gitignore
+++ b/benchmarks/.gitignore
@@ -1 +1,2 @@
-data
\ No newline at end of file
+data
+data-tpcds
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index 29694a3b4f..c107aaee5d 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "ballista-benchmarks"
 description = "Ballista Benchmarks"
-version = "53.0.0"
+version = "54.0.0"
 edition = "2024"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
 homepage = "https://datafusion.apache.org/ballista/"
@@ -31,8 +31,10 @@ ci = []
 default = ["mimalloc"]
 
 [dependencies]
-ballista = { path = "../ballista/client", version = "53.0.0" }
-ballista-core = { path = "../ballista/core", version = "53.0.0" }
+ballista = { path = "../ballista/client", version = "54.0.0" }
+ballista-core = { path = "../ballista/core", version = "54.0.0", features = [
+    "build-binary",
+] }
 clap = { workspace = true }
 datafusion = { workspace = true }
 datafusion-proto = { workspace = true }
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 14711180be..a92286cea3 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -298,4 +298,84 @@ $SPARK_HOME/bin/spark-submit \
   --debug
 ```
 
+## TPC-DS Correctness Tests
+
+Unlike the TPC-H suite above (which measures performance), the TPC-DS suite is
+a correctness gate: it runs each query on a Ballista cluster and compares the
+result, row-by-row, against a single-process DataFusion oracle running the
+same query.
+
+### Vendoring the queries
+
+The 99 TPC-DS queries are checked in at `benchmarks/queries-tpcds/qN.sql`, so
+no fetch step is needed to run the harness. They were vendored from DataFusion's
+`branch-54` (the queries DataFusion uses in its own TPC-DS tests) with:
+
+```bash
+./dev/vendor-tpcds-queries.sh
+```
+
+That script fetches `1.sql` .. `99.sql` from `datafusion/core/tests/tpc-ds` on
+the configured branch (`DATAFUSION_BRANCH`, default `branch-54`) and writes them
+to `benchmarks/queries-tpcds/qN.sql` (prefixed with `q` to match the TPC-H
+naming convention). Re-run it only to refresh the committed queries when the
+DataFusion pin changes.
+
+### Generating Test Data
+
+TPC-DS data is generated with the same [tpchgen-rs](https://github.com/clflushopt/tpchgen-rs)
+project as TPC-H, via its `tpcgen-cli tpcds` subcommand. TPC-DS support is not
+yet published to crates.io, so the script installs `tpcgen-cli` from git,
+pinned to a fixed rev:
+
+```bash
+SCALE_FACTOR=1 OUTPUT_DIR=./data-tpcds ./benchmarks/tpcds-gen.sh
+```
+
+`SCALE_FACTOR` (default `1`) and `OUTPUT_DIR` (default `benchmarks/data-tpcds`,
+resolved relative to the script) are env overrides; `TPCGEN_REV` overrides the
+pinned `tpcgen-cli` git rev if needed. Note that, unlike the TPC-H generator, `tpcgen-cli tpcds` has no
+`--parts` flag — it writes a single `<table>.parquet` file per table for all
+24 TPC-DS tables.
+
+### Running the correctness check
+
+Bring up a scheduler and executor as described above in "Running the
+Ballista Benchmarks", then run the `tpcds` binary with `--verify`:
+
+```bash
+cargo run --release --bin tpcds -- \
+  --host localhost --port 50050 \
+  --path $(pwd)/data-tpcds \
+  --partitions 16 \
+  --verify \
+  -c datafusion.optimizer.prefer_hash_join=false
+```
+
+With no `--query` given, this runs every non-skipped query (see "Skip list"
+below) on the Ballista cluster and, because `--verify` is set, also runs it
+against a single-process DataFusion `SessionContext` and diffs the results.
+The process exits non-zero and prints a summary if any query fails or
+mismatches. `--verify` assumes a local `--path`: the oracle context registers
+the tables directly and does not apply object-store credentials.
+
+### Skip list
+
+Some queries are excluded from the gate. These are listed in the `SKIP` const
+near the top of `benchmarks/src/bin/tpcds.rs`, as `(query_id, reason)` pairs,
+grouped by cause: queries whose distributed result diverges from DataFusion
+(tracked as bugs), queries that are non-deterministic under `LIMIT`/`ORDER BY`
+ties, and queries whose `tpcgen-cli` schema column names differ from the
+DataFusion query text. A default run (no `--query`) executes `1..=99` minus
+`SKIP`.
+
+### CI
+
+`.github/workflows/tpcds.yml` runs the SF1 suite against a local
+scheduler + executor on every push/PR touching `ballista/**` or
+`benchmarks/**`, under the default (static) planner. The adaptive planner
+(AQE on) is not gated yet — it currently fails many TPC-DS queries with an
+`EmptyExec invalid partition` assertion (issue #2047); re-enable an AQE-on run
+once that is fixed.
+
 [1]: http://www.tpc.org/tpch/
diff --git a/benchmarks/benches/sort_shuffle.rs b/benchmarks/benches/sort_shuffle.rs
index 749b0f925a..3e135551fb 100644
--- a/benchmarks/benches/sort_shuffle.rs
+++ b/benchmarks/benches/sort_shuffle.rs
@@ -27,7 +27,6 @@ use datafusion::arrow::array::{
     Int64Array, StringArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array,
 };
 use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion::arrow::ipc::CompressionType;
 use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::datasource::memory::MemorySourceConfig;
 use datafusion::datasource::source::DataSourceExec;
@@ -236,7 +235,7 @@ fn run_sort_shuffle(
     };
     let task_ctx = session_ctx.task_ctx();
 
-    let config = SortShuffleConfig::new(true, CompressionType::LZ4_FRAME, 8192);
+    let config = SortShuffleConfig::new(true, 8192);
 
     let writer = SortShuffleWriterExec::try_new(
         "bench_job".into(),
diff --git a/benchmarks/queries-tpcds/q1.sql b/benchmarks/queries-tpcds/q1.sql
new file mode 100644
index 0000000000..b90411d043
--- /dev/null
+++ b/benchmarks/queries-tpcds/q1.sql
@@ -0,0 +1,26 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with customer_total_return as
+(select sr_customer_sk as ctr_customer_sk
+,sr_store_sk as ctr_store_sk
+,sum(SR_FEE) as ctr_total_return
+from store_returns
+,date_dim
+where sr_returned_date_sk = d_date_sk
+and d_year =2000
+group by sr_customer_sk
+,sr_store_sk)
+ select  c_customer_id
+from customer_total_return ctr1
+,store
+,customer
+where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2
+from customer_total_return ctr2
+where ctr1.ctr_store_sk = ctr2.ctr_store_sk)
+and s_store_sk = ctr1.ctr_store_sk
+and s_state = 'SD'
+and ctr1.ctr_customer_sk = c_customer_sk
+order by c_customer_id
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q10.sql b/benchmarks/queries-tpcds/q10.sql
new file mode 100644
index 0000000000..37111a7c1b
--- /dev/null
+++ b/benchmarks/queries-tpcds/q10.sql
@@ -0,0 +1,60 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+  cd_gender,
+  cd_marital_status,
+  cd_education_status,
+  count(*) cnt1,
+  cd_purchase_estimate,
+  count(*) cnt2,
+  cd_credit_rating,
+  count(*) cnt3,
+  cd_dep_count,
+  count(*) cnt4,
+  cd_dep_employed_count,
+  count(*) cnt5,
+  cd_dep_college_count,
+  count(*) cnt6
+ from
+  customer c,customer_address ca,customer_demographics
+ where
+  c.c_current_addr_sk = ca.ca_address_sk and
+  ca_county in ('Walker County','Richland County','Gaines County','Douglas County','Dona Ana County') and
+  cd_demo_sk = c.c_current_cdemo_sk and 
+  exists (select *
+          from store_sales,date_dim
+          where c.c_customer_sk = ss_customer_sk and
+                ss_sold_date_sk = d_date_sk and
+                d_year = 2002 and
+                d_moy between 4 and 4+3) and
+   (exists (select *
+            from web_sales,date_dim
+            where c.c_customer_sk = ws_bill_customer_sk and
+                  ws_sold_date_sk = d_date_sk and
+                  d_year = 2002 and
+                  d_moy between 4 ANd 4+3) or 
+    exists (select * 
+            from catalog_sales,date_dim
+            where c.c_customer_sk = cs_ship_customer_sk and
+                  cs_sold_date_sk = d_date_sk and
+                  d_year = 2002 and
+                  d_moy between 4 and 4+3))
+ group by cd_gender,
+          cd_marital_status,
+          cd_education_status,
+          cd_purchase_estimate,
+          cd_credit_rating,
+          cd_dep_count,
+          cd_dep_employed_count,
+          cd_dep_college_count
+ order by cd_gender,
+          cd_marital_status,
+          cd_education_status,
+          cd_purchase_estimate,
+          cd_credit_rating,
+          cd_dep_count,
+          cd_dep_employed_count,
+          cd_dep_college_count
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q11.sql b/benchmarks/queries-tpcds/q11.sql
new file mode 100644
index 0000000000..776ee37125
--- /dev/null
+++ b/benchmarks/queries-tpcds/q11.sql
@@ -0,0 +1,82 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with year_total as (
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,c_preferred_cust_flag customer_preferred_cust_flag
+       ,c_birth_country customer_birth_country
+       ,c_login customer_login
+       ,c_email_address customer_email_address
+       ,d_year dyear
+       ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total
+       ,'s' sale_type
+ from customer
+     ,store_sales
+     ,date_dim
+ where c_customer_sk = ss_customer_sk
+   and ss_sold_date_sk = d_date_sk
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,c_preferred_cust_flag 
+         ,c_birth_country
+         ,c_login
+         ,c_email_address
+         ,d_year 
+ union all
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,c_preferred_cust_flag customer_preferred_cust_flag
+       ,c_birth_country customer_birth_country
+       ,c_login customer_login
+       ,c_email_address customer_email_address
+       ,d_year dyear
+       ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total
+       ,'w' sale_type
+ from customer
+     ,web_sales
+     ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+   and ws_sold_date_sk = d_date_sk
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,c_preferred_cust_flag 
+         ,c_birth_country
+         ,c_login
+         ,c_email_address
+         ,d_year
+         )
+  select  
+                  t_s_secyear.customer_id
+                 ,t_s_secyear.customer_first_name
+                 ,t_s_secyear.customer_last_name
+                 ,t_s_secyear.customer_email_address
+ from year_total t_s_firstyear
+     ,year_total t_s_secyear
+     ,year_total t_w_firstyear
+     ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+         and t_s_firstyear.customer_id = t_w_secyear.customer_id
+         and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+         and t_s_firstyear.sale_type = 's'
+         and t_w_firstyear.sale_type = 'w'
+         and t_s_secyear.sale_type = 's'
+         and t_w_secyear.sale_type = 'w'
+         and t_s_firstyear.dyear = 2001
+         and t_s_secyear.dyear = 2001+1
+         and t_w_firstyear.dyear = 2001
+         and t_w_secyear.dyear = 2001+1
+         and t_s_firstyear.year_total > 0
+         and t_w_firstyear.year_total > 0
+         and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end
+             > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end
+ order by t_s_secyear.customer_id
+         ,t_s_secyear.customer_first_name
+         ,t_s_secyear.customer_last_name
+         ,t_s_secyear.customer_email_address
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q12.sql b/benchmarks/queries-tpcds/q12.sql
new file mode 100644
index 0000000000..c624417595
--- /dev/null
+++ b/benchmarks/queries-tpcds/q12.sql
@@ -0,0 +1,35 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id
+      ,i_item_desc 
+      ,i_category 
+      ,i_class 
+      ,i_current_price
+      ,sum(ws_ext_sales_price) as itemrevenue 
+      ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over
+          (partition by i_class) as revenueratio
+from	
+	web_sales
+    	,item 
+    	,date_dim
+where 
+	ws_item_sk = i_item_sk 
+  	and i_category in ('Jewelry', 'Sports', 'Books')
+  	and ws_sold_date_sk = d_date_sk
+	and d_date between cast('2001-01-12' as date) 
+				and cast('2001-01-12' as date) + INTERVAL '30 days'
+group by 
+	i_item_id
+        ,i_item_desc 
+        ,i_category
+        ,i_class
+        ,i_current_price
+order by 
+	i_category
+        ,i_class
+        ,i_item_id
+        ,i_item_desc
+        ,revenueratio
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q13.sql b/benchmarks/queries-tpcds/q13.sql
new file mode 100644
index 0000000000..d0b85b7b21
--- /dev/null
+++ b/benchmarks/queries-tpcds/q13.sql
@@ -0,0 +1,53 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select avg(ss_quantity)
+       ,avg(ss_ext_sales_price)
+       ,avg(ss_ext_wholesale_cost)
+       ,sum(ss_ext_wholesale_cost)
+ from store_sales
+     ,store
+     ,customer_demographics
+     ,household_demographics
+     ,customer_address
+     ,date_dim
+ where s_store_sk = ss_store_sk
+ and  ss_sold_date_sk = d_date_sk and d_year = 2001
+ and((ss_hdemo_sk=hd_demo_sk
+  and cd_demo_sk = ss_cdemo_sk
+  and cd_marital_status = 'D'
+  and cd_education_status = '2 yr Degree'
+  and ss_sales_price between 100.00 and 150.00
+  and hd_dep_count = 3   
+     )or
+     (ss_hdemo_sk=hd_demo_sk
+  and cd_demo_sk = ss_cdemo_sk
+  and cd_marital_status = 'S'
+  and cd_education_status = 'Secondary'
+  and ss_sales_price between 50.00 and 100.00   
+  and hd_dep_count = 1
+     ) or 
+     (ss_hdemo_sk=hd_demo_sk
+  and cd_demo_sk = ss_cdemo_sk
+  and cd_marital_status = 'W'
+  and cd_education_status = 'Advanced Degree'
+  and ss_sales_price between 150.00 and 200.00 
+  and hd_dep_count = 1  
+     ))
+ and((ss_addr_sk = ca_address_sk
+  and ca_country = 'United States'
+  and ca_state in ('CO', 'IL', 'MN')
+  and ss_net_profit between 100 and 200  
+     ) or
+     (ss_addr_sk = ca_address_sk
+  and ca_country = 'United States'
+  and ca_state in ('OH', 'MT', 'NM')
+  and ss_net_profit between 150 and 300  
+     ) or
+     (ss_addr_sk = ca_address_sk
+  and ca_country = 'United States'
+  and ca_state in ('TX', 'MO', 'MI')
+  and ss_net_profit between 50 and 250  
+     ))
+;
+
+
diff --git a/benchmarks/queries-tpcds/q14.sql b/benchmarks/queries-tpcds/q14.sql
new file mode 100644
index 0000000000..97af50bbbc
--- /dev/null
+++ b/benchmarks/queries-tpcds/q14.sql
@@ -0,0 +1,211 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with  cross_items as
+ (select i_item_sk ss_item_sk
+ from item,
+ (select iss.i_brand_id brand_id
+     ,iss.i_class_id class_id
+     ,iss.i_category_id category_id
+ from store_sales
+     ,item iss
+     ,date_dim d1
+ where ss_item_sk = iss.i_item_sk
+   and ss_sold_date_sk = d1.d_date_sk
+   and d1.d_year between 1998 AND 1998 + 2
+ intersect 
+ select ics.i_brand_id
+     ,ics.i_class_id
+     ,ics.i_category_id
+ from catalog_sales
+     ,item ics
+     ,date_dim d2
+ where cs_item_sk = ics.i_item_sk
+   and cs_sold_date_sk = d2.d_date_sk
+   and d2.d_year between 1998 AND 1998 + 2
+ intersect
+ select iws.i_brand_id
+     ,iws.i_class_id
+     ,iws.i_category_id
+ from web_sales
+     ,item iws
+     ,date_dim d3
+ where ws_item_sk = iws.i_item_sk
+   and ws_sold_date_sk = d3.d_date_sk
+   and d3.d_year between 1998 AND 1998 + 2)
+ where i_brand_id = brand_id
+      and i_class_id = class_id
+      and i_category_id = category_id
+),
+ avg_sales as
+ (select avg(quantity*list_price) average_sales
+  from (select ss_quantity quantity
+             ,ss_list_price list_price
+       from store_sales
+           ,date_dim
+       where ss_sold_date_sk = d_date_sk
+         and d_year between 1998 and 1998 + 2
+       union all 
+       select cs_quantity quantity 
+             ,cs_list_price list_price
+       from catalog_sales
+           ,date_dim
+       where cs_sold_date_sk = d_date_sk
+         and d_year between 1998 and 1998 + 2 
+       union all
+       select ws_quantity quantity
+             ,ws_list_price list_price
+       from web_sales
+           ,date_dim
+       where ws_sold_date_sk = d_date_sk
+         and d_year between 1998 and 1998 + 2) x)
+  select  channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales)
+ from(
+       select 'store' channel, i_brand_id,i_class_id
+             ,i_category_id,sum(ss_quantity*ss_list_price) sales
+             , count(*) number_sales
+       from store_sales
+           ,item
+           ,date_dim
+       where ss_item_sk in (select ss_item_sk from cross_items)
+         and ss_item_sk = i_item_sk
+         and ss_sold_date_sk = d_date_sk
+         and d_year = 1998+2 
+         and d_moy = 11
+       group by i_brand_id,i_class_id,i_category_id
+       having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)
+       union all
+       select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales
+       from catalog_sales
+           ,item
+           ,date_dim
+       where cs_item_sk in (select ss_item_sk from cross_items)
+         and cs_item_sk = i_item_sk
+         and cs_sold_date_sk = d_date_sk
+         and d_year = 1998+2 
+         and d_moy = 11
+       group by i_brand_id,i_class_id,i_category_id
+       having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales)
+       union all
+       select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales
+       from web_sales
+           ,item
+           ,date_dim
+       where ws_item_sk in (select ss_item_sk from cross_items)
+         and ws_item_sk = i_item_sk
+         and ws_sold_date_sk = d_date_sk
+         and d_year = 1998+2
+         and d_moy = 11
+       group by i_brand_id,i_class_id,i_category_id
+       having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales)
+ ) y
+ group by rollup (channel, i_brand_id,i_class_id,i_category_id)
+ order by channel,i_brand_id,i_class_id,i_category_id
+ limit 100;
+with  cross_items as
+ (select i_item_sk ss_item_sk
+ from item,
+ (select iss.i_brand_id brand_id
+     ,iss.i_class_id class_id
+     ,iss.i_category_id category_id
+ from store_sales
+     ,item iss
+     ,date_dim d1
+ where ss_item_sk = iss.i_item_sk
+   and ss_sold_date_sk = d1.d_date_sk
+   and d1.d_year between 1998 AND 1998 + 2
+ intersect
+ select ics.i_brand_id
+     ,ics.i_class_id
+     ,ics.i_category_id
+ from catalog_sales
+     ,item ics
+     ,date_dim d2
+ where cs_item_sk = ics.i_item_sk
+   and cs_sold_date_sk = d2.d_date_sk
+   and d2.d_year between 1998 AND 1998 + 2
+ intersect
+ select iws.i_brand_id
+     ,iws.i_class_id
+     ,iws.i_category_id
+ from web_sales
+     ,item iws
+     ,date_dim d3
+ where ws_item_sk = iws.i_item_sk
+   and ws_sold_date_sk = d3.d_date_sk
+   and d3.d_year between 1998 AND 1998 + 2) x
+ where i_brand_id = brand_id
+      and i_class_id = class_id
+      and i_category_id = category_id
+),
+ avg_sales as
+(select avg(quantity*list_price) average_sales
+  from (select ss_quantity quantity
+             ,ss_list_price list_price
+       from store_sales
+           ,date_dim
+       where ss_sold_date_sk = d_date_sk
+         and d_year between 1998 and 1998 + 2
+       union all
+       select cs_quantity quantity
+             ,cs_list_price list_price
+       from catalog_sales
+           ,date_dim
+       where cs_sold_date_sk = d_date_sk
+         and d_year between 1998 and 1998 + 2
+       union all
+       select ws_quantity quantity
+             ,ws_list_price list_price
+       from web_sales
+           ,date_dim
+       where ws_sold_date_sk = d_date_sk
+         and d_year between 1998 and 1998 + 2) x)
+  select  this_year.channel ty_channel
+                           ,this_year.i_brand_id ty_brand
+                           ,this_year.i_class_id ty_class
+                           ,this_year.i_category_id ty_category
+                           ,this_year.sales ty_sales
+                           ,this_year.number_sales ty_number_sales
+                           ,last_year.channel ly_channel
+                           ,last_year.i_brand_id ly_brand
+                           ,last_year.i_class_id ly_class
+                           ,last_year.i_category_id ly_category
+                           ,last_year.sales ly_sales
+                           ,last_year.number_sales ly_number_sales 
+ from
+ (select 'store' channel, i_brand_id,i_class_id,i_category_id
+        ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales
+ from store_sales 
+     ,item
+     ,date_dim
+ where ss_item_sk in (select ss_item_sk from cross_items)
+   and ss_item_sk = i_item_sk
+   and ss_sold_date_sk = d_date_sk
+   and d_week_seq = (select d_week_seq
+                     from date_dim
+                     where d_year = 1998 + 1
+                       and d_moy = 12
+                       and d_dom = 16)
+ group by i_brand_id,i_class_id,i_category_id
+ having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year,
+ (select 'store' channel, i_brand_id,i_class_id
+        ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales
+ from store_sales
+     ,item
+     ,date_dim
+ where ss_item_sk in (select ss_item_sk from cross_items)
+   and ss_item_sk = i_item_sk
+   and ss_sold_date_sk = d_date_sk
+   and d_week_seq = (select d_week_seq
+                     from date_dim
+                     where d_year = 1998
+                       and d_moy = 12
+                       and d_dom = 16)
+ group by i_brand_id,i_class_id,i_category_id
+ having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year
+ where this_year.i_brand_id= last_year.i_brand_id
+   and this_year.i_class_id = last_year.i_class_id
+   and this_year.i_category_id = last_year.i_category_id
+ order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q15.sql b/benchmarks/queries-tpcds/q15.sql
new file mode 100644
index 0000000000..37494c0308
--- /dev/null
+++ b/benchmarks/queries-tpcds/q15.sql
@@ -0,0 +1,21 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  ca_zip
+       ,sum(cs_sales_price)
+ from catalog_sales
+     ,customer
+     ,customer_address
+     ,date_dim
+ where cs_bill_customer_sk = c_customer_sk
+ 	and c_current_addr_sk = ca_address_sk 
+ 	and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475',
+                                   '85392', '85460', '80348', '81792')
+ 	      or ca_state in ('CA','WA','GA')
+ 	      or cs_sales_price > 500)
+ 	and cs_sold_date_sk = d_date_sk
+ 	and d_qoy = 2 and d_year = 2000
+ group by ca_zip
+ order by ca_zip
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q16.sql b/benchmarks/queries-tpcds/q16.sql
new file mode 100644
index 0000000000..83edc5b198
--- /dev/null
+++ b/benchmarks/queries-tpcds/q16.sql
@@ -0,0 +1,32 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   count(distinct cs_order_number) as 'order count'
+  ,sum(cs_ext_ship_cost) as 'total shipping cost'
+  ,sum(cs_net_profit) as 'total net profit'
+from
+   catalog_sales cs1
+  ,date_dim
+  ,customer_address
+  ,call_center
+where
+    d_date between '1999-4-01' and 
+           (cast('1999-4-01' as date) + INTERVAL '60 days')
+and cs1.cs_ship_date_sk = d_date_sk
+and cs1.cs_ship_addr_sk = ca_address_sk
+and ca_state = 'GA'
+and cs1.cs_call_center_sk = cc_call_center_sk
+and cc_county in ('Daviess County','Franklin Parish','Barrow County','Luce County',
+                  'Fairfield County'
+)
+and exists (select *
+            from catalog_sales cs2
+            where cs1.cs_order_number = cs2.cs_order_number
+              and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk)
+and not exists(select *
+               from catalog_returns cr1
+               where cs1.cs_order_number = cr1.cr_order_number)
+order by count(distinct cs_order_number)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q17.sql b/benchmarks/queries-tpcds/q17.sql
new file mode 100644
index 0000000000..ebf93109b0
--- /dev/null
+++ b/benchmarks/queries-tpcds/q17.sql
@@ -0,0 +1,46 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id
+       ,i_item_desc
+       ,s_state
+       ,count(ss_quantity) as store_sales_quantitycount
+       ,avg(ss_quantity) as store_sales_quantityave
+       ,stddev_samp(ss_quantity) as store_sales_quantitystdev
+       ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov
+       ,count(sr_return_quantity) as store_returns_quantitycount
+       ,avg(sr_return_quantity) as store_returns_quantityave
+       ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev
+       ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov
+       ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave
+       ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev
+       ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov
+ from store_sales
+     ,store_returns
+     ,catalog_sales
+     ,date_dim d1
+     ,date_dim d2
+     ,date_dim d3
+     ,store
+     ,item
+ where d1.d_quarter_name = '1998Q1'
+   and d1.d_date_sk = ss_sold_date_sk
+   and i_item_sk = ss_item_sk
+   and s_store_sk = ss_store_sk
+   and ss_customer_sk = sr_customer_sk
+   and ss_item_sk = sr_item_sk
+   and ss_ticket_number = sr_ticket_number
+   and sr_returned_date_sk = d2.d_date_sk
+   and d2.d_quarter_name in ('1998Q1','1998Q2','1998Q3')
+   and sr_customer_sk = cs_bill_customer_sk
+   and sr_item_sk = cs_item_sk
+   and cs_sold_date_sk = d3.d_date_sk
+   and d3.d_quarter_name in ('1998Q1','1998Q2','1998Q3')
+ group by i_item_id
+         ,i_item_desc
+         ,s_state
+ order by i_item_id
+         ,i_item_desc
+         ,s_state
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q18.sql b/benchmarks/queries-tpcds/q18.sql
new file mode 100644
index 0000000000..fb1101906b
--- /dev/null
+++ b/benchmarks/queries-tpcds/q18.sql
@@ -0,0 +1,35 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id,
+        ca_country,
+        ca_state, 
+        ca_county,
+        avg( cast(cs_quantity as decimal(12,2))) agg1,
+        avg( cast(cs_list_price as decimal(12,2))) agg2,
+        avg( cast(cs_coupon_amt as decimal(12,2))) agg3,
+        avg( cast(cs_sales_price as decimal(12,2))) agg4,
+        avg( cast(cs_net_profit as decimal(12,2))) agg5,
+        avg( cast(c_birth_year as decimal(12,2))) agg6,
+        avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7
+ from catalog_sales, customer_demographics cd1, 
+      customer_demographics cd2, customer, customer_address, date_dim, item
+ where cs_sold_date_sk = d_date_sk and
+       cs_item_sk = i_item_sk and
+       cs_bill_cdemo_sk = cd1.cd_demo_sk and
+       cs_bill_customer_sk = c_customer_sk and
+       cd1.cd_gender = 'M' and 
+       cd1.cd_education_status = 'College' and
+       c_current_cdemo_sk = cd2.cd_demo_sk and
+       c_current_addr_sk = ca_address_sk and
+       c_birth_month in (9,5,12,4,1,10) and
+       d_year = 2001 and
+       ca_state in ('ND','WI','AL'
+                   ,'NC','OK','MS','TN')
+ group by rollup (i_item_id, ca_country, ca_state, ca_county)
+ order by ca_country,
+        ca_state, 
+        ca_county,
+	i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q19.sql b/benchmarks/queries-tpcds/q19.sql
new file mode 100644
index 0000000000..d6908a02e8
--- /dev/null
+++ b/benchmarks/queries-tpcds/q19.sql
@@ -0,0 +1,26 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact,
+ 	sum(ss_ext_sales_price) ext_price
+ from date_dim, store_sales, item,customer,customer_address,store
+ where d_date_sk = ss_sold_date_sk
+   and ss_item_sk = i_item_sk
+   and i_manager_id=7
+   and d_moy=11
+   and d_year=1999
+   and ss_customer_sk = c_customer_sk 
+   and c_current_addr_sk = ca_address_sk
+   and substr(ca_zip,1,5) <> substr(s_zip,1,5) 
+   and ss_store_sk = s_store_sk 
+ group by i_brand
+      ,i_brand_id
+      ,i_manufact_id
+      ,i_manufact
+ order by ext_price desc
+         ,i_brand
+         ,i_brand_id
+         ,i_manufact_id
+         ,i_manufact
+limit 100 ;
+
+
diff --git a/benchmarks/queries-tpcds/q2.sql b/benchmarks/queries-tpcds/q2.sql
new file mode 100644
index 0000000000..4fb1813e95
--- /dev/null
+++ b/benchmarks/queries-tpcds/q2.sql
@@ -0,0 +1,61 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with wscs as
+ (select sold_date_sk
+        ,sales_price
+  from (select ws_sold_date_sk sold_date_sk
+              ,ws_ext_sales_price sales_price
+        from web_sales 
+        union all
+        select cs_sold_date_sk sold_date_sk
+              ,cs_ext_sales_price sales_price
+        from catalog_sales)),
+ wswscs as 
+ (select d_week_seq,
+        sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales,
+        sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales,
+        sum(case when (d_day_name='Tuesday') then sales_price else  null end) tue_sales,
+        sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales,
+        sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales,
+        sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales,
+        sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales
+ from wscs
+     ,date_dim
+ where d_date_sk = sold_date_sk
+ group by d_week_seq)
+ select d_week_seq1
+       ,round(sun_sales1/sun_sales2,2)
+       ,round(mon_sales1/mon_sales2,2)
+       ,round(tue_sales1/tue_sales2,2)
+       ,round(wed_sales1/wed_sales2,2)
+       ,round(thu_sales1/thu_sales2,2)
+       ,round(fri_sales1/fri_sales2,2)
+       ,round(sat_sales1/sat_sales2,2)
+ from
+ (select wswscs.d_week_seq d_week_seq1
+        ,sun_sales sun_sales1
+        ,mon_sales mon_sales1
+        ,tue_sales tue_sales1
+        ,wed_sales wed_sales1
+        ,thu_sales thu_sales1
+        ,fri_sales fri_sales1
+        ,sat_sales sat_sales1
+  from wswscs,date_dim 
+  where date_dim.d_week_seq = wswscs.d_week_seq and
+        d_year = 2001) y,
+ (select wswscs.d_week_seq d_week_seq2
+        ,sun_sales sun_sales2
+        ,mon_sales mon_sales2
+        ,tue_sales tue_sales2
+        ,wed_sales wed_sales2
+        ,thu_sales thu_sales2
+        ,fri_sales fri_sales2
+        ,sat_sales sat_sales2
+  from wswscs
+      ,date_dim 
+  where date_dim.d_week_seq = wswscs.d_week_seq and
+        d_year = 2001+1) z
+ where d_week_seq1=d_week_seq2-53
+ order by d_week_seq1;
+
+
diff --git a/benchmarks/queries-tpcds/q20.sql b/benchmarks/queries-tpcds/q20.sql
new file mode 100644
index 0000000000..7d45843269
--- /dev/null
+++ b/benchmarks/queries-tpcds/q20.sql
@@ -0,0 +1,31 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id
+       ,i_item_desc 
+       ,i_category 
+       ,i_class 
+       ,i_current_price
+       ,sum(cs_ext_sales_price) as itemrevenue 
+       ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over
+           (partition by i_class) as revenueratio
+ from	catalog_sales
+     ,item 
+     ,date_dim
+ where cs_item_sk = i_item_sk 
+   and i_category in ('Jewelry', 'Sports', 'Books')
+   and cs_sold_date_sk = d_date_sk
+ and d_date between cast('2001-01-12' as date) 
+ 				and (cast('2001-01-12' as date) + INTERVAL '30 days')
+ group by i_item_id
+         ,i_item_desc 
+         ,i_category
+         ,i_class
+         ,i_current_price
+ order by i_category
+         ,i_class
+         ,i_item_id
+         ,i_item_desc
+         ,revenueratio
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q21.sql b/benchmarks/queries-tpcds/q21.sql
new file mode 100644
index 0000000000..7d1ea8a7a9
--- /dev/null
+++ b/benchmarks/queries-tpcds/q21.sql
@@ -0,0 +1,31 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  *
+ from(select w_warehouse_name
+            ,i_item_id
+            ,sum(case when (cast(d_date as date) < cast ('1998-04-08' as date))
+	                then inv_quantity_on_hand 
+                      else 0 end) as inv_before
+            ,sum(case when (cast(d_date as date) >= cast ('1998-04-08' as date))
+                      then inv_quantity_on_hand 
+                      else 0 end) as inv_after
+   from inventory
+       ,warehouse
+       ,item
+       ,date_dim
+   where i_current_price between 0.99 and 1.49
+     and i_item_sk          = inv_item_sk
+     and inv_warehouse_sk   = w_warehouse_sk
+     and inv_date_sk    = d_date_sk
+     and d_date between (cast ('1998-04-08' as date) - INTERVAL '30 days')
+                    and (cast ('1998-04-08' as date) + INTERVAL '30 days')
+   group by w_warehouse_name, i_item_id) x
+ where (case when inv_before > 0 
+             then inv_after / inv_before 
+             else null
+             end) between 2.0/3.0 and 3.0/2.0
+ order by w_warehouse_name
+         ,i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q22.sql b/benchmarks/queries-tpcds/q22.sql
new file mode 100644
index 0000000000..9f0dccf334
--- /dev/null
+++ b/benchmarks/queries-tpcds/q22.sql
@@ -0,0 +1,21 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_product_name
+             ,i_brand
+             ,i_class
+             ,i_category
+             ,avg(inv_quantity_on_hand) qoh
+       from inventory
+           ,date_dim
+           ,item
+       where inv_date_sk=d_date_sk
+              and inv_item_sk=i_item_sk
+              and d_month_seq between 1212 and 1212 + 11
+       group by rollup(i_product_name
+                       ,i_brand
+                       ,i_class
+                       ,i_category)
+order by qoh, i_product_name, i_brand, i_class, i_category
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q23.sql b/benchmarks/queries-tpcds/q23.sql
new file mode 100644
index 0000000000..d16cfe65d8
--- /dev/null
+++ b/benchmarks/queries-tpcds/q23.sql
@@ -0,0 +1,108 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with frequent_ss_items as 
+ (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
+  from store_sales
+      ,date_dim 
+      ,item
+  where ss_sold_date_sk = d_date_sk
+    and ss_item_sk = i_item_sk 
+    and d_year in (1999,1999+1,1999+2,1999+3)
+  group by substr(i_item_desc,1,30),i_item_sk,d_date
+  having count(*) >4),
+ max_store_sales as
+ (select max(csales) tpcds_cmax 
+  from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales
+        from store_sales
+            ,customer
+            ,date_dim 
+        where ss_customer_sk = c_customer_sk
+         and ss_sold_date_sk = d_date_sk
+         and d_year in (1999,1999+1,1999+2,1999+3) 
+        group by c_customer_sk)),
+ best_ss_customer as
+ (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales
+  from store_sales
+      ,customer
+  where ss_customer_sk = c_customer_sk
+  group by c_customer_sk
+  having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select
+  *
+from
+ max_store_sales))
+  select  sum(sales)
+ from (select cs_quantity*cs_list_price sales
+       from catalog_sales
+           ,date_dim 
+       where d_year = 1999 
+         and d_moy = 1 
+         and cs_sold_date_sk = d_date_sk 
+         and cs_item_sk in (select item_sk from frequent_ss_items)
+         and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer)
+      union all
+      select ws_quantity*ws_list_price sales
+       from web_sales 
+           ,date_dim 
+       where d_year = 1999 
+         and d_moy = 1 
+         and ws_sold_date_sk = d_date_sk 
+         and ws_item_sk in (select item_sk from frequent_ss_items)
+         and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) 
+ limit 100;
+with frequent_ss_items as
+ (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
+  from store_sales
+      ,date_dim
+      ,item
+  where ss_sold_date_sk = d_date_sk
+    and ss_item_sk = i_item_sk
+    and d_year in (1999,1999 + 1,1999 + 2,1999 + 3)
+  group by substr(i_item_desc,1,30),i_item_sk,d_date
+  having count(*) >4),
+ max_store_sales as
+ (select max(csales) tpcds_cmax
+  from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales
+        from store_sales
+            ,customer
+            ,date_dim 
+        where ss_customer_sk = c_customer_sk
+         and ss_sold_date_sk = d_date_sk
+         and d_year in (1999,1999+1,1999+2,1999+3)
+        group by c_customer_sk)),
+ best_ss_customer as
+ (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales
+  from store_sales
+      ,customer
+  where ss_customer_sk = c_customer_sk
+  group by c_customer_sk
+  having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select
+  *
+ from max_store_sales))
+  select  c_last_name,c_first_name,sales
+ from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales
+        from catalog_sales
+            ,customer
+            ,date_dim 
+        where d_year = 1999 
+         and d_moy = 1 
+         and cs_sold_date_sk = d_date_sk 
+         and cs_item_sk in (select item_sk from frequent_ss_items)
+         and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer)
+         and cs_bill_customer_sk = c_customer_sk 
+       group by c_last_name,c_first_name
+      union all
+      select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales
+       from web_sales
+           ,customer
+           ,date_dim 
+       where d_year = 1999 
+         and d_moy = 1 
+         and ws_sold_date_sk = d_date_sk 
+         and ws_item_sk in (select item_sk from frequent_ss_items)
+         and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)
+         and ws_bill_customer_sk = c_customer_sk
+       group by c_last_name,c_first_name) 
+     order by c_last_name,c_first_name,sales
+  limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q24.sql b/benchmarks/queries-tpcds/q24.sql
new file mode 100644
index 0000000000..417b609db7
--- /dev/null
+++ b/benchmarks/queries-tpcds/q24.sql
@@ -0,0 +1,108 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ssales as
+(select c_last_name
+      ,c_first_name
+      ,s_store_name
+      ,ca_state
+      ,s_state
+      ,i_color
+      ,i_current_price
+      ,i_manager_id
+      ,i_units
+      ,i_size
+      ,sum(ss_sales_price) netpaid
+from store_sales
+    ,store_returns
+    ,store
+    ,item
+    ,customer
+    ,customer_address
+where ss_ticket_number = sr_ticket_number
+  and ss_item_sk = sr_item_sk
+  and ss_customer_sk = c_customer_sk
+  and ss_item_sk = i_item_sk
+  and ss_store_sk = s_store_sk
+  and c_current_addr_sk = ca_address_sk
+  and c_birth_country <> upper(ca_country)
+  and s_zip = ca_zip
+and s_market_id=7
+group by c_last_name
+        ,c_first_name
+        ,s_store_name
+        ,ca_state
+        ,s_state
+        ,i_color
+        ,i_current_price
+        ,i_manager_id
+        ,i_units
+        ,i_size)
+select c_last_name
+      ,c_first_name
+      ,s_store_name
+      ,sum(netpaid) paid
+from ssales
+where i_color = 'orchid'
+group by c_last_name
+        ,c_first_name
+        ,s_store_name
+having sum(netpaid) > (select 0.05*avg(netpaid)
+                                 from ssales)
+order by c_last_name
+        ,c_first_name
+        ,s_store_name
+;
+with ssales as
+(select c_last_name
+      ,c_first_name
+      ,s_store_name
+      ,ca_state
+      ,s_state
+      ,i_color
+      ,i_current_price
+      ,i_manager_id
+      ,i_units
+      ,i_size
+      ,sum(ss_sales_price) netpaid
+from store_sales
+    ,store_returns
+    ,store
+    ,item
+    ,customer
+    ,customer_address
+where ss_ticket_number = sr_ticket_number
+  and ss_item_sk = sr_item_sk
+  and ss_customer_sk = c_customer_sk
+  and ss_item_sk = i_item_sk
+  and ss_store_sk = s_store_sk
+  and c_current_addr_sk = ca_address_sk
+  and c_birth_country <> upper(ca_country)
+  and s_zip = ca_zip
+  and s_market_id = 7
+group by c_last_name
+        ,c_first_name
+        ,s_store_name
+        ,ca_state
+        ,s_state
+        ,i_color
+        ,i_current_price
+        ,i_manager_id
+        ,i_units
+        ,i_size)
+select c_last_name
+      ,c_first_name
+      ,s_store_name
+      ,sum(netpaid) paid
+from ssales
+where i_color = 'chiffon'
+group by c_last_name
+        ,c_first_name
+        ,s_store_name
+having sum(netpaid) > (select 0.05*avg(netpaid)
+                           from ssales)
+order by c_last_name
+        ,c_first_name
+        ,s_store_name
+;
+
+
diff --git a/benchmarks/queries-tpcds/q25.sql b/benchmarks/queries-tpcds/q25.sql
new file mode 100644
index 0000000000..d3eccad89a
--- /dev/null
+++ b/benchmarks/queries-tpcds/q25.sql
@@ -0,0 +1,49 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+ i_item_id
+ ,i_item_desc
+ ,s_store_id
+ ,s_store_name
+ ,sum(ss_net_profit) as store_sales_profit
+ ,sum(sr_net_loss) as store_returns_loss
+ ,sum(cs_net_profit) as catalog_sales_profit
+ from
+ store_sales
+ ,store_returns
+ ,catalog_sales
+ ,date_dim d1
+ ,date_dim d2
+ ,date_dim d3
+ ,store
+ ,item
+ where
+ d1.d_moy = 4
+ and d1.d_year = 2000
+ and d1.d_date_sk = ss_sold_date_sk
+ and i_item_sk = ss_item_sk
+ and s_store_sk = ss_store_sk
+ and ss_customer_sk = sr_customer_sk
+ and ss_item_sk = sr_item_sk
+ and ss_ticket_number = sr_ticket_number
+ and sr_returned_date_sk = d2.d_date_sk
+ and d2.d_moy               between 4 and  10
+ and d2.d_year              = 2000
+ and sr_customer_sk = cs_bill_customer_sk
+ and sr_item_sk = cs_item_sk
+ and cs_sold_date_sk = d3.d_date_sk
+ and d3.d_moy               between 4 and  10 
+ and d3.d_year              = 2000
+ group by
+ i_item_id
+ ,i_item_desc
+ ,s_store_id
+ ,s_store_name
+ order by
+ i_item_id
+ ,i_item_desc
+ ,s_store_id
+ ,s_store_name
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q26.sql b/benchmarks/queries-tpcds/q26.sql
new file mode 100644
index 0000000000..5c71a1e241
--- /dev/null
+++ b/benchmarks/queries-tpcds/q26.sql
@@ -0,0 +1,22 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id, 
+        avg(cs_quantity) agg1,
+        avg(cs_list_price) agg2,
+        avg(cs_coupon_amt) agg3,
+        avg(cs_sales_price) agg4 
+ from catalog_sales, customer_demographics, date_dim, item, promotion
+ where cs_sold_date_sk = d_date_sk and
+       cs_item_sk = i_item_sk and
+       cs_bill_cdemo_sk = cd_demo_sk and
+       cs_promo_sk = p_promo_sk and
+       cd_gender = 'F' and 
+       cd_marital_status = 'W' and
+       cd_education_status = 'Primary' and
+       (p_channel_email = 'N' or p_channel_event = 'N') and
+       d_year = 1998 
+ group by i_item_id
+ order by i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q27.sql b/benchmarks/queries-tpcds/q27.sql
new file mode 100644
index 0000000000..e060cc2681
--- /dev/null
+++ b/benchmarks/queries-tpcds/q27.sql
@@ -0,0 +1,24 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id,
+        s_state, grouping(s_state) g_state,
+        avg(ss_quantity) agg1,
+        avg(ss_list_price) agg2,
+        avg(ss_coupon_amt) agg3,
+        avg(ss_sales_price) agg4
+ from store_sales, customer_demographics, date_dim, store, item
+ where ss_sold_date_sk = d_date_sk and
+       ss_item_sk = i_item_sk and
+       ss_store_sk = s_store_sk and
+       ss_cdemo_sk = cd_demo_sk and
+       cd_gender = 'M' and
+       cd_marital_status = 'W' and
+       cd_education_status = 'College' and
+       d_year = 2002 and
+       s_state in ('MO','LA', 'GA', 'MI', 'SC', 'OH')
+ group by rollup (i_item_id, s_state)
+ order by i_item_id
+         ,s_state
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q28.sql b/benchmarks/queries-tpcds/q28.sql
new file mode 100644
index 0000000000..e177b3ea79
--- /dev/null
+++ b/benchmarks/queries-tpcds/q28.sql
@@ -0,0 +1,54 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  *
+from (select avg(ss_list_price) B1_LP
+            ,count(ss_list_price) B1_CNT
+            ,count(distinct ss_list_price) B1_CNTD
+      from store_sales
+      where ss_quantity between 0 and 5
+        and (ss_list_price between 11 and 11+10 
+             or ss_coupon_amt between 460 and 460+1000
+             or ss_wholesale_cost between 14 and 14+20)) B1,
+     (select avg(ss_list_price) B2_LP
+            ,count(ss_list_price) B2_CNT
+            ,count(distinct ss_list_price) B2_CNTD
+      from store_sales
+      where ss_quantity between 6 and 10
+        and (ss_list_price between 91 and 91+10
+          or ss_coupon_amt between 1430 and 1430+1000
+          or ss_wholesale_cost between 32 and 32+20)) B2,
+     (select avg(ss_list_price) B3_LP
+            ,count(ss_list_price) B3_CNT
+            ,count(distinct ss_list_price) B3_CNTD
+      from store_sales
+      where ss_quantity between 11 and 15
+        and (ss_list_price between 66 and 66+10
+          or ss_coupon_amt between 920 and 920+1000
+          or ss_wholesale_cost between 4 and 4+20)) B3,
+     (select avg(ss_list_price) B4_LP
+            ,count(ss_list_price) B4_CNT
+            ,count(distinct ss_list_price) B4_CNTD
+      from store_sales
+      where ss_quantity between 16 and 20
+        and (ss_list_price between 142 and 142+10
+          or ss_coupon_amt between 3054 and 3054+1000
+          or ss_wholesale_cost between 80 and 80+20)) B4,
+     (select avg(ss_list_price) B5_LP
+            ,count(ss_list_price) B5_CNT
+            ,count(distinct ss_list_price) B5_CNTD
+      from store_sales
+      where ss_quantity between 21 and 25
+        and (ss_list_price between 135 and 135+10
+          or ss_coupon_amt between 14180 and 14180+1000
+          or ss_wholesale_cost between 38 and 38+20)) B5,
+     (select avg(ss_list_price) B6_LP
+            ,count(ss_list_price) B6_CNT
+            ,count(distinct ss_list_price) B6_CNTD
+      from store_sales
+      where ss_quantity between 26 and 30
+        and (ss_list_price between 28 and 28+10
+          or ss_coupon_amt between 2513 and 2513+1000
+          or ss_wholesale_cost between 42 and 42+20)) B6
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q29.sql b/benchmarks/queries-tpcds/q29.sql
new file mode 100644
index 0000000000..d8e7e809a1
--- /dev/null
+++ b/benchmarks/queries-tpcds/q29.sql
@@ -0,0 +1,48 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select   
+     i_item_id
+    ,i_item_desc
+    ,s_store_id
+    ,s_store_name
+    ,sum(ss_quantity)        as store_sales_quantity
+    ,sum(sr_return_quantity) as store_returns_quantity
+    ,sum(cs_quantity)        as catalog_sales_quantity
+ from
+    store_sales
+   ,store_returns
+   ,catalog_sales
+   ,date_dim             d1
+   ,date_dim             d2
+   ,date_dim             d3
+   ,store
+   ,item
+ where
+     d1.d_moy               = 4 
+ and d1.d_year              = 1999
+ and d1.d_date_sk           = ss_sold_date_sk
+ and i_item_sk              = ss_item_sk
+ and s_store_sk             = ss_store_sk
+ and ss_customer_sk         = sr_customer_sk
+ and ss_item_sk             = sr_item_sk
+ and ss_ticket_number       = sr_ticket_number
+ and sr_returned_date_sk    = d2.d_date_sk
+ and d2.d_moy               between 4 and  4 + 3 
+ and d2.d_year              = 1999
+ and sr_customer_sk         = cs_bill_customer_sk
+ and sr_item_sk             = cs_item_sk
+ and cs_sold_date_sk        = d3.d_date_sk     
+ and d3.d_year              in (1999,1999+1,1999+2)
+ group by
+    i_item_id
+   ,i_item_desc
+   ,s_store_id
+   ,s_store_name
+ order by
+    i_item_id 
+   ,i_item_desc
+   ,s_store_id
+   ,s_store_name
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q3.sql b/benchmarks/queries-tpcds/q3.sql
new file mode 100644
index 0000000000..af376402c6
--- /dev/null
+++ b/benchmarks/queries-tpcds/q3.sql
@@ -0,0 +1,22 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  dt.d_year 
+       ,item.i_brand_id brand_id 
+       ,item.i_brand brand
+       ,sum(ss_ext_sales_price) sum_agg
+ from  date_dim dt 
+      ,store_sales
+      ,item
+ where dt.d_date_sk = store_sales.ss_sold_date_sk
+   and store_sales.ss_item_sk = item.i_item_sk
+   and item.i_manufact_id = 436
+   and dt.d_moy=12
+ group by dt.d_year
+      ,item.i_brand
+      ,item.i_brand_id
+ order by dt.d_year
+         ,sum_agg desc
+         ,brand_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q30.sql b/benchmarks/queries-tpcds/q30.sql
new file mode 100644
index 0000000000..d1682e1266
--- /dev/null
+++ b/benchmarks/queries-tpcds/q30.sql
@@ -0,0 +1,31 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with customer_total_return as
+ (select wr_returning_customer_sk as ctr_customer_sk
+        ,ca_state as ctr_state, 
+ 	sum(wr_return_amt) as ctr_total_return
+ from web_returns
+     ,date_dim
+     ,customer_address
+ where wr_returned_date_sk = d_date_sk 
+   and d_year =2002
+   and wr_returning_addr_sk = ca_address_sk 
+ group by wr_returning_customer_sk
+         ,ca_state)
+  select  c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag
+       ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address
+       ,c_last_review_date_sk,ctr_total_return
+ from customer_total_return ctr1
+     ,customer_address
+     ,customer
+ where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2
+ 			  from customer_total_return ctr2 
+                  	  where ctr1.ctr_state = ctr2.ctr_state)
+       and ca_address_sk = c_current_addr_sk
+       and ca_state = 'IL'
+       and ctr1.ctr_customer_sk = c_customer_sk
+ order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag
+                  ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address
+                  ,c_last_review_date_sk,ctr_total_return
+limit 100;
+
diff --git a/benchmarks/queries-tpcds/q31.sql b/benchmarks/queries-tpcds/q31.sql
new file mode 100644
index 0000000000..f72e384d8d
--- /dev/null
+++ b/benchmarks/queries-tpcds/q31.sql
@@ -0,0 +1,53 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ss as
+ (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales
+ from store_sales,date_dim,customer_address
+ where ss_sold_date_sk = d_date_sk
+  and ss_addr_sk=ca_address_sk
+ group by ca_county,d_qoy, d_year),
+ ws as
+ (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales
+ from web_sales,date_dim,customer_address
+ where ws_sold_date_sk = d_date_sk
+  and ws_bill_addr_sk=ca_address_sk
+ group by ca_county,d_qoy, d_year)
+ select 
+        ss1.ca_county
+       ,ss1.d_year
+       ,ws2.web_sales/ws1.web_sales web_q1_q2_increase
+       ,ss2.store_sales/ss1.store_sales store_q1_q2_increase
+       ,ws3.web_sales/ws2.web_sales web_q2_q3_increase
+       ,ss3.store_sales/ss2.store_sales store_q2_q3_increase
+ from
+        ss ss1
+       ,ss ss2
+       ,ss ss3
+       ,ws ws1
+       ,ws ws2
+       ,ws ws3
+ where
+    ss1.d_qoy = 1
+    and ss1.d_year = 2000
+    and ss1.ca_county = ss2.ca_county
+    and ss2.d_qoy = 2
+    and ss2.d_year = 2000
+ and ss2.ca_county = ss3.ca_county
+    and ss3.d_qoy = 3
+    and ss3.d_year = 2000
+    and ss1.ca_county = ws1.ca_county
+    and ws1.d_qoy = 1
+    and ws1.d_year = 2000
+    and ws1.ca_county = ws2.ca_county
+    and ws2.d_qoy = 2
+    and ws2.d_year = 2000
+    and ws1.ca_county = ws3.ca_county
+    and ws3.d_qoy = 3
+    and ws3.d_year =2000
+    and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end 
+       > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end
+    and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end
+       > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end
+ order by ss1.d_year;
+
+
diff --git a/benchmarks/queries-tpcds/q32.sql b/benchmarks/queries-tpcds/q32.sql
new file mode 100644
index 0000000000..3e9690787c
--- /dev/null
+++ b/benchmarks/queries-tpcds/q32.sql
@@ -0,0 +1,29 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  sum(cs_ext_discount_amt)  as 'excess discount amount'
+from 
+   catalog_sales 
+   ,item 
+   ,date_dim
+where
+i_manufact_id = 269
+and i_item_sk = cs_item_sk 
+and d_date between '1998-03-18' and 
+        (cast('1998-03-18' as date) + INTERVAL '90 days')
+and d_date_sk = cs_sold_date_sk 
+and cs_ext_discount_amt  
+     > ( 
+         select 
+            1.3 * avg(cs_ext_discount_amt) 
+         from 
+            catalog_sales 
+           ,date_dim
+         where 
+              cs_item_sk = i_item_sk 
+          and d_date between '1998-03-18' and
+                             (cast('1998-03-18' as date) + INTERVAL '90 days')
+          and d_date_sk = cs_sold_date_sk 
+      ) 
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q33.sql b/benchmarks/queries-tpcds/q33.sql
new file mode 100644
index 0000000000..0b1a5d7714
--- /dev/null
+++ b/benchmarks/queries-tpcds/q33.sql
@@ -0,0 +1,76 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ss as (
+ select
+          i_manufact_id,sum(ss_ext_sales_price) total_sales
+ from
+ 	store_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_manufact_id in (select
+  i_manufact_id
+from
+ item
+where i_category in ('Books'))
+ and     ss_item_sk              = i_item_sk
+ and     ss_sold_date_sk         = d_date_sk
+ and     d_year                  = 1999
+ and     d_moy                   = 3
+ and     ss_addr_sk              = ca_address_sk
+ and     ca_gmt_offset           = -6 
+ group by i_manufact_id),
+ cs as (
+ select
+          i_manufact_id,sum(cs_ext_sales_price) total_sales
+ from
+ 	catalog_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_manufact_id               in (select
+  i_manufact_id
+from
+ item
+where i_category in ('Books'))
+ and     cs_item_sk              = i_item_sk
+ and     cs_sold_date_sk         = d_date_sk
+ and     d_year                  = 1999
+ and     d_moy                   = 3
+ and     cs_bill_addr_sk         = ca_address_sk
+ and     ca_gmt_offset           = -6 
+ group by i_manufact_id),
+ ws as (
+ select
+          i_manufact_id,sum(ws_ext_sales_price) total_sales
+ from
+ 	web_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_manufact_id               in (select
+  i_manufact_id
+from
+ item
+where i_category in ('Books'))
+ and     ws_item_sk              = i_item_sk
+ and     ws_sold_date_sk         = d_date_sk
+ and     d_year                  = 1999
+ and     d_moy                   = 3
+ and     ws_bill_addr_sk         = ca_address_sk
+ and     ca_gmt_offset           = -6
+ group by i_manufact_id)
+  select  i_manufact_id ,sum(total_sales) total_sales
+ from  (select * from ss 
+        union all
+        select * from cs 
+        union all
+        select * from ws) tmp1
+ group by i_manufact_id
+ order by total_sales
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q34.sql b/benchmarks/queries-tpcds/q34.sql
new file mode 100644
index 0000000000..af31c6db1d
--- /dev/null
+++ b/benchmarks/queries-tpcds/q34.sql
@@ -0,0 +1,32 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select c_last_name
+       ,c_first_name
+       ,c_salutation
+       ,c_preferred_cust_flag
+       ,ss_ticket_number
+       ,cnt from
+   (select ss_ticket_number
+          ,ss_customer_sk
+          ,count(*) cnt
+    from store_sales,date_dim,store,household_demographics
+    where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+    and store_sales.ss_store_sk = store.s_store_sk  
+    and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk
+    and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28)
+    and (household_demographics.hd_buy_potential = '>10000' or
+         household_demographics.hd_buy_potential = '5001-10000')
+    and household_demographics.hd_vehicle_count > 0
+    and (case when household_demographics.hd_vehicle_count > 0 
+	then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count 
+	else null 
+	end)  > 1.2
+    and date_dim.d_year in (1999,1999+1,1999+2)
+    and store.s_county in ('Daviess County','Franklin Parish','Barrow County','Luce County',
+                           'Fairfield County','Richland County','Ziebach County','Walker County')
+    group by ss_ticket_number,ss_customer_sk) dn,customer
+    where ss_customer_sk = c_customer_sk
+      and cnt between 15 and 20
+    order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number;
+
+
diff --git a/benchmarks/queries-tpcds/q35.sql b/benchmarks/queries-tpcds/q35.sql
new file mode 100644
index 0000000000..cafce6b595
--- /dev/null
+++ b/benchmarks/queries-tpcds/q35.sql
@@ -0,0 +1,59 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select   
+  ca_state,
+  cd_gender,
+  cd_marital_status,
+  cd_dep_count,
+  count(*) cnt1,
+  avg(cd_dep_count),
+  max(cd_dep_count),
+  sum(cd_dep_count),
+  cd_dep_employed_count,
+  count(*) cnt2,
+  avg(cd_dep_employed_count),
+  max(cd_dep_employed_count),
+  sum(cd_dep_employed_count),
+  cd_dep_college_count,
+  count(*) cnt3,
+  avg(cd_dep_college_count),
+  max(cd_dep_college_count),
+  sum(cd_dep_college_count)
+ from
+  customer c,customer_address ca,customer_demographics
+ where
+  c.c_current_addr_sk = ca.ca_address_sk and
+  cd_demo_sk = c.c_current_cdemo_sk and 
+  exists (select *
+          from store_sales,date_dim
+          where c.c_customer_sk = ss_customer_sk and
+                ss_sold_date_sk = d_date_sk and
+                d_year = 1999 and
+                d_qoy < 4) and
+   (exists (select *
+            from web_sales,date_dim
+            where c.c_customer_sk = ws_bill_customer_sk and
+                  ws_sold_date_sk = d_date_sk and
+                  d_year = 1999 and
+                  d_qoy < 4) or 
+    exists (select * 
+            from catalog_sales,date_dim
+            where c.c_customer_sk = cs_ship_customer_sk and
+                  cs_sold_date_sk = d_date_sk and
+                  d_year = 1999 and
+                  d_qoy < 4))
+ group by ca_state,
+          cd_gender,
+          cd_marital_status,
+          cd_dep_count,
+          cd_dep_employed_count,
+          cd_dep_college_count
+ order by ca_state,
+          cd_gender,
+          cd_marital_status,
+          cd_dep_count,
+          cd_dep_employed_count,
+          cd_dep_college_count
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q36.sql b/benchmarks/queries-tpcds/q36.sql
new file mode 100644
index 0000000000..0186229cec
--- /dev/null
+++ b/benchmarks/queries-tpcds/q36.sql
@@ -0,0 +1,31 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+    sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin
+   ,i_category
+   ,i_class
+   ,grouping(i_category)+grouping(i_class) as lochierarchy
+   ,rank() over (
+ 	partition by grouping(i_category)+grouping(i_class),
+ 	case when grouping(i_class) = 0 then i_category end 
+ 	order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent
+ from
+    store_sales
+   ,date_dim       d1
+   ,item
+   ,store
+ where
+    d1.d_year = 2000 
+ and d1.d_date_sk = ss_sold_date_sk
+ and i_item_sk  = ss_item_sk 
+ and s_store_sk  = ss_store_sk
+ and s_state in ('MO','LA','GA','MI',
+                 'SC','OH','SD','AL')
+ group by rollup(i_category,i_class)
+ order by
+   lochierarchy desc
+  ,case when lochierarchy = 0 then i_category end
+  ,rank_within_parent
+  limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q37.sql b/benchmarks/queries-tpcds/q37.sql
new file mode 100644
index 0000000000..ff478d0b6d
--- /dev/null
+++ b/benchmarks/queries-tpcds/q37.sql
@@ -0,0 +1,18 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id
+       ,i_item_desc
+       ,i_current_price
+ from item, inventory, date_dim, catalog_sales
+ where i_current_price between 22 and 22 + 30
+ and inv_item_sk = i_item_sk
+ and d_date_sk=inv_date_sk
+ and d_date between cast('2001-06-02' as date) and (cast('2001-06-02' as date) + INTERVAL '60 days')
+ and i_manufact_id in (678,964,918,849)
+ and inv_quantity_on_hand between 100 and 500
+ and cs_item_sk = i_item_sk
+ group by i_item_id,i_item_desc,i_current_price
+ order by i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q38.sql b/benchmarks/queries-tpcds/q38.sql
new file mode 100644
index 0000000000..ab0322220f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q38.sql
@@ -0,0 +1,24 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  count(*) from (
+    select distinct c_last_name, c_first_name, d_date
+    from store_sales, date_dim, customer
+          where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+      and store_sales.ss_customer_sk = customer.c_customer_sk
+      and d_month_seq between 1212 and 1212 + 11
+  intersect
+    select distinct c_last_name, c_first_name, d_date
+    from catalog_sales, date_dim, customer
+          where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk
+      and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk
+      and d_month_seq between 1212 and 1212 + 11
+  intersect
+    select distinct c_last_name, c_first_name, d_date
+    from web_sales, date_dim, customer
+          where web_sales.ws_sold_date_sk = date_dim.d_date_sk
+      and web_sales.ws_bill_customer_sk = customer.c_customer_sk
+      and d_month_seq between 1212 and 1212 + 11
+) hot_cust
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q39.sql b/benchmarks/queries-tpcds/q39.sql
new file mode 100644
index 0000000000..e49f38e039
--- /dev/null
+++ b/benchmarks/queries-tpcds/q39.sql
@@ -0,0 +1,55 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with inv as
+(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+       ,stdev,mean, case mean when 0 then null else stdev/mean end cov
+ from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+            ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean
+      from inventory
+          ,item
+          ,warehouse
+          ,date_dim
+      where inv_item_sk = i_item_sk
+        and inv_warehouse_sk = w_warehouse_sk
+        and inv_date_sk = d_date_sk
+        and d_year =1998
+      group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo
+ where case mean when 0 then 0 else stdev/mean end > 1)
+select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov
+        ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov
+from inv inv1,inv inv2
+where inv1.i_item_sk = inv2.i_item_sk
+  and inv1.w_warehouse_sk =  inv2.w_warehouse_sk
+  and inv1.d_moy=4
+  and inv2.d_moy=4+1
+order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov
+        ,inv2.d_moy,inv2.mean, inv2.cov
+;
+with inv as
+(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+       ,stdev,mean, case mean when 0 then null else stdev/mean end cov
+ from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+            ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean
+      from inventory
+          ,item
+          ,warehouse
+          ,date_dim
+      where inv_item_sk = i_item_sk
+        and inv_warehouse_sk = w_warehouse_sk
+        and inv_date_sk = d_date_sk
+        and d_year =1998
+      group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo
+ where case mean when 0 then 0 else stdev/mean end > 1)
+select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov
+        ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov
+from inv inv1,inv inv2
+where inv1.i_item_sk = inv2.i_item_sk
+  and inv1.w_warehouse_sk =  inv2.w_warehouse_sk
+  and inv1.d_moy=4
+  and inv2.d_moy=4+1
+  and inv1.cov > 1.5
+order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov
+        ,inv2.d_moy,inv2.mean, inv2.cov
+;
+
+
diff --git a/benchmarks/queries-tpcds/q4.sql b/benchmarks/queries-tpcds/q4.sql
new file mode 100644
index 0000000000..30aa0a72d1
--- /dev/null
+++ b/benchmarks/queries-tpcds/q4.sql
@@ -0,0 +1,117 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with year_total as (
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,c_preferred_cust_flag customer_preferred_cust_flag
+       ,c_birth_country customer_birth_country
+       ,c_login customer_login
+       ,c_email_address customer_email_address
+       ,d_year dyear
+       ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total
+       ,'s' sale_type
+ from customer
+     ,store_sales
+     ,date_dim
+ where c_customer_sk = ss_customer_sk
+   and ss_sold_date_sk = d_date_sk
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,c_preferred_cust_flag
+         ,c_birth_country
+         ,c_login
+         ,c_email_address
+         ,d_year
+ union all
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,c_preferred_cust_flag customer_preferred_cust_flag
+       ,c_birth_country customer_birth_country
+       ,c_login customer_login
+       ,c_email_address customer_email_address
+       ,d_year dyear
+       ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total
+       ,'c' sale_type
+ from customer
+     ,catalog_sales
+     ,date_dim
+ where c_customer_sk = cs_bill_customer_sk
+   and cs_sold_date_sk = d_date_sk
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,c_preferred_cust_flag
+         ,c_birth_country
+         ,c_login
+         ,c_email_address
+         ,d_year
+union all
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,c_preferred_cust_flag customer_preferred_cust_flag
+       ,c_birth_country customer_birth_country
+       ,c_login customer_login
+       ,c_email_address customer_email_address
+       ,d_year dyear
+       ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total
+       ,'w' sale_type
+ from customer
+     ,web_sales
+     ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+   and ws_sold_date_sk = d_date_sk
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,c_preferred_cust_flag
+         ,c_birth_country
+         ,c_login
+         ,c_email_address
+         ,d_year
+         )
+  select  
+                  t_s_secyear.customer_id
+                 ,t_s_secyear.customer_first_name
+                 ,t_s_secyear.customer_last_name
+                 ,t_s_secyear.customer_email_address
+ from year_total t_s_firstyear
+     ,year_total t_s_secyear
+     ,year_total t_c_firstyear
+     ,year_total t_c_secyear
+     ,year_total t_w_firstyear
+     ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+   and t_s_firstyear.customer_id = t_c_secyear.customer_id
+   and t_s_firstyear.customer_id = t_c_firstyear.customer_id
+   and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+   and t_s_firstyear.customer_id = t_w_secyear.customer_id
+   and t_s_firstyear.sale_type = 's'
+   and t_c_firstyear.sale_type = 'c'
+   and t_w_firstyear.sale_type = 'w'
+   and t_s_secyear.sale_type = 's'
+   and t_c_secyear.sale_type = 'c'
+   and t_w_secyear.sale_type = 'w'
+   and t_s_firstyear.dyear =  2001
+   and t_s_secyear.dyear = 2001+1
+   and t_c_firstyear.dyear =  2001
+   and t_c_secyear.dyear =  2001+1
+   and t_w_firstyear.dyear = 2001
+   and t_w_secyear.dyear = 2001+1
+   and t_s_firstyear.year_total > 0
+   and t_c_firstyear.year_total > 0
+   and t_w_firstyear.year_total > 0
+   and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+           > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+   and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+           > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ order by t_s_secyear.customer_id
+         ,t_s_secyear.customer_first_name
+         ,t_s_secyear.customer_last_name
+         ,t_s_secyear.customer_email_address
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q40.sql b/benchmarks/queries-tpcds/q40.sql
new file mode 100644
index 0000000000..883148a0af
--- /dev/null
+++ b/benchmarks/queries-tpcds/q40.sql
@@ -0,0 +1,29 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   w_state
+  ,i_item_id
+  ,sum(case when (cast(d_date as date) < cast ('1998-04-08' as date)) 
+ 		then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before
+  ,sum(case when (cast(d_date as date) >= cast ('1998-04-08' as date)) 
+ 		then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after
+ from
+   catalog_sales left outer join catalog_returns on
+       (cs_order_number = cr_order_number 
+        and cs_item_sk = cr_item_sk)
+  ,warehouse 
+  ,item
+  ,date_dim
+ where
+     i_current_price between 0.99 and 1.49
+ and i_item_sk          = cs_item_sk
+ and cs_warehouse_sk    = w_warehouse_sk 
+ and cs_sold_date_sk    = d_date_sk
+ and d_date between (cast ('1998-04-08' as date) - INTERVAL '30 days')
+                and (cast ('1998-04-08' as date) + INTERVAL '30 days')
+ group by
+    w_state,i_item_id
+ order by w_state,i_item_id
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q41.sql b/benchmarks/queries-tpcds/q41.sql
new file mode 100644
index 0000000000..036b97f853
--- /dev/null
+++ b/benchmarks/queries-tpcds/q41.sql
@@ -0,0 +1,53 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  distinct(i_product_name)
+ from item i1
+ where i_manufact_id between 742 and 742+40 
+   and (select count(*) as item_cnt
+        from item
+        where (i_manufact = i1.i_manufact and
+        ((i_category = 'Women' and 
+        (i_color = 'orchid' or i_color = 'papaya') and 
+        (i_units = 'Pound' or i_units = 'Lb') and
+        (i_size = 'petite' or i_size = 'medium')
+        ) or
+        (i_category = 'Women' and
+        (i_color = 'burlywood' or i_color = 'navy') and
+        (i_units = 'Bundle' or i_units = 'Each') and
+        (i_size = 'N/A' or i_size = 'extra large')
+        ) or
+        (i_category = 'Men' and
+        (i_color = 'bisque' or i_color = 'azure') and
+        (i_units = 'N/A' or i_units = 'Tsp') and
+        (i_size = 'small' or i_size = 'large')
+        ) or
+        (i_category = 'Men' and
+        (i_color = 'chocolate' or i_color = 'cornflower') and
+        (i_units = 'Bunch' or i_units = 'Gross') and
+        (i_size = 'petite' or i_size = 'medium')
+        ))) or
+       (i_manufact = i1.i_manufact and
+        ((i_category = 'Women' and 
+        (i_color = 'salmon' or i_color = 'midnight') and 
+        (i_units = 'Oz' or i_units = 'Box') and
+        (i_size = 'petite' or i_size = 'medium')
+        ) or
+        (i_category = 'Women' and
+        (i_color = 'snow' or i_color = 'steel') and
+        (i_units = 'Carton' or i_units = 'Tbl') and
+        (i_size = 'N/A' or i_size = 'extra large')
+        ) or
+        (i_category = 'Men' and
+        (i_color = 'purple' or i_color = 'gainsboro') and
+        (i_units = 'Dram' or i_units = 'Unknown') and
+        (i_size = 'small' or i_size = 'large')
+        ) or
+        (i_category = 'Men' and
+        (i_color = 'metallic' or i_color = 'forest') and
+        (i_units = 'Gram' or i_units = 'Ounce') and
+        (i_size = 'petite' or i_size = 'medium')
+        )))) > 0
+ order by i_product_name
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q42.sql b/benchmarks/queries-tpcds/q42.sql
new file mode 100644
index 0000000000..cb3b789a81
--- /dev/null
+++ b/benchmarks/queries-tpcds/q42.sql
@@ -0,0 +1,23 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  dt.d_year
+ 	,item.i_category_id
+ 	,item.i_category
+ 	,sum(ss_ext_sales_price)
+ from 	date_dim dt
+ 	,store_sales
+ 	,item
+ where dt.d_date_sk = store_sales.ss_sold_date_sk
+ 	and store_sales.ss_item_sk = item.i_item_sk
+ 	and item.i_manager_id = 1  	
+ 	and dt.d_moy=12
+ 	and dt.d_year=1998
+ group by 	dt.d_year
+ 		,item.i_category_id
+ 		,item.i_category
+ order by       sum(ss_ext_sales_price) desc,dt.d_year
+ 		,item.i_category_id
+ 		,item.i_category
+limit 100 ;
+
+
diff --git a/benchmarks/queries-tpcds/q43.sql b/benchmarks/queries-tpcds/q43.sql
new file mode 100644
index 0000000000..ca9db995b0
--- /dev/null
+++ b/benchmarks/queries-tpcds/q43.sql
@@ -0,0 +1,20 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  s_store_name, s_store_id,
+        sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales,
+        sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales,
+        sum(case when (d_day_name='Tuesday') then ss_sales_price else  null end) tue_sales,
+        sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales,
+        sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales,
+        sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales,
+        sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales
+ from date_dim, store_sales, store
+ where d_date_sk = ss_sold_date_sk and
+       s_store_sk = ss_store_sk and
+       s_gmt_offset = -6 and
+       d_year = 1998 
+ group by s_store_name, s_store_id
+ order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q44.sql b/benchmarks/queries-tpcds/q44.sql
new file mode 100644
index 0000000000..923b531b70
--- /dev/null
+++ b/benchmarks/queries-tpcds/q44.sql
@@ -0,0 +1,36 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing
+from(select *
+     from (select item_sk,rank() over (order by rank_col asc) rnk
+           from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col 
+                 from store_sales ss1
+                 where ss_store_sk = 50
+                 group by ss_item_sk
+                 having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col
+                                                  from store_sales
+                                                  where ss_store_sk = 50
+                                                    and ss_hdemo_sk is null
+                                                  group by ss_store_sk))V1)V11
+     where rnk  < 11) asceding,
+    (select *
+     from (select item_sk,rank() over (order by rank_col desc) rnk
+           from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col
+                 from store_sales ss1
+                 where ss_store_sk = 50
+                 group by ss_item_sk
+                 having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col
+                                                  from store_sales
+                                                  where ss_store_sk = 50
+                                                    and ss_hdemo_sk is null
+                                                  group by ss_store_sk))V2)V21
+     where rnk  < 11) descending,
+item i1,
+item i2
+where asceding.rnk = descending.rnk 
+  and i1.i_item_sk=asceding.item_sk
+  and i2.i_item_sk=descending.item_sk
+order by asceding.rnk
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q45.sql b/benchmarks/queries-tpcds/q45.sql
new file mode 100644
index 0000000000..ff7420f00f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q45.sql
@@ -0,0 +1,21 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  ca_zip, ca_county, sum(ws_sales_price)
+ from web_sales, customer, customer_address, date_dim, item
+ where ws_bill_customer_sk = c_customer_sk
+ 	and c_current_addr_sk = ca_address_sk 
+ 	and ws_item_sk = i_item_sk 
+ 	and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792')
+ 	      or 
+ 	      i_item_id in (select i_item_id
+                             from item
+                             where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29)
+                             )
+ 	    )
+ 	and ws_sold_date_sk = d_date_sk
+ 	and d_qoy = 2 and d_year = 2000
+ group by ca_zip, ca_county
+ order by ca_zip, ca_county
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q46.sql b/benchmarks/queries-tpcds/q46.sql
new file mode 100644
index 0000000000..2f6218a09b
--- /dev/null
+++ b/benchmarks/queries-tpcds/q46.sql
@@ -0,0 +1,36 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  c_last_name
+       ,c_first_name
+       ,ca_city
+       ,bought_city
+       ,ss_ticket_number
+       ,amt,profit 
+ from
+   (select ss_ticket_number
+          ,ss_customer_sk
+          ,ca_city bought_city
+          ,sum(ss_coupon_amt) amt
+          ,sum(ss_net_profit) profit
+    from store_sales,date_dim,store,household_demographics,customer_address 
+    where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+    and store_sales.ss_store_sk = store.s_store_sk  
+    and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk
+    and store_sales.ss_addr_sk = customer_address.ca_address_sk
+    and (household_demographics.hd_dep_count = 6 or
+         household_demographics.hd_vehicle_count= 3)
+    and date_dim.d_dow in (6,0)
+    and date_dim.d_year in (1999,1999+1,1999+2) 
+    and store.s_city in ('Oakland','Riverside','Union','Salem','Greenwood') 
+    group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr
+    where ss_customer_sk = c_customer_sk
+      and customer.c_current_addr_sk = current_addr.ca_address_sk
+      and current_addr.ca_city <> bought_city
+  order by c_last_name
+          ,c_first_name
+          ,ca_city
+          ,bought_city
+          ,ss_ticket_number
+  limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q47.sql b/benchmarks/queries-tpcds/q47.sql
new file mode 100644
index 0000000000..18e61ceb17
--- /dev/null
+++ b/benchmarks/queries-tpcds/q47.sql
@@ -0,0 +1,52 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with v1 as(
+ select i_category, i_brand,
+        s_store_name, s_company_name,
+        d_year, d_moy,
+        sum(ss_sales_price) sum_sales,
+        avg(sum(ss_sales_price)) over
+          (partition by i_category, i_brand,
+                     s_store_name, s_company_name, d_year)
+          avg_monthly_sales,
+        rank() over
+          (partition by i_category, i_brand,
+                     s_store_name, s_company_name
+           order by d_year, d_moy) rn
+ from item, store_sales, date_dim, store
+ where ss_item_sk = i_item_sk and
+       ss_sold_date_sk = d_date_sk and
+       ss_store_sk = s_store_sk and
+       (
+         d_year = 2000 or
+         ( d_year = 2000-1 and d_moy =12) or
+         ( d_year = 2000+1 and d_moy =1)
+       )
+ group by i_category, i_brand,
+          s_store_name, s_company_name,
+          d_year, d_moy),
+ v2 as(
+ select v1.i_category, v1.i_brand
+        ,v1.d_year, v1.d_moy
+        ,v1.avg_monthly_sales
+        ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+       v1.i_category = v1_lead.i_category and
+       v1.i_brand = v1_lag.i_brand and
+       v1.i_brand = v1_lead.i_brand and
+       v1.s_store_name = v1_lag.s_store_name and
+       v1.s_store_name = v1_lead.s_store_name and
+       v1.s_company_name = v1_lag.s_company_name and
+       v1.s_company_name = v1_lead.s_company_name and
+       v1.rn = v1_lag.rn + 1 and
+       v1.rn = v1_lead.rn - 1)
+  select  *
+ from v2
+ where  d_year = 2000 and    
+        avg_monthly_sales > 0 and
+        case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, nsum
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q48.sql b/benchmarks/queries-tpcds/q48.sql
new file mode 100644
index 0000000000..057a6fb9a2
--- /dev/null
+++ b/benchmarks/queries-tpcds/q48.sql
@@ -0,0 +1,68 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select sum (ss_quantity)
+ from store_sales, store, customer_demographics, customer_address, date_dim
+ where s_store_sk = ss_store_sk
+ and  ss_sold_date_sk = d_date_sk and d_year = 1998
+ and  
+ (
+  (
+   cd_demo_sk = ss_cdemo_sk
+   and 
+   cd_marital_status = 'M'
+   and 
+   cd_education_status = '4 yr Degree'
+   and 
+   ss_sales_price between 100.00 and 150.00  
+   )
+ or
+  (
+  cd_demo_sk = ss_cdemo_sk
+   and 
+   cd_marital_status = 'D'
+   and 
+   cd_education_status = 'Primary'
+   and 
+   ss_sales_price between 50.00 and 100.00   
+  )
+ or 
+ (
+  cd_demo_sk = ss_cdemo_sk
+  and 
+   cd_marital_status = 'U'
+   and 
+   cd_education_status = 'Advanced Degree'
+   and 
+   ss_sales_price between 150.00 and 200.00  
+ )
+ )
+ and
+ (
+  (
+  ss_addr_sk = ca_address_sk
+  and
+  ca_country = 'United States'
+  and
+  ca_state in ('KY', 'GA', 'NM')
+  and ss_net_profit between 0 and 2000  
+  )
+ or
+  (ss_addr_sk = ca_address_sk
+  and
+  ca_country = 'United States'
+  and
+  ca_state in ('MT', 'OR', 'IN')
+  and ss_net_profit between 150 and 3000 
+  )
+ or
+  (ss_addr_sk = ca_address_sk
+  and
+  ca_country = 'United States'
+  and
+  ca_state in ('WI', 'MO', 'WV')
+  and ss_net_profit between 50 and 25000 
+  )
+ )
+;
+
+
diff --git a/benchmarks/queries-tpcds/q49.sql b/benchmarks/queries-tpcds/q49.sql
new file mode 100644
index 0000000000..219877719f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q49.sql
@@ -0,0 +1,130 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  channel, item, return_ratio, return_rank, currency_rank from
+ (select
+ 'web' as channel
+ ,web.item
+ ,web.return_ratio
+ ,web.return_rank
+ ,web.currency_rank
+ from (
+ 	select 
+ 	 item
+ 	,return_ratio
+ 	,currency_ratio
+ 	,rank() over (order by return_ratio) as return_rank
+ 	,rank() over (order by currency_ratio) as currency_rank
+ 	from
+ 	(	select ws.ws_item_sk as item
+ 		,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/
+ 		cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio
+ 		,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/
+ 		cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio
+ 		from 
+ 		 web_sales ws left outer join web_returns wr 
+ 			on (ws.ws_order_number = wr.wr_order_number and 
+ 			ws.ws_item_sk = wr.wr_item_sk)
+                 ,date_dim
+ 		where 
+ 			wr.wr_return_amt > 10000 
+ 			and ws.ws_net_profit > 1
+                         and ws.ws_net_paid > 0
+                         and ws.ws_quantity > 0
+                         and ws_sold_date_sk = d_date_sk
+                         and d_year = 2000
+                         and d_moy = 12
+ 		group by ws.ws_item_sk
+ 	) in_web
+ ) web
+ where 
+ (
+ web.return_rank <= 10
+ or
+ web.currency_rank <= 10
+ )
+ union
+ select 
+ 'catalog' as channel
+ ,catalog.item
+ ,catalog.return_ratio
+ ,catalog.return_rank
+ ,catalog.currency_rank
+ from (
+ 	select 
+ 	 item
+ 	,return_ratio
+ 	,currency_ratio
+ 	,rank() over (order by return_ratio) as return_rank
+ 	,rank() over (order by currency_ratio) as currency_rank
+ 	from
+ 	(	select 
+ 		cs.cs_item_sk as item
+ 		,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/
+ 		cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio
+ 		,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/
+ 		cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio
+ 		from 
+ 		catalog_sales cs left outer join catalog_returns cr
+ 			on (cs.cs_order_number = cr.cr_order_number and 
+ 			cs.cs_item_sk = cr.cr_item_sk)
+                ,date_dim
+ 		where 
+ 			cr.cr_return_amount > 10000 
+ 			and cs.cs_net_profit > 1
+                         and cs.cs_net_paid > 0
+                         and cs.cs_quantity > 0
+                         and cs_sold_date_sk = d_date_sk
+                         and d_year = 2000
+                         and d_moy = 12
+                 group by cs.cs_item_sk
+ 	) in_cat
+ ) catalog
+ where 
+ (
+ catalog.return_rank <= 10
+ or
+ catalog.currency_rank <=10
+ )
+ union
+ select 
+ 'store' as channel
+ ,store.item
+ ,store.return_ratio
+ ,store.return_rank
+ ,store.currency_rank
+ from (
+ 	select 
+ 	 item
+ 	,return_ratio
+ 	,currency_ratio
+ 	,rank() over (order by return_ratio) as return_rank
+ 	,rank() over (order by currency_ratio) as currency_rank
+ 	from
+ 	(	select sts.ss_item_sk as item
+ 		,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio
+ 		,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio
+ 		from 
+ 		store_sales sts left outer join store_returns sr
+ 			on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk)
+                ,date_dim
+ 		where 
+ 			sr.sr_return_amt > 10000 
+ 			and sts.ss_net_profit > 1
+                         and sts.ss_net_paid > 0
+                         and sts.ss_quantity > 0
+                         and ss_sold_date_sk = d_date_sk
+                         and d_year = 2000
+                         and d_moy = 12
+ 		group by sts.ss_item_sk
+ 	) in_store
+ ) store
+ where  (
+ store.return_rank <= 10
+ or 
+ store.currency_rank <= 10
+ )
+ )
+ order by 1,4,5,2
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q5.sql b/benchmarks/queries-tpcds/q5.sql
new file mode 100644
index 0000000000..60943526bb
--- /dev/null
+++ b/benchmarks/queries-tpcds/q5.sql
@@ -0,0 +1,129 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ssr as
+ (select s_store_id,
+        sum(sales_price) as sales,
+        sum(profit) as profit,
+        sum(return_amt) as returns,
+        sum(net_loss) as profit_loss
+ from
+  ( select  ss_store_sk as store_sk,
+            ss_sold_date_sk  as date_sk,
+            ss_ext_sales_price as sales_price,
+            ss_net_profit as profit,
+            cast(0 as decimal(7,2)) as return_amt,
+            cast(0 as decimal(7,2)) as net_loss
+    from store_sales
+    union all
+    select sr_store_sk as store_sk,
+           sr_returned_date_sk as date_sk,
+           cast(0 as decimal(7,2)) as sales_price,
+           cast(0 as decimal(7,2)) as profit,
+           sr_return_amt as return_amt,
+           sr_net_loss as net_loss
+    from store_returns
+   ) salesreturns,
+     date_dim,
+     store
+ where date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date) 
+                  and (cast('1998-08-04' as date) + INTERVAL '14 days')
+       and store_sk = s_store_sk
+ group by s_store_id)
+ ,
+ csr as
+ (select cp_catalog_page_id,
+        sum(sales_price) as sales,
+        sum(profit) as profit,
+        sum(return_amt) as returns,
+        sum(net_loss) as profit_loss
+ from
+  ( select  cs_catalog_page_sk as page_sk,
+            cs_sold_date_sk  as date_sk,
+            cs_ext_sales_price as sales_price,
+            cs_net_profit as profit,
+            cast(0 as decimal(7,2)) as return_amt,
+            cast(0 as decimal(7,2)) as net_loss
+    from catalog_sales
+    union all
+    select cr_catalog_page_sk as page_sk,
+           cr_returned_date_sk as date_sk,
+           cast(0 as decimal(7,2)) as sales_price,
+           cast(0 as decimal(7,2)) as profit,
+           cr_return_amount as return_amt,
+           cr_net_loss as net_loss
+    from catalog_returns
+   ) salesreturns,
+     date_dim,
+     catalog_page
+ where date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '14 days')
+       and page_sk = cp_catalog_page_sk
+ group by cp_catalog_page_id)
+ ,
+ wsr as
+ (select web_site_id,
+        sum(sales_price) as sales,
+        sum(profit) as profit,
+        sum(return_amt) as returns,
+        sum(net_loss) as profit_loss
+ from
+  ( select  ws_web_site_sk as wsr_web_site_sk,
+            ws_sold_date_sk  as date_sk,
+            ws_ext_sales_price as sales_price,
+            ws_net_profit as profit,
+            cast(0 as decimal(7,2)) as return_amt,
+            cast(0 as decimal(7,2)) as net_loss
+    from web_sales
+    union all
+    select ws_web_site_sk as wsr_web_site_sk,
+           wr_returned_date_sk as date_sk,
+           cast(0 as decimal(7,2)) as sales_price,
+           cast(0 as decimal(7,2)) as profit,
+           wr_return_amt as return_amt,
+           wr_net_loss as net_loss
+    from web_returns left outer join web_sales on
+         ( wr_item_sk = ws_item_sk
+           and wr_order_number = ws_order_number)
+   ) salesreturns,
+     date_dim,
+     web_site
+ where date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '14 days')
+       and wsr_web_site_sk = web_site_sk
+ group by web_site_id)
+  select  channel
+        , id
+        , sum(sales) as sales
+        , sum(returns) as returns
+        , sum(profit) as profit
+ from 
+ (select 'store channel' as channel
+        , 'store' || s_store_id as id
+        , sales
+        , returns
+        , (profit - profit_loss) as profit
+ from   ssr
+ union all
+ select 'catalog channel' as channel
+        , 'catalog_page' || cp_catalog_page_id as id
+        , sales
+        , returns
+        , (profit - profit_loss) as profit
+ from  csr
+ union all
+ select 'web channel' as channel
+        , 'web_site' || web_site_id as id
+        , sales
+        , returns
+        , (profit - profit_loss) as profit
+ from   wsr
+ ) x
+ group by rollup (channel, id)
+ order by channel
+         ,id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q50.sql b/benchmarks/queries-tpcds/q50.sql
new file mode 100644
index 0000000000..f11a6562fc
--- /dev/null
+++ b/benchmarks/queries-tpcds/q50.sql
@@ -0,0 +1,60 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   s_store_name
+  ,s_company_id
+  ,s_street_number
+  ,s_street_name
+  ,s_street_type
+  ,s_suite_number
+  ,s_city
+  ,s_county
+  ,s_state
+  ,s_zip
+  ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end)  as '30 days'
+  ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and 
+                 (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end )  as '31-60 days'
+  ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and 
+                 (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end)  as '61-90 days'
+  ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and
+                 (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end)  as '91-120 days'
+  ,sum(case when (sr_returned_date_sk - ss_sold_date_sk  > 120) then 1 else 0 end)  as '>120 days'
+from
+   store_sales
+  ,store_returns
+  ,store
+  ,date_dim d1
+  ,date_dim d2
+where
+    d2.d_year = 2000
+and d2.d_moy  = 9
+and ss_ticket_number = sr_ticket_number
+and ss_item_sk = sr_item_sk
+and ss_sold_date_sk   = d1.d_date_sk
+and sr_returned_date_sk   = d2.d_date_sk
+and ss_customer_sk = sr_customer_sk
+and ss_store_sk = s_store_sk
+group by
+   s_store_name
+  ,s_company_id
+  ,s_street_number
+  ,s_street_name
+  ,s_street_type
+  ,s_suite_number
+  ,s_city
+  ,s_county
+  ,s_state
+  ,s_zip
+order by s_store_name
+        ,s_company_id
+        ,s_street_number
+        ,s_street_name
+        ,s_street_type
+        ,s_suite_number
+        ,s_city
+        ,s_county
+        ,s_state
+        ,s_zip
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q51.sql b/benchmarks/queries-tpcds/q51.sql
new file mode 100644
index 0000000000..a12d53b5e2
--- /dev/null
+++ b/benchmarks/queries-tpcds/q51.sql
@@ -0,0 +1,46 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+WITH web_v1 as (
+select
+  ws_item_sk item_sk, d_date,
+  sum(sum(ws_sales_price))
+      over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales
+from web_sales
+    ,date_dim
+where ws_sold_date_sk=d_date_sk
+  and d_month_seq between 1212 and 1212+11
+  and ws_item_sk is not NULL
+group by ws_item_sk, d_date),
+store_v1 as (
+select
+  ss_item_sk item_sk, d_date,
+  sum(sum(ss_sales_price))
+      over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales
+from store_sales
+    ,date_dim
+where ss_sold_date_sk=d_date_sk
+  and d_month_seq between 1212 and 1212+11
+  and ss_item_sk is not NULL
+group by ss_item_sk, d_date)
+ select  *
+from (select item_sk
+     ,d_date
+     ,web_sales
+     ,store_sales
+     ,max(web_sales)
+         over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative
+     ,max(store_sales)
+         over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative
+     from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk
+                 ,case when web.d_date is not null then web.d_date else store.d_date end d_date
+                 ,web.cume_sales web_sales
+                 ,store.cume_sales store_sales
+           from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk
+                                                          and web.d_date = store.d_date)
+          )x )y
+where web_cumulative > store_cumulative
+order by item_sk
+        ,d_date
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q52.sql b/benchmarks/queries-tpcds/q52.sql
new file mode 100644
index 0000000000..2b042cfd3d
--- /dev/null
+++ b/benchmarks/queries-tpcds/q52.sql
@@ -0,0 +1,23 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  dt.d_year
+ 	,item.i_brand_id brand_id
+ 	,item.i_brand brand
+ 	,sum(ss_ext_sales_price) ext_price
+ from date_dim dt
+     ,store_sales
+     ,item
+ where dt.d_date_sk = store_sales.ss_sold_date_sk
+    and store_sales.ss_item_sk = item.i_item_sk
+    and item.i_manager_id = 1
+    and dt.d_moy=12
+    and dt.d_year=1998
+ group by dt.d_year
+ 	,item.i_brand
+ 	,item.i_brand_id
+ order by dt.d_year
+ 	,ext_price desc
+ 	,brand_id
+limit 100 ;
+
+
diff --git a/benchmarks/queries-tpcds/q53.sql b/benchmarks/queries-tpcds/q53.sql
new file mode 100644
index 0000000000..2fa19a1bb9
--- /dev/null
+++ b/benchmarks/queries-tpcds/q53.sql
@@ -0,0 +1,29 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  * from 
+(select i_manufact_id,
+sum(ss_sales_price) sum_sales,
+avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales
+from item, store_sales, date_dim, store
+where ss_item_sk = i_item_sk and
+ss_sold_date_sk = d_date_sk and
+ss_store_sk = s_store_sk and
+d_month_seq in (1212,1212+1,1212+2,1212+3,1212+4,1212+5,1212+6,1212+7,1212+8,1212+9,1212+10,1212+11) and
+((i_category in ('Books','Children','Electronics') and
+i_class in ('personal','portable','reference','self-help') and
+i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7',
+		'exportiunivamalg #9','scholaramalgamalg #9'))
+or(i_category in ('Women','Music','Men') and
+i_class in ('accessories','classical','fragrances','pants') and
+i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1',
+		'importoamalg #1')))
+group by i_manufact_id, d_qoy ) tmp1
+where case when avg_quarterly_sales > 0 
+	then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales 
+	else null end > 0.1
+order by avg_quarterly_sales,
+	 sum_sales,
+	 i_manufact_id
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q54.sql b/benchmarks/queries-tpcds/q54.sql
new file mode 100644
index 0000000000..54173c7f30
--- /dev/null
+++ b/benchmarks/queries-tpcds/q54.sql
@@ -0,0 +1,57 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with my_customers as (
+ select distinct c_customer_sk
+        , c_current_addr_sk
+ from   
+        ( select cs_sold_date_sk sold_date_sk,
+                 cs_bill_customer_sk customer_sk,
+                 cs_item_sk item_sk
+          from   catalog_sales
+          union all
+          select ws_sold_date_sk sold_date_sk,
+                 ws_bill_customer_sk customer_sk,
+                 ws_item_sk item_sk
+          from   web_sales
+         ) cs_or_ws_sales,
+         item,
+         date_dim,
+         customer
+ where   sold_date_sk = d_date_sk
+         and item_sk = i_item_sk
+         and i_category = 'Jewelry'
+         and i_class = 'consignment'
+         and c_customer_sk = cs_or_ws_sales.customer_sk
+         and d_moy = 3
+         and d_year = 1999
+ )
+ , my_revenue as (
+ select c_customer_sk,
+        sum(ss_ext_sales_price) as revenue
+ from   my_customers,
+        store_sales,
+        customer_address,
+        store,
+        date_dim
+ where  c_current_addr_sk = ca_address_sk
+        and ca_county = s_county
+        and ca_state = s_state
+        and ss_sold_date_sk = d_date_sk
+        and c_customer_sk = ss_customer_sk
+        and d_month_seq between (select distinct d_month_seq+1
+                                 from   date_dim where d_year = 1999 and d_moy = 3)
+                           and  (select distinct d_month_seq+3
+                                 from   date_dim where d_year = 1999 and d_moy = 3)
+ group by c_customer_sk
+ )
+ , segments as
+ (select cast((revenue/50) as int) as segment
+  from   my_revenue
+ )
+  select  segment, count(*) as num_customers, segment*50 as segment_base
+ from segments
+ group by segment
+ order by segment, num_customers
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q55.sql b/benchmarks/queries-tpcds/q55.sql
new file mode 100644
index 0000000000..006e2493a6
--- /dev/null
+++ b/benchmarks/queries-tpcds/q55.sql
@@ -0,0 +1,15 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_brand_id brand_id, i_brand brand,
+ 	sum(ss_ext_sales_price) ext_price
+ from date_dim, store_sales, item
+ where d_date_sk = ss_sold_date_sk
+ 	and ss_item_sk = i_item_sk
+ 	and i_manager_id=36
+ 	and d_moy=12
+ 	and d_year=2001
+ group by i_brand, i_brand_id
+ order by ext_price desc, i_brand_id
+limit 100 ;
+
+
diff --git a/benchmarks/queries-tpcds/q56.sql b/benchmarks/queries-tpcds/q56.sql
new file mode 100644
index 0000000000..d811f5fbb8
--- /dev/null
+++ b/benchmarks/queries-tpcds/q56.sql
@@ -0,0 +1,70 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ss as (
+ select i_item_id,sum(ss_ext_sales_price) total_sales
+ from
+ 	store_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where i_item_id in (select
+     i_item_id
+from item
+where i_color in ('orchid','chiffon','lace'))
+ and     ss_item_sk              = i_item_sk
+ and     ss_sold_date_sk         = d_date_sk
+ and     d_year                  = 2000
+ and     d_moy                   = 1
+ and     ss_addr_sk              = ca_address_sk
+ and     ca_gmt_offset           = -8 
+ group by i_item_id),
+ cs as (
+ select i_item_id,sum(cs_ext_sales_price) total_sales
+ from
+ 	catalog_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_item_id               in (select
+  i_item_id
+from item
+where i_color in ('orchid','chiffon','lace'))
+ and     cs_item_sk              = i_item_sk
+ and     cs_sold_date_sk         = d_date_sk
+ and     d_year                  = 2000
+ and     d_moy                   = 1
+ and     cs_bill_addr_sk         = ca_address_sk
+ and     ca_gmt_offset           = -8 
+ group by i_item_id),
+ ws as (
+ select i_item_id,sum(ws_ext_sales_price) total_sales
+ from
+ 	web_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_item_id               in (select
+  i_item_id
+from item
+where i_color in ('orchid','chiffon','lace'))
+ and     ws_item_sk              = i_item_sk
+ and     ws_sold_date_sk         = d_date_sk
+ and     d_year                  = 2000
+ and     d_moy                   = 1
+ and     ws_bill_addr_sk         = ca_address_sk
+ and     ca_gmt_offset           = -8
+ group by i_item_id)
+  select  i_item_id ,sum(total_sales) total_sales
+ from  (select * from ss 
+        union all
+        select * from cs 
+        union all
+        select * from ws) tmp1
+ group by i_item_id
+ order by total_sales,
+          i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q57.sql b/benchmarks/queries-tpcds/q57.sql
new file mode 100644
index 0000000000..68f8c9f61a
--- /dev/null
+++ b/benchmarks/queries-tpcds/q57.sql
@@ -0,0 +1,49 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with v1 as(
+ select i_category, i_brand,
+        cc_name,
+        d_year, d_moy,
+        sum(cs_sales_price) sum_sales,
+        avg(sum(cs_sales_price)) over
+          (partition by i_category, i_brand,
+                     cc_name, d_year)
+          avg_monthly_sales,
+        rank() over
+          (partition by i_category, i_brand,
+                     cc_name
+           order by d_year, d_moy) rn
+ from item, catalog_sales, date_dim, call_center
+ where cs_item_sk = i_item_sk and
+       cs_sold_date_sk = d_date_sk and
+       cc_call_center_sk= cs_call_center_sk and
+       (
+         d_year = 2000 or
+         ( d_year = 2000-1 and d_moy =12) or
+         ( d_year = 2000+1 and d_moy =1)
+       )
+ group by i_category, i_brand,
+          cc_name , d_year, d_moy),
+ v2 as(
+ select v1.cc_name
+        ,v1.d_year, v1.d_moy
+        ,v1.avg_monthly_sales
+        ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+       v1.i_category = v1_lead.i_category and
+       v1.i_brand = v1_lag.i_brand and
+       v1.i_brand = v1_lead.i_brand and
+       v1. cc_name = v1_lag. cc_name and
+       v1. cc_name = v1_lead. cc_name and
+       v1.rn = v1_lag.rn + 1 and
+       v1.rn = v1_lead.rn - 1)
+  select  *
+ from v2
+ where  d_year = 2000 and
+        avg_monthly_sales > 0 and
+        case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, nsum
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q58.sql b/benchmarks/queries-tpcds/q58.sql
new file mode 100644
index 0000000000..bafaabee07
--- /dev/null
+++ b/benchmarks/queries-tpcds/q58.sql
@@ -0,0 +1,66 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ss_items as
+ (select i_item_id item_id
+        ,sum(ss_ext_sales_price) ss_item_rev 
+ from store_sales
+     ,item
+     ,date_dim
+ where ss_item_sk = i_item_sk
+   and d_date in (select d_date
+                  from date_dim
+                  where d_week_seq = (select d_week_seq 
+                                      from date_dim
+                                      where d_date = '1998-02-19'))
+   and ss_sold_date_sk   = d_date_sk
+ group by i_item_id),
+ cs_items as
+ (select i_item_id item_id
+        ,sum(cs_ext_sales_price) cs_item_rev
+  from catalog_sales
+      ,item
+      ,date_dim
+ where cs_item_sk = i_item_sk
+  and  d_date in (select d_date
+                  from date_dim
+                  where d_week_seq = (select d_week_seq 
+                                      from date_dim
+                                      where d_date = '1998-02-19'))
+  and  cs_sold_date_sk = d_date_sk
+ group by i_item_id),
+ ws_items as
+ (select i_item_id item_id
+        ,sum(ws_ext_sales_price) ws_item_rev
+  from web_sales
+      ,item
+      ,date_dim
+ where ws_item_sk = i_item_sk
+  and  d_date in (select d_date
+                  from date_dim
+                  where d_week_seq =(select d_week_seq 
+                                     from date_dim
+                                     where d_date = '1998-02-19'))
+  and ws_sold_date_sk   = d_date_sk
+ group by i_item_id)
+  select  ss_items.item_id
+       ,ss_item_rev
+       ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev
+       ,cs_item_rev
+       ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev
+       ,ws_item_rev
+       ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev
+       ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average
+ from ss_items,cs_items,ws_items
+ where ss_items.item_id=cs_items.item_id
+   and ss_items.item_id=ws_items.item_id 
+   and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev
+   and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev
+   and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev
+   and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev
+   and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev
+   and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev
+ order by item_id
+         ,ss_item_rev
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q59.sql b/benchmarks/queries-tpcds/q59.sql
new file mode 100644
index 0000000000..58d66b8577
--- /dev/null
+++ b/benchmarks/queries-tpcds/q59.sql
@@ -0,0 +1,45 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with wss as 
+ (select d_week_seq,
+        ss_store_sk,
+        sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales,
+        sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales,
+        sum(case when (d_day_name='Tuesday') then ss_sales_price else  null end) tue_sales,
+        sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales,
+        sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales,
+        sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales,
+        sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales
+ from store_sales,date_dim
+ where d_date_sk = ss_sold_date_sk
+ group by d_week_seq,ss_store_sk
+ )
+  select  s_store_name1,s_store_id1,d_week_seq1
+       ,sun_sales1/sun_sales2,mon_sales1/mon_sales2
+       ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2
+       ,fri_sales1/fri_sales2,sat_sales1/sat_sales2
+ from
+ (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1
+        ,s_store_id s_store_id1,sun_sales sun_sales1
+        ,mon_sales mon_sales1,tue_sales tue_sales1
+        ,wed_sales wed_sales1,thu_sales thu_sales1
+        ,fri_sales fri_sales1,sat_sales sat_sales1
+  from wss,store,date_dim d
+  where d.d_week_seq = wss.d_week_seq and
+        ss_store_sk = s_store_sk and 
+        d_month_seq between 1185 and 1185 + 11) y,
+ (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2
+        ,s_store_id s_store_id2,sun_sales sun_sales2
+        ,mon_sales mon_sales2,tue_sales tue_sales2
+        ,wed_sales wed_sales2,thu_sales thu_sales2
+        ,fri_sales fri_sales2,sat_sales sat_sales2
+  from wss,store,date_dim d
+  where d.d_week_seq = wss.d_week_seq and
+        ss_store_sk = s_store_sk and 
+        d_month_seq between 1185+ 12 and 1185 + 23) x
+ where s_store_id1=s_store_id2
+   and d_week_seq1=d_week_seq2-52
+ order by s_store_name1,s_store_id1,d_week_seq1
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q6.sql b/benchmarks/queries-tpcds/q6.sql
new file mode 100644
index 0000000000..7ecee6ae77
--- /dev/null
+++ b/benchmarks/queries-tpcds/q6.sql
@@ -0,0 +1,27 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  a.ca_state state, count(*) cnt
+ from customer_address a
+     ,customer c
+     ,store_sales s
+     ,date_dim d
+     ,item i
+ where       a.ca_address_sk = c.c_current_addr_sk
+ 	and c.c_customer_sk = s.ss_customer_sk
+ 	and s.ss_sold_date_sk = d.d_date_sk
+ 	and s.ss_item_sk = i.i_item_sk
+ 	and d.d_month_seq = 
+ 	     (select distinct (d_month_seq)
+ 	      from date_dim
+               where d_year = 2000
+ 	        and d_moy = 2 )
+ 	and i.i_current_price > 1.2 * 
+             (select avg(j.i_current_price) 
+ 	     from item j 
+ 	     where j.i_category = i.i_category)
+ group by a.ca_state
+ having count(*) >= 10
+ order by cnt, a.ca_state 
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q60.sql b/benchmarks/queries-tpcds/q60.sql
new file mode 100644
index 0000000000..205ebc87ed
--- /dev/null
+++ b/benchmarks/queries-tpcds/q60.sql
@@ -0,0 +1,79 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ss as (
+ select
+          i_item_id,sum(ss_ext_sales_price) total_sales
+ from
+ 	store_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_item_id in (select
+  i_item_id
+from
+ item
+where i_category in ('Children'))
+ and     ss_item_sk              = i_item_sk
+ and     ss_sold_date_sk         = d_date_sk
+ and     d_year                  = 1999
+ and     d_moy                   = 9
+ and     ss_addr_sk              = ca_address_sk
+ and     ca_gmt_offset           = -6 
+ group by i_item_id),
+ cs as (
+ select
+          i_item_id,sum(cs_ext_sales_price) total_sales
+ from
+ 	catalog_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_item_id               in (select
+  i_item_id
+from
+ item
+where i_category in ('Children'))
+ and     cs_item_sk              = i_item_sk
+ and     cs_sold_date_sk         = d_date_sk
+ and     d_year                  = 1999
+ and     d_moy                   = 9
+ and     cs_bill_addr_sk         = ca_address_sk
+ and     ca_gmt_offset           = -6 
+ group by i_item_id),
+ ws as (
+ select
+          i_item_id,sum(ws_ext_sales_price) total_sales
+ from
+ 	web_sales,
+ 	date_dim,
+         customer_address,
+         item
+ where
+         i_item_id               in (select
+  i_item_id
+from
+ item
+where i_category in ('Children'))
+ and     ws_item_sk              = i_item_sk
+ and     ws_sold_date_sk         = d_date_sk
+ and     d_year                  = 1999
+ and     d_moy                   = 9
+ and     ws_bill_addr_sk         = ca_address_sk
+ and     ca_gmt_offset           = -6
+ group by i_item_id)
+  select   
+  i_item_id
+,sum(total_sales) total_sales
+ from  (select * from ss 
+        union all
+        select * from cs 
+        union all
+        select * from ws) tmp1
+ group by i_item_id
+ order by i_item_id
+      ,total_sales
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q61.sql b/benchmarks/queries-tpcds/q61.sql
new file mode 100644
index 0000000000..1194485fa4
--- /dev/null
+++ b/benchmarks/queries-tpcds/q61.sql
@@ -0,0 +1,45 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100
+from
+  (select sum(ss_ext_sales_price) promotions
+   from  store_sales
+        ,store
+        ,promotion
+        ,date_dim
+        ,customer
+        ,customer_address 
+        ,item
+   where ss_sold_date_sk = d_date_sk
+   and   ss_store_sk = s_store_sk
+   and   ss_promo_sk = p_promo_sk
+   and   ss_customer_sk= c_customer_sk
+   and   ca_address_sk = c_current_addr_sk
+   and   ss_item_sk = i_item_sk 
+   and   ca_gmt_offset = -7
+   and   i_category = 'Books'
+   and   (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y')
+   and   s_gmt_offset = -7
+   and   d_year = 1999
+   and   d_moy  = 11) promotional_sales,
+  (select sum(ss_ext_sales_price) total
+   from  store_sales
+        ,store
+        ,date_dim
+        ,customer
+        ,customer_address
+        ,item
+   where ss_sold_date_sk = d_date_sk
+   and   ss_store_sk = s_store_sk
+   and   ss_customer_sk= c_customer_sk
+   and   ca_address_sk = c_current_addr_sk
+   and   ss_item_sk = i_item_sk
+   and   ca_gmt_offset = -7
+   and   i_category = 'Books'
+   and   s_gmt_offset = -7
+   and   d_year = 1999
+   and   d_moy  = 11) all_sales
+order by promotions, total
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q62.sql b/benchmarks/queries-tpcds/q62.sql
new file mode 100644
index 0000000000..fe8371fc28
--- /dev/null
+++ b/benchmarks/queries-tpcds/q62.sql
@@ -0,0 +1,36 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,web_name
+  ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end)  as '30 days'
+  ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and 
+                 (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end )  as '31-60 days'
+  ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and 
+                 (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end)  as '61-90 days'
+  ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and
+                 (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end)  as '91-120 days'
+  ,sum(case when (ws_ship_date_sk - ws_sold_date_sk  > 120) then 1 else 0 end)  as '>120 days'
+from
+   web_sales
+  ,warehouse
+  ,ship_mode
+  ,web_site
+  ,date_dim
+where
+    d_month_seq between 1212 and 1212 + 11
+and ws_ship_date_sk   = d_date_sk
+and ws_warehouse_sk   = w_warehouse_sk
+and ws_ship_mode_sk   = sm_ship_mode_sk
+and ws_web_site_sk    = web_site_sk
+group by
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,web_name
+order by substr(w_warehouse_name,1,20)
+        ,sm_type
+       ,web_name
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q63.sql b/benchmarks/queries-tpcds/q63.sql
new file mode 100644
index 0000000000..d298609237
--- /dev/null
+++ b/benchmarks/queries-tpcds/q63.sql
@@ -0,0 +1,30 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  * 
+from (select i_manager_id
+             ,sum(ss_sales_price) sum_sales
+             ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales
+      from item
+          ,store_sales
+          ,date_dim
+          ,store
+      where ss_item_sk = i_item_sk
+        and ss_sold_date_sk = d_date_sk
+        and ss_store_sk = s_store_sk
+        and d_month_seq in (1212,1212+1,1212+2,1212+3,1212+4,1212+5,1212+6,1212+7,1212+8,1212+9,1212+10,1212+11)
+        and ((    i_category in ('Books','Children','Electronics')
+              and i_class in ('personal','portable','reference','self-help')
+              and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7',
+		                  'exportiunivamalg #9','scholaramalgamalg #9'))
+           or(    i_category in ('Women','Music','Men')
+              and i_class in ('accessories','classical','fragrances','pants')
+              and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1',
+		                 'importoamalg #1')))
+group by i_manager_id, d_moy) tmp1
+where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+order by i_manager_id
+        ,avg_monthly_sales
+        ,sum_sales
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q64.sql b/benchmarks/queries-tpcds/q64.sql
new file mode 100644
index 0000000000..26ddceb941
--- /dev/null
+++ b/benchmarks/queries-tpcds/q64.sql
@@ -0,0 +1,122 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with cs_ui as
+ (select cs_item_sk
+        ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund
+  from catalog_sales
+      ,catalog_returns
+  where cs_item_sk = cr_item_sk
+    and cs_order_number = cr_order_number
+  group by cs_item_sk
+  having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)),
+cross_sales as
+ (select i_product_name product_name
+     ,i_item_sk item_sk
+     ,s_store_name store_name
+     ,s_zip store_zip
+     ,ad1.ca_street_number b_street_number
+     ,ad1.ca_street_name b_street_name
+     ,ad1.ca_city b_city
+     ,ad1.ca_zip b_zip
+     ,ad2.ca_street_number c_street_number
+     ,ad2.ca_street_name c_street_name
+     ,ad2.ca_city c_city
+     ,ad2.ca_zip c_zip
+     ,d1.d_year as syear
+     ,d2.d_year as fsyear
+     ,d3.d_year s2year
+     ,count(*) cnt
+     ,sum(ss_wholesale_cost) s1
+     ,sum(ss_list_price) s2
+     ,sum(ss_coupon_amt) s3
+  FROM   store_sales
+        ,store_returns
+        ,cs_ui
+        ,date_dim d1
+        ,date_dim d2
+        ,date_dim d3
+        ,store
+        ,customer
+        ,customer_demographics cd1
+        ,customer_demographics cd2
+        ,promotion
+        ,household_demographics hd1
+        ,household_demographics hd2
+        ,customer_address ad1
+        ,customer_address ad2
+        ,income_band ib1
+        ,income_band ib2
+        ,item
+  WHERE  ss_store_sk = s_store_sk AND
+         ss_sold_date_sk = d1.d_date_sk AND
+         ss_customer_sk = c_customer_sk AND
+         ss_cdemo_sk= cd1.cd_demo_sk AND
+         ss_hdemo_sk = hd1.hd_demo_sk AND
+         ss_addr_sk = ad1.ca_address_sk and
+         ss_item_sk = i_item_sk and
+         ss_item_sk = sr_item_sk and
+         ss_ticket_number = sr_ticket_number and
+         ss_item_sk = cs_ui.cs_item_sk and
+         c_current_cdemo_sk = cd2.cd_demo_sk AND
+         c_current_hdemo_sk = hd2.hd_demo_sk AND
+         c_current_addr_sk = ad2.ca_address_sk and
+         c_first_sales_date_sk = d2.d_date_sk and
+         c_first_shipto_date_sk = d3.d_date_sk and
+         ss_promo_sk = p_promo_sk and
+         hd1.hd_income_band_sk = ib1.ib_income_band_sk and
+         hd2.hd_income_band_sk = ib2.ib_income_band_sk and
+         cd1.cd_marital_status <> cd2.cd_marital_status and
+         i_color in ('maroon','burnished','dim','steel','navajo','chocolate') and
+         i_current_price between 35 and 35 + 10 and
+         i_current_price between 35 + 1 and 35 + 15
+group by i_product_name
+       ,i_item_sk
+       ,s_store_name
+       ,s_zip
+       ,ad1.ca_street_number
+       ,ad1.ca_street_name
+       ,ad1.ca_city
+       ,ad1.ca_zip
+       ,ad2.ca_street_number
+       ,ad2.ca_street_name
+       ,ad2.ca_city
+       ,ad2.ca_zip
+       ,d1.d_year
+       ,d2.d_year
+       ,d3.d_year
+)
+select cs1.product_name
+     ,cs1.store_name
+     ,cs1.store_zip
+     ,cs1.b_street_number
+     ,cs1.b_street_name
+     ,cs1.b_city
+     ,cs1.b_zip
+     ,cs1.c_street_number
+     ,cs1.c_street_name
+     ,cs1.c_city
+     ,cs1.c_zip
+     ,cs1.syear
+     ,cs1.cnt
+     ,cs1.s1 as s11
+     ,cs1.s2 as s21
+     ,cs1.s3 as s31
+     ,cs2.s1 as s12
+     ,cs2.s2 as s22
+     ,cs2.s3 as s32
+     ,cs2.syear
+     ,cs2.cnt
+from cross_sales cs1,cross_sales cs2
+where cs1.item_sk=cs2.item_sk and
+     cs1.syear = 2000 and
+     cs2.syear = 2000 + 1 and
+     cs2.cnt <= cs1.cnt and
+     cs1.store_name = cs2.store_name and
+     cs1.store_zip = cs2.store_zip
+order by cs1.product_name
+       ,cs1.store_name
+       ,cs2.cnt
+       ,cs1.s1
+       ,cs2.s1;
+
+
diff --git a/benchmarks/queries-tpcds/q65.sql b/benchmarks/queries-tpcds/q65.sql
new file mode 100644
index 0000000000..36e3e48518
--- /dev/null
+++ b/benchmarks/queries-tpcds/q65.sql
@@ -0,0 +1,30 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select 
+	s_store_name,
+	i_item_desc,
+	sc.revenue,
+	i_current_price,
+	i_wholesale_cost,
+	i_brand
+ from store, item,
+     (select ss_store_sk, avg(revenue) as ave
+ 	from
+ 	    (select  ss_store_sk, ss_item_sk, 
+ 		     sum(ss_sales_price) as revenue
+ 		from store_sales, date_dim
+ 		where ss_sold_date_sk = d_date_sk and d_month_seq between 1212 and 1212+11
+ 		group by ss_store_sk, ss_item_sk) sa
+ 	group by ss_store_sk) sb,
+     (select  ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue
+ 	from store_sales, date_dim
+ 	where ss_sold_date_sk = d_date_sk and d_month_seq between 1212 and 1212+11
+ 	group by ss_store_sk, ss_item_sk) sc
+ where sb.ss_store_sk = sc.ss_store_sk and 
+       sc.revenue <= 0.1 * sb.ave and
+       s_store_sk = sc.ss_store_sk and
+       i_item_sk = sc.ss_item_sk
+ order by s_store_name, i_item_desc
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q66.sql b/benchmarks/queries-tpcds/q66.sql
new file mode 100644
index 0000000000..0840f234aa
--- /dev/null
+++ b/benchmarks/queries-tpcds/q66.sql
@@ -0,0 +1,221 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select   
+         w_warehouse_name
+ 	,w_warehouse_sq_ft
+ 	,w_city
+ 	,w_county
+ 	,w_state
+ 	,w_country
+        ,ship_carriers
+        ,year
+ 	,sum(jan_sales) as jan_sales
+ 	,sum(feb_sales) as feb_sales
+ 	,sum(mar_sales) as mar_sales
+ 	,sum(apr_sales) as apr_sales
+ 	,sum(may_sales) as may_sales
+ 	,sum(jun_sales) as jun_sales
+ 	,sum(jul_sales) as jul_sales
+ 	,sum(aug_sales) as aug_sales
+ 	,sum(sep_sales) as sep_sales
+ 	,sum(oct_sales) as oct_sales
+ 	,sum(nov_sales) as nov_sales
+ 	,sum(dec_sales) as dec_sales
+ 	,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot
+ 	,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot
+ 	,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot
+ 	,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot
+ 	,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot
+ 	,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot
+ 	,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot
+ 	,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot
+ 	,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot
+ 	,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot
+ 	,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot
+ 	,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot
+ 	,sum(jan_net) as jan_net
+ 	,sum(feb_net) as feb_net
+ 	,sum(mar_net) as mar_net
+ 	,sum(apr_net) as apr_net
+ 	,sum(may_net) as may_net
+ 	,sum(jun_net) as jun_net
+ 	,sum(jul_net) as jul_net
+ 	,sum(aug_net) as aug_net
+ 	,sum(sep_net) as sep_net
+ 	,sum(oct_net) as oct_net
+ 	,sum(nov_net) as nov_net
+ 	,sum(dec_net) as dec_net
+ from (
+     select 
+ 	w_warehouse_name
+ 	,w_warehouse_sq_ft
+ 	,w_city
+ 	,w_county
+ 	,w_state
+ 	,w_country
+ 	,'DIAMOND' || ',' || 'AIRBORNE' as ship_carriers
+       ,d_year as year
+ 	,sum(case when d_moy = 1 
+ 		then ws_sales_price* ws_quantity else 0 end) as jan_sales
+ 	,sum(case when d_moy = 2 
+ 		then ws_sales_price* ws_quantity else 0 end) as feb_sales
+ 	,sum(case when d_moy = 3 
+ 		then ws_sales_price* ws_quantity else 0 end) as mar_sales
+ 	,sum(case when d_moy = 4 
+ 		then ws_sales_price* ws_quantity else 0 end) as apr_sales
+ 	,sum(case when d_moy = 5 
+ 		then ws_sales_price* ws_quantity else 0 end) as may_sales
+ 	,sum(case when d_moy = 6 
+ 		then ws_sales_price* ws_quantity else 0 end) as jun_sales
+ 	,sum(case when d_moy = 7 
+ 		then ws_sales_price* ws_quantity else 0 end) as jul_sales
+ 	,sum(case when d_moy = 8 
+ 		then ws_sales_price* ws_quantity else 0 end) as aug_sales
+ 	,sum(case when d_moy = 9 
+ 		then ws_sales_price* ws_quantity else 0 end) as sep_sales
+ 	,sum(case when d_moy = 10 
+ 		then ws_sales_price* ws_quantity else 0 end) as oct_sales
+ 	,sum(case when d_moy = 11
+ 		then ws_sales_price* ws_quantity else 0 end) as nov_sales
+ 	,sum(case when d_moy = 12
+ 		then ws_sales_price* ws_quantity else 0 end) as dec_sales
+ 	,sum(case when d_moy = 1 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as jan_net
+ 	,sum(case when d_moy = 2
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as feb_net
+ 	,sum(case when d_moy = 3 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as mar_net
+ 	,sum(case when d_moy = 4 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as apr_net
+ 	,sum(case when d_moy = 5 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as may_net
+ 	,sum(case when d_moy = 6 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as jun_net
+ 	,sum(case when d_moy = 7 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as jul_net
+ 	,sum(case when d_moy = 8 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as aug_net
+ 	,sum(case when d_moy = 9 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as sep_net
+ 	,sum(case when d_moy = 10 
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as oct_net
+ 	,sum(case when d_moy = 11
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as nov_net
+ 	,sum(case when d_moy = 12
+ 		then ws_net_paid_inc_tax * ws_quantity else 0 end) as dec_net
+     from
+          web_sales
+         ,warehouse
+         ,date_dim
+         ,time_dim
+ 	  ,ship_mode
+     where
+            ws_warehouse_sk =  w_warehouse_sk
+        and ws_sold_date_sk = d_date_sk
+        and ws_sold_time_sk = t_time_sk
+ 	and ws_ship_mode_sk = sm_ship_mode_sk
+        and d_year = 2002
+ 	and t_time between 49530 and 49530+28800 
+ 	and sm_carrier in ('DIAMOND','AIRBORNE')
+     group by 
+        w_warehouse_name
+ 	,w_warehouse_sq_ft
+ 	,w_city
+ 	,w_county
+ 	,w_state
+ 	,w_country
+       ,d_year
+ union all
+     select 
+ 	w_warehouse_name
+ 	,w_warehouse_sq_ft
+ 	,w_city
+ 	,w_county
+ 	,w_state
+ 	,w_country
+ 	,'DIAMOND' || ',' || 'AIRBORNE' as ship_carriers
+       ,d_year as year
+ 	,sum(case when d_moy = 1 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as jan_sales
+ 	,sum(case when d_moy = 2 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as feb_sales
+ 	,sum(case when d_moy = 3 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as mar_sales
+ 	,sum(case when d_moy = 4 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as apr_sales
+ 	,sum(case when d_moy = 5 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as may_sales
+ 	,sum(case when d_moy = 6 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as jun_sales
+ 	,sum(case when d_moy = 7 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as jul_sales
+ 	,sum(case when d_moy = 8 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as aug_sales
+ 	,sum(case when d_moy = 9 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as sep_sales
+ 	,sum(case when d_moy = 10 
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as oct_sales
+ 	,sum(case when d_moy = 11
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as nov_sales
+ 	,sum(case when d_moy = 12
+ 		then cs_ext_sales_price* cs_quantity else 0 end) as dec_sales
+ 	,sum(case when d_moy = 1 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jan_net
+ 	,sum(case when d_moy = 2 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as feb_net
+ 	,sum(case when d_moy = 3 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as mar_net
+ 	,sum(case when d_moy = 4 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as apr_net
+ 	,sum(case when d_moy = 5 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as may_net
+ 	,sum(case when d_moy = 6 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jun_net
+ 	,sum(case when d_moy = 7 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jul_net
+ 	,sum(case when d_moy = 8 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as aug_net
+ 	,sum(case when d_moy = 9 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as sep_net
+ 	,sum(case when d_moy = 10 
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as oct_net
+ 	,sum(case when d_moy = 11
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as nov_net
+ 	,sum(case when d_moy = 12
+ 		then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as dec_net
+     from
+          catalog_sales
+         ,warehouse
+         ,date_dim
+         ,time_dim
+ 	 ,ship_mode
+     where
+            cs_warehouse_sk =  w_warehouse_sk
+        and cs_sold_date_sk = d_date_sk
+        and cs_sold_time_sk = t_time_sk
+ 	and cs_ship_mode_sk = sm_ship_mode_sk
+        and d_year = 2002
+ 	and t_time between 49530 AND 49530+28800 
+ 	and sm_carrier in ('DIAMOND','AIRBORNE')
+     group by 
+        w_warehouse_name
+ 	,w_warehouse_sq_ft
+ 	,w_city
+ 	,w_county
+ 	,w_state
+ 	,w_country
+       ,d_year
+ ) x
+ group by 
+        w_warehouse_name
+ 	,w_warehouse_sq_ft
+ 	,w_city
+ 	,w_county
+ 	,w_state
+ 	,w_country
+ 	,ship_carriers
+       ,year
+ order by w_warehouse_name
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q67.sql b/benchmarks/queries-tpcds/q67.sql
new file mode 100644
index 0000000000..55fa8d240f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q67.sql
@@ -0,0 +1,45 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  *
+from (select i_category
+            ,i_class
+            ,i_brand
+            ,i_product_name
+            ,d_year
+            ,d_qoy
+            ,d_moy
+            ,s_store_id
+            ,sumsales
+            ,rank() over (partition by i_category order by sumsales desc) rk
+      from (select i_category
+                  ,i_class
+                  ,i_brand
+                  ,i_product_name
+                  ,d_year
+                  ,d_qoy
+                  ,d_moy
+                  ,s_store_id
+                  ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales
+            from store_sales
+                ,date_dim
+                ,store
+                ,item
+       where  ss_sold_date_sk=d_date_sk
+          and ss_item_sk=i_item_sk
+          and ss_store_sk = s_store_sk
+          and d_month_seq between 1212 and 1212+11
+       group by  rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2
+where rk <= 100
+order by i_category
+        ,i_class
+        ,i_brand
+        ,i_product_name
+        ,d_year
+        ,d_qoy
+        ,d_moy
+        ,s_store_id
+        ,sumsales
+        ,rk
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q68.sql b/benchmarks/queries-tpcds/q68.sql
new file mode 100644
index 0000000000..350f7f7c79
--- /dev/null
+++ b/benchmarks/queries-tpcds/q68.sql
@@ -0,0 +1,43 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  c_last_name
+       ,c_first_name
+       ,ca_city
+       ,bought_city
+       ,ss_ticket_number
+       ,extended_price
+       ,extended_tax
+       ,list_price
+ from (select ss_ticket_number
+             ,ss_customer_sk
+             ,ca_city bought_city
+             ,sum(ss_ext_sales_price) extended_price 
+             ,sum(ss_ext_list_price) list_price
+             ,sum(ss_ext_tax) extended_tax 
+       from store_sales
+           ,date_dim
+           ,store
+           ,household_demographics
+           ,customer_address 
+       where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+         and store_sales.ss_store_sk = store.s_store_sk  
+        and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk
+        and store_sales.ss_addr_sk = customer_address.ca_address_sk
+        and date_dim.d_dom between 1 and 2 
+        and (household_demographics.hd_dep_count = 6 or
+             household_demographics.hd_vehicle_count= 3)
+        and date_dim.d_year in (1999,1999+1,1999+2)
+        and store.s_city in ('Oakland','Riverside')
+       group by ss_ticket_number
+               ,ss_customer_sk
+               ,ss_addr_sk,ca_city) dn
+      ,customer
+      ,customer_address current_addr
+ where ss_customer_sk = c_customer_sk
+   and customer.c_current_addr_sk = current_addr.ca_address_sk
+   and current_addr.ca_city <> bought_city
+ order by c_last_name
+         ,ss_ticket_number
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q69.sql b/benchmarks/queries-tpcds/q69.sql
new file mode 100644
index 0000000000..538c96445a
--- /dev/null
+++ b/benchmarks/queries-tpcds/q69.sql
@@ -0,0 +1,48 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+  cd_gender,
+  cd_marital_status,
+  cd_education_status,
+  count(*) cnt1,
+  cd_purchase_estimate,
+  count(*) cnt2,
+  cd_credit_rating,
+  count(*) cnt3
+ from
+  customer c,customer_address ca,customer_demographics
+ where
+  c.c_current_addr_sk = ca.ca_address_sk and
+  ca_state in ('CO','IL','MN') and
+  cd_demo_sk = c.c_current_cdemo_sk and 
+  exists (select *
+          from store_sales,date_dim
+          where c.c_customer_sk = ss_customer_sk and
+                ss_sold_date_sk = d_date_sk and
+                d_year = 1999 and
+                d_moy between 1 and 1+2) and
+   (not exists (select *
+            from web_sales,date_dim
+            where c.c_customer_sk = ws_bill_customer_sk and
+                  ws_sold_date_sk = d_date_sk and
+                  d_year = 1999 and
+                  d_moy between 1 and 1+2) and
+    not exists (select * 
+            from catalog_sales,date_dim
+            where c.c_customer_sk = cs_ship_customer_sk and
+                  cs_sold_date_sk = d_date_sk and
+                  d_year = 1999 and
+                  d_moy between 1 and 1+2))
+ group by cd_gender,
+          cd_marital_status,
+          cd_education_status,
+          cd_purchase_estimate,
+          cd_credit_rating
+ order by cd_gender,
+          cd_marital_status,
+          cd_education_status,
+          cd_purchase_estimate,
+          cd_credit_rating
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q7.sql b/benchmarks/queries-tpcds/q7.sql
new file mode 100644
index 0000000000..3b5a09d6ef
--- /dev/null
+++ b/benchmarks/queries-tpcds/q7.sql
@@ -0,0 +1,22 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id, 
+        avg(ss_quantity) agg1,
+        avg(ss_list_price) agg2,
+        avg(ss_coupon_amt) agg3,
+        avg(ss_sales_price) agg4 
+ from store_sales, customer_demographics, date_dim, item, promotion
+ where ss_sold_date_sk = d_date_sk and
+       ss_item_sk = i_item_sk and
+       ss_cdemo_sk = cd_demo_sk and
+       ss_promo_sk = p_promo_sk and
+       cd_gender = 'F' and 
+       cd_marital_status = 'W' and
+       cd_education_status = 'Primary' and
+       (p_channel_email = 'N' or p_channel_event = 'N') and
+       d_year = 1998 
+ group by i_item_id
+ order by i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q70.sql b/benchmarks/queries-tpcds/q70.sql
new file mode 100644
index 0000000000..ca93a3af04
--- /dev/null
+++ b/benchmarks/queries-tpcds/q70.sql
@@ -0,0 +1,39 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+    sum(ss_net_profit) as total_sum
+   ,s_state
+   ,s_county
+   ,grouping(s_state)+grouping(s_county) as lochierarchy
+   ,rank() over (
+ 	partition by grouping(s_state)+grouping(s_county),
+ 	case when grouping(s_county) = 0 then s_state end 
+ 	order by sum(ss_net_profit) desc) as rank_within_parent
+ from
+    store_sales
+   ,date_dim       d1
+   ,store
+ where
+    d1.d_month_seq between 1212 and 1212+11
+ and d1.d_date_sk = ss_sold_date_sk
+ and s_store_sk  = ss_store_sk
+ and s_state in
+             ( select s_state
+               from  (select s_state as s_state,
+ 			    rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking
+                      from   store_sales, store, date_dim
+                      where  d_month_seq between 1212 and 1212+11
+ 			    and d_date_sk = ss_sold_date_sk
+ 			    and s_store_sk  = ss_store_sk
+                      group by s_state
+                     ) tmp1 
+               where ranking <= 5
+             )
+ group by rollup(s_state,s_county)
+ order by
+   lochierarchy desc
+  ,case when lochierarchy = 0 then s_state end
+  ,rank_within_parent
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q71.sql b/benchmarks/queries-tpcds/q71.sql
new file mode 100644
index 0000000000..4b82c1b1ac
--- /dev/null
+++ b/benchmarks/queries-tpcds/q71.sql
@@ -0,0 +1,41 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select i_brand_id brand_id, i_brand brand,t_hour,t_minute,
+ 	sum(ext_price) ext_price
+ from item, (select ws_ext_sales_price as ext_price, 
+                        ws_sold_date_sk as sold_date_sk,
+                        ws_item_sk as sold_item_sk,
+                        ws_sold_time_sk as time_sk  
+                 from web_sales,date_dim
+                 where d_date_sk = ws_sold_date_sk
+                   and d_moy=12
+                   and d_year=2000
+                 union all
+                 select cs_ext_sales_price as ext_price,
+                        cs_sold_date_sk as sold_date_sk,
+                        cs_item_sk as sold_item_sk,
+                        cs_sold_time_sk as time_sk
+                 from catalog_sales,date_dim
+                 where d_date_sk = cs_sold_date_sk
+                   and d_moy=12
+                   and d_year=2000
+                 union all
+                 select ss_ext_sales_price as ext_price,
+                        ss_sold_date_sk as sold_date_sk,
+                        ss_item_sk as sold_item_sk,
+                        ss_sold_time_sk as time_sk
+                 from store_sales,date_dim
+                 where d_date_sk = ss_sold_date_sk
+                   and d_moy=12
+                   and d_year=2000
+                 ) tmp,time_dim
+ where
+   sold_item_sk = i_item_sk
+   and i_manager_id=1
+   and time_sk = t_time_sk
+   and (t_meal_time = 'breakfast' or t_meal_time = 'dinner')
+ group by i_brand, i_brand_id,t_hour,t_minute
+ order by ext_price desc, i_brand_id
+ ;
+
+
diff --git a/benchmarks/queries-tpcds/q72.sql b/benchmarks/queries-tpcds/q72.sql
new file mode 100644
index 0000000000..b561c46ac0
--- /dev/null
+++ b/benchmarks/queries-tpcds/q72.sql
@@ -0,0 +1,30 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_desc
+      ,w_warehouse_name
+      ,d1.d_week_seq
+      ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo
+      ,sum(case when p_promo_sk is not null then 1 else 0 end) promo
+      ,count(*) total_cnt
+from catalog_sales
+join inventory on (cs_item_sk = inv_item_sk)
+join warehouse on (w_warehouse_sk=inv_warehouse_sk)
+join item on (i_item_sk = cs_item_sk)
+join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk)
+join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk)
+join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk)
+join date_dim d2 on (inv_date_sk = d2.d_date_sk)
+join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk)
+left outer join promotion on (cs_promo_sk=p_promo_sk)
+left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number)
+where d1.d_week_seq = d2.d_week_seq
+  and inv_quantity_on_hand < cs_quantity
+  and d3.d_date > (d1.d_date + INTERVAL '5 days')
+  and hd_buy_potential = '1001-5000'
+  and d1.d_year = 2001
+  and cd_marital_status = 'M'
+group by i_item_desc,w_warehouse_name,d1.d_week_seq
+order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q73.sql b/benchmarks/queries-tpcds/q73.sql
new file mode 100644
index 0000000000..c103533e1f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q73.sql
@@ -0,0 +1,29 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select c_last_name
+       ,c_first_name
+       ,c_salutation
+       ,c_preferred_cust_flag 
+       ,ss_ticket_number
+       ,cnt from
+   (select ss_ticket_number
+          ,ss_customer_sk
+          ,count(*) cnt
+    from store_sales,date_dim,store,household_demographics
+    where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+    and store_sales.ss_store_sk = store.s_store_sk  
+    and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk
+    and date_dim.d_dom between 1 and 2 
+    and (household_demographics.hd_buy_potential = '>10000' or
+         household_demographics.hd_buy_potential = '5001-10000')
+    and household_demographics.hd_vehicle_count > 0
+    and case when household_demographics.hd_vehicle_count > 0 then 
+             household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1
+    and date_dim.d_year in (1999,1999+1,1999+2)
+    and store.s_county in ('Daviess County','Franklin Parish','Barrow County','Luce County')
+    group by ss_ticket_number,ss_customer_sk) dj,customer
+    where ss_customer_sk = c_customer_sk
+      and cnt between 1 and 5
+    order by cnt desc, c_last_name asc;
+
+
diff --git a/benchmarks/queries-tpcds/q74.sql b/benchmarks/queries-tpcds/q74.sql
new file mode 100644
index 0000000000..9804f4dce1
--- /dev/null
+++ b/benchmarks/queries-tpcds/q74.sql
@@ -0,0 +1,62 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with year_total as (
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,d_year as year
+       ,max(ss_net_paid) year_total
+       ,'s' sale_type
+ from customer
+     ,store_sales
+     ,date_dim
+ where c_customer_sk = ss_customer_sk
+   and ss_sold_date_sk = d_date_sk
+   and d_year in (2001,2001+1)
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,d_year
+ union all
+ select c_customer_id customer_id
+       ,c_first_name customer_first_name
+       ,c_last_name customer_last_name
+       ,d_year as year
+       ,max(ws_net_paid) year_total
+       ,'w' sale_type
+ from customer
+     ,web_sales
+     ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+   and ws_sold_date_sk = d_date_sk
+   and d_year in (2001,2001+1)
+ group by c_customer_id
+         ,c_first_name
+         ,c_last_name
+         ,d_year
+         )
+  select 
+        t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name
+ from year_total t_s_firstyear
+     ,year_total t_s_secyear
+     ,year_total t_w_firstyear
+     ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+         and t_s_firstyear.customer_id = t_w_secyear.customer_id
+         and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+         and t_s_firstyear.sale_type = 's'
+         and t_w_firstyear.sale_type = 'w'
+         and t_s_secyear.sale_type = 's'
+         and t_w_secyear.sale_type = 'w'
+         and t_s_firstyear.year = 2001
+         and t_s_secyear.year = 2001+1
+         and t_w_firstyear.year = 2001
+         and t_w_secyear.year = 2001+1
+         and t_s_firstyear.year_total > 0
+         and t_w_firstyear.year_total > 0
+         and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+           > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ order by 2,1,3
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q75.sql b/benchmarks/queries-tpcds/q75.sql
new file mode 100644
index 0000000000..41bd965b9f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q75.sql
@@ -0,0 +1,71 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+WITH all_sales AS (
+ SELECT d_year
+       ,i_brand_id
+       ,i_class_id
+       ,i_category_id
+       ,i_manufact_id
+       ,SUM(sales_cnt) AS sales_cnt
+       ,SUM(sales_amt) AS sales_amt
+ FROM (SELECT d_year
+             ,i_brand_id
+             ,i_class_id
+             ,i_category_id
+             ,i_manufact_id
+             ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt
+             ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt
+       FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk
+                          JOIN date_dim ON d_date_sk=cs_sold_date_sk
+                          LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number 
+                                                    AND cs_item_sk=cr_item_sk)
+       WHERE i_category='Sports'
+       UNION
+       SELECT d_year
+             ,i_brand_id
+             ,i_class_id
+             ,i_category_id
+             ,i_manufact_id
+             ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt
+             ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt
+       FROM store_sales JOIN item ON i_item_sk=ss_item_sk
+                        JOIN date_dim ON d_date_sk=ss_sold_date_sk
+                        LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number 
+                                                AND ss_item_sk=sr_item_sk)
+       WHERE i_category='Sports'
+       UNION
+       SELECT d_year
+             ,i_brand_id
+             ,i_class_id
+             ,i_category_id
+             ,i_manufact_id
+             ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt
+             ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt
+       FROM web_sales JOIN item ON i_item_sk=ws_item_sk
+                      JOIN date_dim ON d_date_sk=ws_sold_date_sk
+                      LEFT JOIN web_returns ON (ws_order_number=wr_order_number 
+                                            AND ws_item_sk=wr_item_sk)
+       WHERE i_category='Sports') sales_detail
+ GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id)
+ SELECT  prev_yr.d_year AS prev_year
+                          ,curr_yr.d_year AS year
+                          ,curr_yr.i_brand_id
+                          ,curr_yr.i_class_id
+                          ,curr_yr.i_category_id
+                          ,curr_yr.i_manufact_id
+                          ,prev_yr.sales_cnt AS prev_yr_cnt
+                          ,curr_yr.sales_cnt AS curr_yr_cnt
+                          ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff
+                          ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff
+ FROM all_sales curr_yr, all_sales prev_yr
+ WHERE curr_yr.i_brand_id=prev_yr.i_brand_id
+   AND curr_yr.i_class_id=prev_yr.i_class_id
+   AND curr_yr.i_category_id=prev_yr.i_category_id
+   AND curr_yr.i_manufact_id=prev_yr.i_manufact_id
+   AND curr_yr.d_year=2002
+   AND prev_yr.d_year=2002-1
+   AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9
+ ORDER BY sales_cnt_diff,sales_amt_diff
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q76.sql b/benchmarks/queries-tpcds/q76.sql
new file mode 100644
index 0000000000..8743a3ebc2
--- /dev/null
+++ b/benchmarks/queries-tpcds/q76.sql
@@ -0,0 +1,25 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM (
+        SELECT 'store' as channel, 'ss_addr_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price
+         FROM store_sales, item, date_dim
+         WHERE ss_addr_sk IS NULL
+           AND ss_sold_date_sk=d_date_sk
+           AND ss_item_sk=i_item_sk
+        UNION ALL
+        SELECT 'web' as channel, 'ws_web_page_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price
+         FROM web_sales, item, date_dim
+         WHERE ws_web_page_sk IS NULL
+           AND ws_sold_date_sk=d_date_sk
+           AND ws_item_sk=i_item_sk
+        UNION ALL
+        SELECT 'catalog' as channel, 'cs_warehouse_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price
+         FROM catalog_sales, item, date_dim
+         WHERE cs_warehouse_sk IS NULL
+           AND cs_sold_date_sk=d_date_sk
+           AND cs_item_sk=i_item_sk) foo
+GROUP BY channel, col_name, d_year, d_qoy, i_category
+ORDER BY channel, col_name, d_year, d_qoy, i_category
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q77.sql b/benchmarks/queries-tpcds/q77.sql
new file mode 100644
index 0000000000..152a945faa
--- /dev/null
+++ b/benchmarks/queries-tpcds/q77.sql
@@ -0,0 +1,109 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ss as
+ (select s_store_sk,
+         sum(ss_ext_sales_price) as sales,
+         sum(ss_net_profit) as profit
+ from store_sales,
+      date_dim,
+      store
+ where ss_sold_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date) 
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+       and ss_store_sk = s_store_sk
+ group by s_store_sk)
+ ,
+ sr as
+ (select s_store_sk,
+         sum(sr_return_amt) as returns,
+         sum(sr_net_loss) as profit_loss
+ from store_returns,
+      date_dim,
+      store
+ where sr_returned_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+       and sr_store_sk = s_store_sk
+ group by s_store_sk), 
+ cs as
+ (select cs_call_center_sk,
+        sum(cs_ext_sales_price) as sales,
+        sum(cs_net_profit) as profit
+ from catalog_sales,
+      date_dim
+ where cs_sold_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+ group by cs_call_center_sk 
+ ), 
+ cr as
+ (select cr_call_center_sk,
+         sum(cr_return_amount) as returns,
+         sum(cr_net_loss) as profit_loss
+ from catalog_returns,
+      date_dim
+ where cr_returned_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+ group by cr_call_center_sk
+ ), 
+ ws as
+ ( select wp_web_page_sk,
+        sum(ws_ext_sales_price) as sales,
+        sum(ws_net_profit) as profit
+ from web_sales,
+      date_dim,
+      web_page
+ where ws_sold_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+       and ws_web_page_sk = wp_web_page_sk
+ group by wp_web_page_sk), 
+ wr as
+ (select wp_web_page_sk,
+        sum(wr_return_amt) as returns,
+        sum(wr_net_loss) as profit_loss
+ from web_returns,
+      date_dim,
+      web_page
+ where wr_returned_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+       and wr_web_page_sk = wp_web_page_sk
+ group by wp_web_page_sk)
+  select  channel
+        , id
+        , sum(sales) as sales
+        , sum(returns) as returns
+        , sum(profit) as profit
+ from 
+ (select 'store channel' as channel
+        , ss.s_store_sk as id
+        , sales
+        , coalesce(returns, 0) as returns
+        , (profit - coalesce(profit_loss,0)) as profit
+ from   ss left join sr
+        on  ss.s_store_sk = sr.s_store_sk
+ union all
+ select 'catalog channel' as channel
+        , cs_call_center_sk as id
+        , sales
+        , returns
+        , (profit - profit_loss) as profit
+ from  cs
+       , cr
+ union all
+ select 'web channel' as channel
+        , ws.wp_web_page_sk as id
+        , sales
+        , coalesce(returns, 0) returns
+        , (profit - coalesce(profit_loss,0)) as profit
+ from   ws left join wr
+        on  ws.wp_web_page_sk = wr.wp_web_page_sk
+ ) x
+ group by rollup (channel, id)
+ order by channel
+         ,id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q78.sql b/benchmarks/queries-tpcds/q78.sql
new file mode 100644
index 0000000000..adc78aa09a
--- /dev/null
+++ b/benchmarks/queries-tpcds/q78.sql
@@ -0,0 +1,59 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ws as
+  (select d_year AS ws_sold_year, ws_item_sk,
+    ws_bill_customer_sk ws_customer_sk,
+    sum(ws_quantity) ws_qty,
+    sum(ws_wholesale_cost) ws_wc,
+    sum(ws_sales_price) ws_sp
+   from web_sales
+   left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk
+   join date_dim on ws_sold_date_sk = d_date_sk
+   where wr_order_number is null
+   group by d_year, ws_item_sk, ws_bill_customer_sk
+   ),
+cs as
+  (select d_year AS cs_sold_year, cs_item_sk,
+    cs_bill_customer_sk cs_customer_sk,
+    sum(cs_quantity) cs_qty,
+    sum(cs_wholesale_cost) cs_wc,
+    sum(cs_sales_price) cs_sp
+   from catalog_sales
+   left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk
+   join date_dim on cs_sold_date_sk = d_date_sk
+   where cr_order_number is null
+   group by d_year, cs_item_sk, cs_bill_customer_sk
+   ),
+ss as
+  (select d_year AS ss_sold_year, ss_item_sk,
+    ss_customer_sk,
+    sum(ss_quantity) ss_qty,
+    sum(ss_wholesale_cost) ss_wc,
+    sum(ss_sales_price) ss_sp
+   from store_sales
+   left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk
+   join date_dim on ss_sold_date_sk = d_date_sk
+   where sr_ticket_number is null
+   group by d_year, ss_item_sk, ss_customer_sk
+   )
+ select 
+ss_sold_year, ss_item_sk, ss_customer_sk,
+round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio,
+ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price,
+coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty,
+coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost,
+coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price
+from ss
+left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk)
+left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk)
+where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2000
+order by 
+  ss_sold_year, ss_item_sk, ss_customer_sk,
+  ss_qty desc, ss_wc desc, ss_sp desc,
+  other_chan_qty,
+  other_chan_wholesale_cost,
+  other_chan_sales_price,
+  ratio
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q79.sql b/benchmarks/queries-tpcds/q79.sql
new file mode 100644
index 0000000000..d0567eff43
--- /dev/null
+++ b/benchmarks/queries-tpcds/q79.sql
@@ -0,0 +1,24 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select 
+  c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit
+  from
+   (select ss_ticket_number
+          ,ss_customer_sk
+          ,store.s_city
+          ,sum(ss_coupon_amt) amt
+          ,sum(ss_net_profit) profit
+    from store_sales,date_dim,store,household_demographics
+    where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+    and store_sales.ss_store_sk = store.s_store_sk  
+    and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk
+    and (household_demographics.hd_dep_count = 8 or household_demographics.hd_vehicle_count > 0)
+    and date_dim.d_dow = 1
+    and date_dim.d_year in (1998,1998+1,1998+2) 
+    and store.s_number_employees between 200 and 295
+    group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer
+    where ss_customer_sk = c_customer_sk
+ order by c_last_name,c_first_name,substr(s_city,1,30), profit
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q8.sql b/benchmarks/queries-tpcds/q8.sql
new file mode 100644
index 0000000000..c831ed9fdf
--- /dev/null
+++ b/benchmarks/queries-tpcds/q8.sql
@@ -0,0 +1,109 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  s_store_name
+      ,sum(ss_net_profit)
+ from store_sales
+     ,date_dim
+     ,store,
+     (select ca_zip
+     from (
+      SELECT substr(ca_zip,1,5) ca_zip
+      FROM customer_address
+      WHERE substr(ca_zip,1,5) IN (
+                          '89436','30868','65085','22977','83927','77557',
+                          '58429','40697','80614','10502','32779',
+                          '91137','61265','98294','17921','18427',
+                          '21203','59362','87291','84093','21505',
+                          '17184','10866','67898','25797','28055',
+                          '18377','80332','74535','21757','29742',
+                          '90885','29898','17819','40811','25990',
+                          '47513','89531','91068','10391','18846',
+                          '99223','82637','41368','83658','86199',
+                          '81625','26696','89338','88425','32200',
+                          '81427','19053','77471','36610','99823',
+                          '43276','41249','48584','83550','82276',
+                          '18842','78890','14090','38123','40936',
+                          '34425','19850','43286','80072','79188',
+                          '54191','11395','50497','84861','90733',
+                          '21068','57666','37119','25004','57835',
+                          '70067','62878','95806','19303','18840',
+                          '19124','29785','16737','16022','49613',
+                          '89977','68310','60069','98360','48649',
+                          '39050','41793','25002','27413','39736',
+                          '47208','16515','94808','57648','15009',
+                          '80015','42961','63982','21744','71853',
+                          '81087','67468','34175','64008','20261',
+                          '11201','51799','48043','45645','61163',
+                          '48375','36447','57042','21218','41100',
+                          '89951','22745','35851','83326','61125',
+                          '78298','80752','49858','52940','96976',
+                          '63792','11376','53582','18717','90226',
+                          '50530','94203','99447','27670','96577',
+                          '57856','56372','16165','23427','54561',
+                          '28806','44439','22926','30123','61451',
+                          '92397','56979','92309','70873','13355',
+                          '21801','46346','37562','56458','28286',
+                          '47306','99555','69399','26234','47546',
+                          '49661','88601','35943','39936','25632',
+                          '24611','44166','56648','30379','59785',
+                          '11110','14329','93815','52226','71381',
+                          '13842','25612','63294','14664','21077',
+                          '82626','18799','60915','81020','56447',
+                          '76619','11433','13414','42548','92713',
+                          '70467','30884','47484','16072','38936',
+                          '13036','88376','45539','35901','19506',
+                          '65690','73957','71850','49231','14276',
+                          '20005','18384','76615','11635','38177',
+                          '55607','41369','95447','58581','58149',
+                          '91946','33790','76232','75692','95464',
+                          '22246','51061','56692','53121','77209',
+                          '15482','10688','14868','45907','73520',
+                          '72666','25734','17959','24677','66446',
+                          '94627','53535','15560','41967','69297',
+                          '11929','59403','33283','52232','57350',
+                          '43933','40921','36635','10827','71286',
+                          '19736','80619','25251','95042','15526',
+                          '36496','55854','49124','81980','35375',
+                          '49157','63512','28944','14946','36503',
+                          '54010','18767','23969','43905','66979',
+                          '33113','21286','58471','59080','13395',
+                          '79144','70373','67031','38360','26705',
+                          '50906','52406','26066','73146','15884',
+                          '31897','30045','61068','45550','92454',
+                          '13376','14354','19770','22928','97790',
+                          '50723','46081','30202','14410','20223',
+                          '88500','67298','13261','14172','81410',
+                          '93578','83583','46047','94167','82564',
+                          '21156','15799','86709','37931','74703',
+                          '83103','23054','70470','72008','49247',
+                          '91911','69998','20961','70070','63197',
+                          '54853','88191','91830','49521','19454',
+                          '81450','89091','62378','25683','61869',
+                          '51744','36580','85778','36871','48121',
+                          '28810','83712','45486','67393','26935',
+                          '42393','20132','55349','86057','21309',
+                          '80218','10094','11357','48819','39734',
+                          '40758','30432','21204','29467','30214',
+                          '61024','55307','74621','11622','68908',
+                          '33032','52868','99194','99900','84936',
+                          '69036','99149','45013','32895','59004',
+                          '32322','14933','32936','33562','72550',
+                          '27385','58049','58200','16808','21360',
+                          '32961','18586','79307','15492')
+     intersect
+      select ca_zip
+      from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt
+            FROM customer_address, customer
+            WHERE ca_address_sk = c_current_addr_sk and
+                  c_preferred_cust_flag='Y'
+            group by ca_zip
+            having count(*) > 10)A1)A2) V1
+ where ss_store_sk = s_store_sk
+  and ss_sold_date_sk = d_date_sk
+  and d_qoy = 1 and d_year = 2002
+  and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2))
+ group by s_store_name
+ order by s_store_name
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q80.sql b/benchmarks/queries-tpcds/q80.sql
new file mode 100644
index 0000000000..230e5d34fb
--- /dev/null
+++ b/benchmarks/queries-tpcds/q80.sql
@@ -0,0 +1,97 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ssr as
+ (select  s_store_id as store_id,
+          sum(ss_ext_sales_price) as sales,
+          sum(coalesce(sr_return_amt, 0)) as returns,
+          sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit
+  from store_sales left outer join store_returns on
+         (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number),
+     date_dim,
+     store,
+     item,
+     promotion
+ where ss_sold_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date) 
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+       and ss_store_sk = s_store_sk
+       and ss_item_sk = i_item_sk
+       and i_current_price > 50
+       and ss_promo_sk = p_promo_sk
+       and p_channel_tv = 'N'
+ group by s_store_id)
+ ,
+ csr as
+ (select  cp_catalog_page_id as catalog_page_id,
+          sum(cs_ext_sales_price) as sales,
+          sum(coalesce(cr_return_amount, 0)) as returns,
+          sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit
+  from catalog_sales left outer join catalog_returns on
+         (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number),
+     date_dim,
+     catalog_page,
+     item,
+     promotion
+ where cs_sold_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+        and cs_catalog_page_sk = cp_catalog_page_sk
+       and cs_item_sk = i_item_sk
+       and i_current_price > 50
+       and cs_promo_sk = p_promo_sk
+       and p_channel_tv = 'N'
+group by cp_catalog_page_id)
+ ,
+ wsr as
+ (select  web_site_id,
+          sum(ws_ext_sales_price) as sales,
+          sum(coalesce(wr_return_amt, 0)) as returns,
+          sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit
+  from web_sales left outer join web_returns on
+         (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number),
+     date_dim,
+     web_site,
+     item,
+     promotion
+ where ws_sold_date_sk = d_date_sk
+       and d_date between cast('1998-08-04' as date)
+                  and (cast('1998-08-04' as date) + INTERVAL '30 days')
+        and ws_web_site_sk = web_site_sk
+       and ws_item_sk = i_item_sk
+       and i_current_price > 50
+       and ws_promo_sk = p_promo_sk
+       and p_channel_tv = 'N'
+group by web_site_id)
+  select  channel
+        , id
+        , sum(sales) as sales
+        , sum(returns) as returns
+        , sum(profit) as profit
+ from 
+ (select 'store channel' as channel
+        , 'store' || store_id as id
+        , sales
+        , returns
+        , profit
+ from   ssr
+ union all
+ select 'catalog channel' as channel
+        , 'catalog_page' || catalog_page_id as id
+        , sales
+        , returns
+        , profit
+ from  csr
+ union all
+ select 'web channel' as channel
+        , 'web_site' || web_site_id as id
+        , sales
+        , returns
+        , profit
+ from   wsr
+ ) x
+ group by rollup (channel, id)
+ order by channel
+         ,id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q81.sql b/benchmarks/queries-tpcds/q81.sql
new file mode 100644
index 0000000000..f7d2047472
--- /dev/null
+++ b/benchmarks/queries-tpcds/q81.sql
@@ -0,0 +1,32 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with customer_total_return as
+ (select cr_returning_customer_sk as ctr_customer_sk
+        ,ca_state as ctr_state, 
+ 	sum(cr_return_amt_inc_tax) as ctr_total_return
+ from catalog_returns
+     ,date_dim
+     ,customer_address
+ where cr_returned_date_sk = d_date_sk 
+   and d_year =1998
+   and cr_returning_addr_sk = ca_address_sk 
+ group by cr_returning_customer_sk
+         ,ca_state )
+  select  c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name
+                   ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset
+                  ,ca_location_type,ctr_total_return
+ from customer_total_return ctr1
+     ,customer_address
+     ,customer
+ where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2
+ 			  from customer_total_return ctr2 
+                  	  where ctr1.ctr_state = ctr2.ctr_state)
+       and ca_address_sk = c_current_addr_sk
+       and ca_state = 'IL'
+       and ctr1.ctr_customer_sk = c_customer_sk
+ order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name
+                   ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset
+                  ,ca_location_type,ctr_total_return
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q82.sql b/benchmarks/queries-tpcds/q82.sql
new file mode 100644
index 0000000000..e2000a98c1
--- /dev/null
+++ b/benchmarks/queries-tpcds/q82.sql
@@ -0,0 +1,18 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  i_item_id
+       ,i_item_desc
+       ,i_current_price
+ from item, inventory, date_dim, store_sales
+ where i_current_price between 30 and 30+30
+ and inv_item_sk = i_item_sk
+ and d_date_sk=inv_date_sk
+ and d_date between cast('2002-05-30' as date) and (cast('2002-05-30' as date) + INTERVAL '60 days')
+ and i_manufact_id in (437,129,727,663)
+ and inv_quantity_on_hand between 100 and 500
+ and ss_item_sk = i_item_sk
+ group by i_item_id,i_item_desc,i_current_price
+ order by i_item_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q83.sql b/benchmarks/queries-tpcds/q83.sql
new file mode 100644
index 0000000000..43db876a4f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q83.sql
@@ -0,0 +1,68 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with sr_items as
+ (select i_item_id item_id,
+        sum(sr_return_quantity) sr_item_qty
+ from store_returns,
+      item,
+      date_dim
+ where sr_item_sk = i_item_sk
+ and   d_date    in 
+	(select d_date
+	from date_dim
+	where d_week_seq in 
+		(select d_week_seq
+		from date_dim
+	  where d_date in ('1998-01-02','1998-10-15','1998-11-10')))
+ and   sr_returned_date_sk   = d_date_sk
+ group by i_item_id),
+ cr_items as
+ (select i_item_id item_id,
+        sum(cr_return_quantity) cr_item_qty
+ from catalog_returns,
+      item,
+      date_dim
+ where cr_item_sk = i_item_sk
+ and   d_date    in 
+	(select d_date
+	from date_dim
+	where d_week_seq in 
+		(select d_week_seq
+		from date_dim
+	  where d_date in ('1998-01-02','1998-10-15','1998-11-10')))
+ and   cr_returned_date_sk   = d_date_sk
+ group by i_item_id),
+ wr_items as
+ (select i_item_id item_id,
+        sum(wr_return_quantity) wr_item_qty
+ from web_returns,
+      item,
+      date_dim
+ where wr_item_sk = i_item_sk
+ and   d_date    in 
+	(select d_date
+	from date_dim
+	where d_week_seq in 
+		(select d_week_seq
+		from date_dim
+		where d_date in ('1998-01-02','1998-10-15','1998-11-10')))
+ and   wr_returned_date_sk   = d_date_sk
+ group by i_item_id)
+  select  sr_items.item_id
+       ,sr_item_qty
+       ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev
+       ,cr_item_qty
+       ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev
+       ,wr_item_qty
+       ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev
+       ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average
+ from sr_items
+     ,cr_items
+     ,wr_items
+ where sr_items.item_id=cr_items.item_id
+   and sr_items.item_id=wr_items.item_id 
+ order by sr_items.item_id
+         ,sr_item_qty
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q84.sql b/benchmarks/queries-tpcds/q84.sql
new file mode 100644
index 0000000000..8bfed192c6
--- /dev/null
+++ b/benchmarks/queries-tpcds/q84.sql
@@ -0,0 +1,22 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  c_customer_id as customer_id
+       , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername
+ from customer
+     ,customer_address
+     ,customer_demographics
+     ,household_demographics
+     ,income_band
+     ,store_returns
+ where ca_city	        =  'Hopewell'
+   and c_current_addr_sk = ca_address_sk
+   and ib_lower_bound   >=  32287
+   and ib_upper_bound   <=  32287 + 50000
+   and ib_income_band_sk = hd_income_band_sk
+   and cd_demo_sk = c_current_cdemo_sk
+   and hd_demo_sk = c_current_hdemo_sk
+   and sr_cdemo_sk = cd_demo_sk
+ order by c_customer_id
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q85.sql b/benchmarks/queries-tpcds/q85.sql
new file mode 100644
index 0000000000..748657d30e
--- /dev/null
+++ b/benchmarks/queries-tpcds/q85.sql
@@ -0,0 +1,85 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  substr(r_reason_desc,1,20)
+       ,avg(ws_quantity)
+       ,avg(wr_refunded_cash)
+       ,avg(wr_fee)
+ from web_sales, web_returns, web_page, customer_demographics cd1,
+      customer_demographics cd2, customer_address, date_dim, reason 
+ where ws_web_page_sk = wp_web_page_sk
+   and ws_item_sk = wr_item_sk
+   and ws_order_number = wr_order_number
+   and ws_sold_date_sk = d_date_sk and d_year = 1998
+   and cd1.cd_demo_sk = wr_refunded_cdemo_sk 
+   and cd2.cd_demo_sk = wr_returning_cdemo_sk
+   and ca_address_sk = wr_refunded_addr_sk
+   and r_reason_sk = wr_reason_sk
+   and
+   (
+    (
+     cd1.cd_marital_status = 'M'
+     and
+     cd1.cd_marital_status = cd2.cd_marital_status
+     and
+     cd1.cd_education_status = '4 yr Degree'
+     and 
+     cd1.cd_education_status = cd2.cd_education_status
+     and
+     ws_sales_price between 100.00 and 150.00
+    )
+   or
+    (
+     cd1.cd_marital_status = 'D'
+     and
+     cd1.cd_marital_status = cd2.cd_marital_status
+     and
+     cd1.cd_education_status = 'Primary' 
+     and
+     cd1.cd_education_status = cd2.cd_education_status
+     and
+     ws_sales_price between 50.00 and 100.00
+    )
+   or
+    (
+     cd1.cd_marital_status = 'U'
+     and
+     cd1.cd_marital_status = cd2.cd_marital_status
+     and
+     cd1.cd_education_status = 'Advanced Degree'
+     and
+     cd1.cd_education_status = cd2.cd_education_status
+     and
+     ws_sales_price between 150.00 and 200.00
+    )
+   )
+   and
+   (
+    (
+     ca_country = 'United States'
+     and
+     ca_state in ('KY', 'GA', 'NM')
+     and ws_net_profit between 100 and 200  
+    )
+    or
+    (
+     ca_country = 'United States'
+     and
+     ca_state in ('MT', 'OR', 'IN')
+     and ws_net_profit between 150 and 300  
+    )
+    or
+    (
+     ca_country = 'United States'
+     and
+     ca_state in ('WI', 'MO', 'WV')
+     and ws_net_profit between 50 and 250  
+    )
+   )
+group by r_reason_desc
+order by substr(r_reason_desc,1,20)
+        ,avg(ws_quantity)
+        ,avg(wr_refunded_cash)
+        ,avg(wr_fee)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q86.sql b/benchmarks/queries-tpcds/q86.sql
new file mode 100644
index 0000000000..0dcff5e810
--- /dev/null
+++ b/benchmarks/queries-tpcds/q86.sql
@@ -0,0 +1,27 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select   
+    sum(ws_net_paid) as total_sum
+   ,i_category
+   ,i_class
+   ,grouping(i_category)+grouping(i_class) as lochierarchy
+   ,rank() over (
+ 	partition by grouping(i_category)+grouping(i_class),
+ 	case when grouping(i_class) = 0 then i_category end 
+ 	order by sum(ws_net_paid) desc) as rank_within_parent
+ from
+    web_sales
+   ,date_dim       d1
+   ,item
+ where
+    d1.d_month_seq between 1212 and 1212+11
+ and d1.d_date_sk = ws_sold_date_sk
+ and i_item_sk  = ws_item_sk
+ group by rollup(i_category,i_class)
+ order by
+   lochierarchy desc,
+   case when lochierarchy = 0 then i_category end,
+   rank_within_parent
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q87.sql b/benchmarks/queries-tpcds/q87.sql
new file mode 100644
index 0000000000..24e448891f
--- /dev/null
+++ b/benchmarks/queries-tpcds/q87.sql
@@ -0,0 +1,24 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select count(*) 
+from ((select distinct c_last_name, c_first_name, d_date
+       from store_sales, date_dim, customer
+       where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+         and store_sales.ss_customer_sk = customer.c_customer_sk
+         and d_month_seq between 1212 and 1212+11)
+       except
+      (select distinct c_last_name, c_first_name, d_date
+       from catalog_sales, date_dim, customer
+       where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk
+         and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk
+         and d_month_seq between 1212 and 1212+11)
+       except
+      (select distinct c_last_name, c_first_name, d_date
+       from web_sales, date_dim, customer
+       where web_sales.ws_sold_date_sk = date_dim.d_date_sk
+         and web_sales.ws_bill_customer_sk = customer.c_customer_sk
+         and d_month_seq between 1212 and 1212+11)
+) cool_cust
+;
+
+
diff --git a/benchmarks/queries-tpcds/q88.sql b/benchmarks/queries-tpcds/q88.sql
new file mode 100644
index 0000000000..393ff02c32
--- /dev/null
+++ b/benchmarks/queries-tpcds/q88.sql
@@ -0,0 +1,95 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  *
+from
+ (select count(*) h8_30_to_9
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk   
+     and ss_hdemo_sk = household_demographics.hd_demo_sk 
+     and ss_store_sk = s_store_sk
+     and time_dim.t_hour = 8
+     and time_dim.t_minute >= 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) 
+     and store.s_store_name = 'ese') s1,
+ (select count(*) h9_to_9_30 
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk 
+     and time_dim.t_hour = 9 
+     and time_dim.t_minute < 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s2,
+ (select count(*) h9_30_to_10 
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk
+     and time_dim.t_hour = 9
+     and time_dim.t_minute >= 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s3,
+ (select count(*) h10_to_10_30
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk
+     and time_dim.t_hour = 10 
+     and time_dim.t_minute < 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s4,
+ (select count(*) h10_30_to_11
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk
+     and time_dim.t_hour = 10 
+     and time_dim.t_minute >= 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s5,
+ (select count(*) h11_to_11_30
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk 
+     and time_dim.t_hour = 11
+     and time_dim.t_minute < 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s6,
+ (select count(*) h11_30_to_12
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk
+     and time_dim.t_hour = 11
+     and time_dim.t_minute >= 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s7,
+ (select count(*) h12_to_12_30
+ from store_sales, household_demographics , time_dim, store
+ where ss_sold_time_sk = time_dim.t_time_sk
+     and ss_hdemo_sk = household_demographics.hd_demo_sk
+     and ss_store_sk = s_store_sk
+     and time_dim.t_hour = 12
+     and time_dim.t_minute < 30
+     and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or
+          (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or
+          (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2))
+     and store.s_store_name = 'ese') s8
+;
+
+
diff --git a/benchmarks/queries-tpcds/q89.sql b/benchmarks/queries-tpcds/q89.sql
new file mode 100644
index 0000000000..d87c2c629e
--- /dev/null
+++ b/benchmarks/queries-tpcds/q89.sql
@@ -0,0 +1,29 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  *
+from(
+select i_category, i_class, i_brand,
+       s_store_name, s_company_name,
+       d_moy,
+       sum(ss_sales_price) sum_sales,
+       avg(sum(ss_sales_price)) over
+         (partition by i_category, i_brand, s_store_name, s_company_name)
+         avg_monthly_sales
+from item, store_sales, date_dim, store
+where ss_item_sk = i_item_sk and
+      ss_sold_date_sk = d_date_sk and
+      ss_store_sk = s_store_sk and
+      d_year in (2000) and
+        ((i_category in ('Home','Books','Electronics') and
+          i_class in ('wallpaper','parenting','musical')
+         )
+      or (i_category in ('Shoes','Jewelry','Men') and
+          i_class in ('womens','birdal','pants') 
+        ))
+group by i_category, i_class, i_brand,
+         s_store_name, s_company_name, d_moy) tmp1
+where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1
+order by sum_sales - avg_monthly_sales, s_store_name
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q9.sql b/benchmarks/queries-tpcds/q9.sql
new file mode 100644
index 0000000000..0b5e4d0636
--- /dev/null
+++ b/benchmarks/queries-tpcds/q9.sql
@@ -0,0 +1,52 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select case when (select count(*) 
+                  from store_sales 
+                  where ss_quantity between 1 and 20) > 409437
+            then (select avg(ss_ext_tax) 
+                  from store_sales 
+                  where ss_quantity between 1 and 20) 
+            else (select avg(ss_net_paid)
+                  from store_sales
+                  where ss_quantity between 1 and 20) end bucket1 ,
+       case when (select count(*)
+                  from store_sales
+                  where ss_quantity between 21 and 40) > 4595804
+            then (select avg(ss_ext_tax)
+                  from store_sales
+                  where ss_quantity between 21 and 40) 
+            else (select avg(ss_net_paid)
+                  from store_sales
+                  where ss_quantity between 21 and 40) end bucket2,
+       case when (select count(*)
+                  from store_sales
+                  where ss_quantity between 41 and 60) > 1333710
+            then (select avg(ss_ext_tax)
+                  from store_sales
+                  where ss_quantity between 41 and 60)
+            else (select avg(ss_net_paid)
+                  from store_sales
+                  where ss_quantity between 41 and 60) end bucket3,
+       case when (select count(*)
+                  from store_sales
+                  where ss_quantity between 61 and 80) > 2361102
+            then (select avg(ss_ext_tax)
+                  from store_sales
+                  where ss_quantity between 61 and 80)
+            else (select avg(ss_net_paid)
+                  from store_sales
+                  where ss_quantity between 61 and 80) end bucket4,
+       case when (select count(*)
+                  from store_sales
+                  where ss_quantity between 81 and 100) > 1517817
+            then (select avg(ss_ext_tax)
+                  from store_sales
+                  where ss_quantity between 81 and 100)
+            else (select avg(ss_net_paid)
+                  from store_sales
+                  where ss_quantity between 81 and 100) end bucket5
+from reason
+where r_reason_sk = 1
+;
+
+
diff --git a/benchmarks/queries-tpcds/q90.sql b/benchmarks/queries-tpcds/q90.sql
new file mode 100644
index 0000000000..62a62d0f41
--- /dev/null
+++ b/benchmarks/queries-tpcds/q90.sql
@@ -0,0 +1,23 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio
+ from ( select count(*) amc
+       from web_sales, household_demographics , time_dim, web_page
+       where ws_sold_time_sk = time_dim.t_time_sk
+         and ws_ship_hdemo_sk = household_demographics.hd_demo_sk
+         and ws_web_page_sk = web_page.wp_web_page_sk
+         and time_dim.t_hour between 6 and 6+1
+         and household_demographics.hd_dep_count = 8
+         and web_page.wp_char_count between 5000 and 5200) at,
+      ( select count(*) pmc
+       from web_sales, household_demographics , time_dim, web_page
+       where ws_sold_time_sk = time_dim.t_time_sk
+         and ws_ship_hdemo_sk = household_demographics.hd_demo_sk
+         and ws_web_page_sk = web_page.wp_web_page_sk
+         and time_dim.t_hour between 14 and 14+1
+         and household_demographics.hd_dep_count = 8
+         and web_page.wp_char_count between 5000 and 5200) pt
+ order by am_pm_ratio
+ limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q91.sql b/benchmarks/queries-tpcds/q91.sql
new file mode 100644
index 0000000000..42aa647ee8
--- /dev/null
+++ b/benchmarks/queries-tpcds/q91.sql
@@ -0,0 +1,32 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+        cc_call_center_id Call_Center,
+        cc_name Call_Center_Name,
+        cc_manager Manager,
+        sum(cr_net_loss) Returns_Loss
+from
+        call_center,
+        catalog_returns,
+        date_dim,
+        customer,
+        customer_address,
+        customer_demographics,
+        household_demographics
+where
+        cr_call_center_sk       = cc_call_center_sk
+and     cr_returned_date_sk     = d_date_sk
+and     cr_returning_customer_sk= c_customer_sk
+and     cd_demo_sk              = c_current_cdemo_sk
+and     hd_demo_sk              = c_current_hdemo_sk
+and     ca_address_sk           = c_current_addr_sk
+and     d_year                  = 1999 
+and     d_moy                   = 11
+and     ( (cd_marital_status       = 'M' and cd_education_status     = 'Unknown')
+        or(cd_marital_status       = 'W' and cd_education_status     = 'Advanced Degree'))
+and     hd_buy_potential like '0-500%'
+and     ca_gmt_offset           = -7
+group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status
+order by sum(cr_net_loss) desc;
+
+
diff --git a/benchmarks/queries-tpcds/q92.sql b/benchmarks/queries-tpcds/q92.sql
new file mode 100644
index 0000000000..425962e5b1
--- /dev/null
+++ b/benchmarks/queries-tpcds/q92.sql
@@ -0,0 +1,31 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   sum(ws_ext_discount_amt)  as 'Excess Discount Amount'
+from 
+    web_sales 
+   ,item 
+   ,date_dim
+where
+i_manufact_id = 269
+and i_item_sk = ws_item_sk 
+and d_date between '1998-03-18' and 
+        (cast('1998-03-18' as date) + INTERVAL '90 days')
+and d_date_sk = ws_sold_date_sk 
+and ws_ext_discount_amt  
+     > ( 
+         SELECT 
+            1.3 * avg(ws_ext_discount_amt) 
+         FROM 
+            web_sales 
+           ,date_dim
+         WHERE 
+              ws_item_sk = i_item_sk 
+          and d_date between '1998-03-18' and
+                             (cast('1998-03-18' as date) + INTERVAL '90 days')
+          and d_date_sk = ws_sold_date_sk 
+      ) 
+order by sum(ws_ext_discount_amt)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q93.sql b/benchmarks/queries-tpcds/q93.sql
new file mode 100644
index 0000000000..3349cda897
--- /dev/null
+++ b/benchmarks/queries-tpcds/q93.sql
@@ -0,0 +1,19 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  ss_customer_sk
+            ,sum(act_sales) sumsales
+      from (select ss_item_sk
+                  ,ss_ticket_number
+                  ,ss_customer_sk
+                  ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price
+                                                            else (ss_quantity*ss_sales_price) end act_sales
+            from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk
+                                                               and sr_ticket_number = ss_ticket_number)
+                ,reason
+            where sr_reason_sk = r_reason_sk
+              and r_reason_desc = 'Did not like the warranty') t
+      group by ss_customer_sk
+      order by sumsales, ss_customer_sk
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q94.sql b/benchmarks/queries-tpcds/q94.sql
new file mode 100644
index 0000000000..17e788d443
--- /dev/null
+++ b/benchmarks/queries-tpcds/q94.sql
@@ -0,0 +1,30 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   count(distinct ws_order_number) as 'order count'
+  ,sum(ws_ext_ship_cost) as 'total shipping cost'
+  ,sum(ws_net_profit) as 'total net profit'
+from
+   web_sales ws1
+  ,date_dim
+  ,customer_address
+  ,web_site
+where
+    d_date between '1999-5-01' and 
+           (cast('1999-5-01' as date) + INTERVAL '60 days')
+and ws1.ws_ship_date_sk = d_date_sk
+and ws1.ws_ship_addr_sk = ca_address_sk
+and ca_state = 'TX'
+and ws1.ws_web_site_sk = web_site_sk
+and web_company_name = 'pri'
+and exists (select *
+            from web_sales ws2
+            where ws1.ws_order_number = ws2.ws_order_number
+              and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk)
+and not exists(select *
+               from web_returns wr1
+               where ws1.ws_order_number = wr1.wr_order_number)
+order by count(distinct ws_order_number)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q95.sql b/benchmarks/queries-tpcds/q95.sql
new file mode 100644
index 0000000000..1f1c9331cd
--- /dev/null
+++ b/benchmarks/queries-tpcds/q95.sql
@@ -0,0 +1,33 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ws_wh as
+(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2
+ from web_sales ws1,web_sales ws2
+ where ws1.ws_order_number = ws2.ws_order_number
+   and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk)
+ select  
+   count(distinct ws_order_number) as 'order count'
+  ,sum(ws_ext_ship_cost) as 'total shipping cost'
+  ,sum(ws_net_profit) as 'total net profit'
+from
+   web_sales ws1
+  ,date_dim
+  ,customer_address
+  ,web_site
+where
+    d_date between '1999-5-01' and 
+           (cast('1999-5-01' as date) + INTERVAL '60 days')
+and ws1.ws_ship_date_sk = d_date_sk
+and ws1.ws_ship_addr_sk = ca_address_sk
+and ca_state = 'TX'
+and ws1.ws_web_site_sk = web_site_sk
+and web_company_name = 'pri'
+and ws1.ws_order_number in (select ws_order_number
+                            from ws_wh)
+and ws1.ws_order_number in (select wr_order_number
+                            from web_returns,ws_wh
+                            where wr_order_number = ws_wh.ws_order_number)
+order by count(distinct ws_order_number)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q96.sql b/benchmarks/queries-tpcds/q96.sql
new file mode 100644
index 0000000000..26491404ac
--- /dev/null
+++ b/benchmarks/queries-tpcds/q96.sql
@@ -0,0 +1,17 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  count(*) 
+from store_sales
+    ,household_demographics 
+    ,time_dim, store
+where ss_sold_time_sk = time_dim.t_time_sk   
+    and ss_hdemo_sk = household_demographics.hd_demo_sk 
+    and ss_store_sk = s_store_sk
+    and time_dim.t_hour = 8
+    and time_dim.t_minute >= 30
+    and household_demographics.hd_dep_count = 5
+    and store.s_store_name = 'ese'
+order by count(*)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q97.sql b/benchmarks/queries-tpcds/q97.sql
new file mode 100644
index 0000000000..814f6c6064
--- /dev/null
+++ b/benchmarks/queries-tpcds/q97.sql
@@ -0,0 +1,26 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+with ssci as (
+select ss_customer_sk customer_sk
+      ,ss_item_sk item_sk
+from store_sales,date_dim
+where ss_sold_date_sk = d_date_sk
+  and d_month_seq between 1212 and 1212 + 11
+group by ss_customer_sk
+        ,ss_item_sk),
+csci as(
+ select cs_bill_customer_sk customer_sk
+      ,cs_item_sk item_sk
+from catalog_sales,date_dim
+where cs_sold_date_sk = d_date_sk
+  and d_month_seq between 1212 and 1212 + 11
+group by cs_bill_customer_sk
+        ,cs_item_sk)
+ select  sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only
+      ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only
+      ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog
+from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk
+                               and ssci.item_sk = csci.item_sk)
+limit 100;
+
+
diff --git a/benchmarks/queries-tpcds/q98.sql b/benchmarks/queries-tpcds/q98.sql
new file mode 100644
index 0000000000..1e0c9738f0
--- /dev/null
+++ b/benchmarks/queries-tpcds/q98.sql
@@ -0,0 +1,34 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select i_item_id
+      ,i_item_desc 
+      ,i_category 
+      ,i_class 
+      ,i_current_price
+      ,sum(ss_ext_sales_price) as itemrevenue 
+      ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over
+          (partition by i_class) as revenueratio
+from	
+	store_sales
+    	,item 
+    	,date_dim
+where 
+	ss_item_sk = i_item_sk 
+  	and i_category in ('Jewelry', 'Sports', 'Books')
+  	and ss_sold_date_sk = d_date_sk
+	and d_date between cast('2001-01-12' as date) 
+				and (cast('2001-01-12' as date) + INTERVAL '30 days')
+group by 
+	i_item_id
+        ,i_item_desc 
+        ,i_category
+        ,i_class
+        ,i_current_price
+order by 
+	i_category
+        ,i_class
+        ,i_item_id
+        ,i_item_desc
+        ,revenueratio;
+
+
diff --git a/benchmarks/queries-tpcds/q99.sql b/benchmarks/queries-tpcds/q99.sql
new file mode 100644
index 0000000000..ca47cd2f65
--- /dev/null
+++ b/benchmarks/queries-tpcds/q99.sql
@@ -0,0 +1,36 @@
+-- Copyright 2015 Transaction Processing Performance Council
+
+select  
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,cc_name
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end)  as '30 days'
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and 
+                 (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end )  as '31-60 days'
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and 
+                 (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end)  as '61-90 days'
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and
+                 (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end)  as '91-120 days'
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk  > 120) then 1 else 0 end)  as '>120 days'
+from
+   catalog_sales
+  ,warehouse
+  ,ship_mode
+  ,call_center
+  ,date_dim
+where
+    d_month_seq between 1212 and 1212 + 11
+and cs_ship_date_sk   = d_date_sk
+and cs_warehouse_sk   = w_warehouse_sk
+and cs_ship_mode_sk   = sm_ship_mode_sk
+and cs_call_center_sk = cc_call_center_sk
+group by
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,cc_name
+order by substr(w_warehouse_name,1,20)
+        ,sm_type
+        ,cc_name
+limit 100;
+
+
diff --git a/benchmarks/src/bin/shuffle_bench.rs b/benchmarks/src/bin/shuffle_bench.rs
index 934f80a236..8fd9184d28 100644
--- a/benchmarks/src/bin/shuffle_bench.rs
+++ b/benchmarks/src/bin/shuffle_bench.rs
@@ -43,7 +43,6 @@ use ballista_core::execution_plans::sort_shuffle::{
 use ballista_core::utils;
 use clap::Parser;
 use datafusion::arrow::datatypes::{DataType, SchemaRef};
-use datafusion::arrow::ipc::CompressionType;
 use datafusion::execution::config::SessionConfig;
 use datafusion::execution::runtime_env::RuntimeEnvBuilder;
 use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
@@ -115,7 +114,7 @@ struct Args {
 
     /// Concurrent shuffle tasks to simulate executor parallelism.
     #[arg(long, default_value_t = 1)]
-    concurrent_tasks: usize,
+    vcores: usize,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -286,8 +285,7 @@ async fn execute_shuffle_write(
             exec.metrics().unwrap_or_default()
         }
         WriterKind::Sort => {
-            let cfg =
-                SortShuffleConfig::new(true, CompressionType::LZ4_FRAME, args.batch_size);
+            let cfg = SortShuffleConfig::new(true, args.batch_size);
             let exec = SortShuffleWriterExec::try_new(
                 format!("bench_job_{task_id}").into(),
                 1,
@@ -315,7 +313,7 @@ fn run_iteration(
     let rt = tokio::runtime::Runtime::new().unwrap();
     rt.block_on(async {
         let start = Instant::now();
-        if args.concurrent_tasks <= 1 {
+        if args.vcores <= 1 {
             let work_dir = args.output_dir.join("task_0");
             let metrics = execute_shuffle_write(
                 args,
@@ -331,8 +329,8 @@ fn run_iteration(
             let _ = fs::remove_dir_all(&work_dir);
             (elapsed, Some(metrics))
         } else {
-            let mut handles = Vec::with_capacity(args.concurrent_tasks);
-            for task_id in 0..args.concurrent_tasks {
+            let mut handles = Vec::with_capacity(args.vcores);
+            for task_id in 0..args.vcores {
                 let args = args.clone();
                 let hash_col_indices = hash_col_indices.to_vec();
                 let work_dir = args.output_dir.join(format!("task_{task_id}"));
@@ -464,8 +462,8 @@ fn main() {
     if let Some(m) = args.memory_limit {
         println!("Memory limit:   {m} bytes");
     }
-    if args.concurrent_tasks > 1 {
-        println!("Concurrent:     {} tasks", args.concurrent_tasks);
+    if args.vcores > 1 {
+        println!("Concurrent:     {} tasks", args.vcores);
     }
     println!(
         "Iterations:     {} (warmup {})",
@@ -497,7 +495,7 @@ fn main() {
 
     if !times.is_empty() {
         let avg = times.iter().sum::<f64>() / times.len() as f64;
-        let total_writer_rows = total_rows * args.concurrent_tasks as u64;
+        let total_writer_rows = total_rows * args.vcores as u64;
         println!();
         println!("=== Results ===");
         println!("avg time: {avg:.3}s");
@@ -509,7 +507,7 @@ fn main() {
         println!(
             "throughput: {} rows/s (total across {} tasks)",
             (total_writer_rows as f64 / avg) as u64,
-            args.concurrent_tasks
+            args.vcores
         );
         if let Some(metrics) = last_metrics {
             println!();
diff --git a/benchmarks/src/bin/tpcds.rs b/benchmarks/src/bin/tpcds.rs
new file mode 100644
index 0000000000..b55cdfeacf
--- /dev/null
+++ b/benchmarks/src/bin/tpcds.rs
@@ -0,0 +1,341 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use ballista::extension::SessionConfigExt;
+use ballista::prelude::SessionContextExt;
+use ballista_benchmarks::{
+    answer_statement_index, compare_results, execute_query_capturing_answer,
+    register_parquet_tables,
+};
+use ballista_core::object_store::{
+    session_config_with_s3_support, session_state_with_s3_support,
+};
+use datafusion::error::{DataFusionError, Result};
+use datafusion::prelude::{SessionConfig, SessionContext};
+use std::fs;
+use std::time::Instant;
+use structopt::StructOpt;
+
+/// The 24 TPC-DS tables.
+const TABLES: &[&str] = &[
+    "call_center",
+    "catalog_page",
+    "catalog_returns",
+    "catalog_sales",
+    "customer",
+    "customer_address",
+    "customer_demographics",
+    "date_dim",
+    "household_demographics",
+    "income_band",
+    "inventory",
+    "item",
+    "promotion",
+    "reason",
+    "ship_mode",
+    "store",
+    "store_returns",
+    "store_sales",
+    "time_dim",
+    "warehouse",
+    "web_page",
+    "web_returns",
+    "web_sales",
+    "web_site",
+];
+
+/// Queries excluded from the correctness gate, with the reason. The gate runs
+/// under the default (static) planner; this list reflects that configuration.
+/// Remove an entry as the underlying cause is fixed.
+const SKIP: &[(usize, &str)] = &[
+    // Distributed execution diverges from single-process DataFusion on the same
+    // data (confirmed reproducible; DataFusion is correct under both join modes).
+    // See https://github.com/apache/datafusion-ballista/issues/2046
+    (
+        38,
+        "distributed INTERSECT diverges from DataFusion (issue #2046)",
+    ),
+    (
+        87,
+        "distributed EXCEPT diverges from DataFusion (issue #2046)",
+    ),
+    // Non-deterministic: LIMIT/ORDER BY ties without a total order make the
+    // result vary run-to-run in both engines, so a row-by-row diff is unstable.
+    (31, "non-deterministic (ORDER BY ties; varies run-to-run)"),
+    (
+        71,
+        "non-deterministic (ORDER BY ext_price ties; varies run-to-run)",
+    ),
+    // tpcgen-cli's TPC-DS schema uses column names that differ from the
+    // DataFusion (branch-54) query text, so these fail at plan time. Not a
+    // Ballista issue (e.g. cr_return_amount_inc_tax vs cr_return_amt_inc_tax).
+    (64, "tpcgen-cli schema column-name mismatch with query text"),
+    (81, "tpcgen-cli schema column-name mismatch with query text"),
+    (84, "tpcgen-cli schema column-name mismatch with query text"),
+    (85, "tpcgen-cli schema column-name mismatch with query text"),
+    (93, "tpcgen-cli schema column-name mismatch with query text"),
+    // Note: the adaptive planner (AQE on) additionally fails many queries with an
+    // `EmptyExec invalid partition` assertion (issue #2047); the gate runs the
+    // static planner, so those are not listed here.
+];
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "tpcds", about = "Ballista TPC-DS correctness runner")]
+struct Opt {
+    /// Query number (1-99). If not specified, runs all non-skipped queries.
+    #[structopt(short, long)]
+    query: Option<usize>,
+
+    /// Show query text and results.
+    #[structopt(short, long)]
+    debug: bool,
+
+    /// Path to data files (local path or object-store URL).
+    #[structopt(required = true, short = "p", long = "path")]
+    path: String,
+
+    /// Number of partitions (session target_partitions).
+    #[structopt(short = "n", long = "partitions", default_value = "2")]
+    partitions: usize,
+
+    /// Batch size.
+    #[structopt(short = "s", long = "batch-size", default_value = "8192")]
+    batch_size: usize,
+
+    /// Ballista scheduler host.
+    #[structopt(long = "host")]
+    host: String,
+
+    /// Ballista scheduler port.
+    #[structopt(long = "port")]
+    port: u16,
+
+    /// Config overrides in key=value form (repeatable).
+    #[structopt(short = "c", long = "config", number_of_values = 1)]
+    config_overrides: Vec<String>,
+
+    /// Verify each Ballista result against single-process DataFusion.
+    #[structopt(long = "verify")]
+    verify: bool,
+}
+
+/// Split a query file into statements, dropping full-line `--` comments and
+/// blank statements. TPC-DS files carry a leading TPC copyright comment.
+fn split_statements(contents: &str) -> Vec<String> {
+    let stripped: String = contents
+        .lines()
+        .filter(|l| !l.trim_start().starts_with("--"))
+        .collect::<Vec<_>>()
+        .join("\n");
+    stripped
+        .split(';')
+        .map(|s| s.trim())
+        .filter(|s| !s.is_empty())
+        .map(|s| s.to_string())
+        .collect()
+}
+
+fn get_query_sql(query: usize) -> Result<Vec<String>> {
+    let possibilities = [
+        format!("queries-tpcds/q{query}.sql"),
+        format!("benchmarks/queries-tpcds/q{query}.sql"),
+    ];
+    let mut errors = vec![];
+    for filename in &possibilities {
+        match fs::read_to_string(filename) {
+            Ok(contents) => return Ok(split_statements(&contents)),
+            Err(e) => errors.push(format!("{filename}: {e}")),
+        }
+    }
+    Err(DataFusionError::Plan(format!(
+        "Could not find query {query}: {errors:?}"
+    )))
+}
+
+/// The queries to run: an explicit `--query`, else 1..=99 minus `skip`.
+fn selected_queries(explicit: Option<usize>, skip: &[(usize, &str)]) -> Vec<usize> {
+    if let Some(q) = explicit {
+        return vec![q];
+    }
+    let skipped: std::collections::HashSet<usize> =
+        skip.iter().map(|(id, _)| *id).collect();
+    (1..=99).filter(|q| !skipped.contains(q)).collect()
+}
+
+/// Run a single TPC-DS query end to end: load its SQL, stand up a fresh
+/// Ballista session, execute it on the cluster, and (if `oracle_ctx` is
+/// set) verify the result against single-process DataFusion.
+///
+/// Every fallible step is tagged with a `<phase>: ` prefix and returned as
+/// an `Err` rather than aborting the process, so the caller can record the
+/// failure against this query and move on to the next one.
+async fn run_one_query(
+    opt: &Opt,
+    address: &str,
+    oracle_ctx: Option<&SessionContext>,
+    query: usize,
+) -> Result<()> {
+    let sqls = get_query_sql(query)
+        .map_err(|e| DataFusionError::Execution(format!("load: {e}")))?;
+
+    // Build a fresh Ballista session per query (mirrors tpch.rs).
+    let mut config = session_config_with_s3_support()
+        .with_target_partitions(opt.partitions)
+        .with_ballista_job_name(&format!("TPC-DS q{query}"))
+        .with_batch_size(opt.batch_size)
+        .with_collect_statistics(true);
+    for kv in &opt.config_overrides {
+        if let Some((k, v)) = kv.split_once('=') {
+            if let Err(e) = config.options_mut().set(k.trim(), v.trim()) {
+                println!("Warning: could not set config '{kv}': {e}");
+            }
+        } else {
+            println!("Warning: ignoring invalid config override '{kv}'");
+        }
+    }
+    let state = session_state_with_s3_support(config)
+        .map_err(|e| DataFusionError::Execution(format!("session-state: {e}")))?;
+    let ctx = SessionContext::remote_with_state(address, state)
+        .await
+        .map_err(|e| DataFusionError::Execution(format!("connect: {e}")))?;
+    register_parquet_tables(&ctx, TABLES, opt.path.as_str(), opt.debug)
+        .await
+        .map_err(|e| DataFusionError::Execution(format!("register-tables: {e}")))?;
+
+    // Run the query on the cluster, capturing the answer statement's result.
+    let answer_idx = answer_statement_index(&sqls);
+    let start = Instant::now();
+    let mut batches = vec![];
+    let mut run_err = None;
+    for (idx, sql) in sqls.iter().enumerate() {
+        if opt.debug {
+            println!("Query {query} SQL:\n{sql}");
+        }
+        match ctx.sql(sql).await {
+            Ok(df) => match df.collect().await {
+                Ok(collected) => {
+                    if idx == answer_idx {
+                        batches = collected;
+                    }
+                }
+                Err(e) => {
+                    run_err = Some(format!("collect: {e}"));
+                    break;
+                }
+            },
+            Err(e) => {
+                run_err = Some(format!("plan: {e}"));
+                break;
+            }
+        }
+    }
+    let elapsed = start.elapsed().as_secs_f64();
+
+    if let Some(e) = run_err {
+        println!("Query {query} FAILED to run in {elapsed:.3}s: {e}");
+        return Err(DataFusionError::Execution(e));
+    }
+    let row_count: usize = batches.iter().map(|b| b.num_rows()).sum();
+    println!("Query {query} took {elapsed:.3}s and returned {row_count} rows");
+
+    if let Some(oracle_ctx) = oracle_ctx {
+        let expected = execute_query_capturing_answer(oracle_ctx, &sqls, opt.debug)
+            .await
+            .map_err(|e| DataFusionError::Execution(format!("oracle: {e}")))?;
+        match compare_results(&expected, &batches) {
+            Ok(()) => println!("Query {query} verified against DataFusion: OK"),
+            Err(e) => {
+                println!("Query {query} VERIFY MISMATCH: {e}");
+                return Err(DataFusionError::Execution(format!("verify: {e}")));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::init();
+    let opt = Opt::from_args();
+    let address = format!("df://{}:{}", opt.host, opt.port);
+
+    // Oracle context (single-process DataFusion), built once when verifying.
+    // Not per-query, so a failure here is genuinely fatal to the whole run.
+    let oracle_ctx = if opt.verify {
+        let cfg = SessionConfig::new()
+            .with_target_partitions(opt.partitions)
+            .with_batch_size(opt.batch_size);
+        let ctx = SessionContext::new_with_config(cfg);
+        register_parquet_tables(&ctx, TABLES, opt.path.as_str(), opt.debug).await?;
+        Some(ctx)
+    } else {
+        None
+    };
+
+    let mut failures: Vec<(usize, String)> = vec![];
+
+    for query in selected_queries(opt.query, SKIP) {
+        if let Err(e) = run_one_query(&opt, &address, oracle_ctx.as_ref(), query).await {
+            eprintln!("Query {query} FAILED: {e}");
+            failures.push((query, e.to_string()));
+        }
+    }
+
+    if !failures.is_empty() {
+        eprintln!("\n{} query failure(s):", failures.len());
+        for (q, e) in &failures {
+            eprintln!("  q{q}: {e}");
+        }
+        return Err(DataFusionError::Execution(format!(
+            "{} TPC-DS query failure(s)",
+            failures.len()
+        )));
+    }
+    println!("\nAll selected TPC-DS queries passed.");
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn loader_strips_comments_and_splits_statements() {
+        let sqls = split_statements("-- c\nselect 1;\n-- d\nselect 2;\n");
+        assert_eq!(sqls, vec!["select 1".to_string(), "select 2".to_string()]);
+    }
+
+    #[test]
+    fn selected_queries_excludes_skiplist() {
+        // With no explicit --query, we run 1..=99 minus the skip list. Use a
+        // synthetic skip list so this assertion is independent of the real
+        // SKIP constant's contents.
+        let skip: &[(usize, &str)] = &[(5, "x"), (42, "y")];
+        let selected = selected_queries(None, skip);
+        assert_eq!(selected.len(), 99 - skip.len());
+        for (id, _) in skip {
+            assert!(!selected.contains(id), "skip {id} must be excluded");
+        }
+    }
+
+    #[test]
+    fn explicit_query_overrides_skiplist() {
+        let skip: &[(usize, &str)] = &[(1, "should be overridden")];
+        assert_eq!(selected_queries(Some(1), skip), vec![1]);
+    }
+}
diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs
index 398c9aa45c..5c561b171f 100644
--- a/benchmarks/src/bin/tpch.rs
+++ b/benchmarks/src/bin/tpch.rs
@@ -19,9 +19,13 @@
 
 use ballista::extension::SessionConfigExt;
 use ballista::prelude::SessionContextExt;
-use datafusion::arrow::array::*;
+use ballista_benchmarks::{
+    answer_statement_index, compare_results, execute_query_capturing_answer, find_path,
+};
+use ballista_core::object_store::{
+    session_config_with_s3_support, session_state_with_s3_support,
+};
 use datafusion::arrow::datatypes::SchemaBuilder;
-use datafusion::arrow::util::display::array_value_to_string;
 use datafusion::common::{DEFAULT_CSV_EXTENSION, DEFAULT_PARQUET_EXTENSION};
 use datafusion::datasource::listing::ListingTableUrl;
 use datafusion::datasource::{MemTable, TableProvider};
@@ -92,9 +96,11 @@ struct BallistaBenchmarkOpt {
     #[structopt(short = "s", long = "batch-size", default_value = "8192")]
     batch_size: usize,
 
-    /// Path to data files
-    #[structopt(parse(from_os_str), required = true, short = "p", long = "path")]
-    path: PathBuf,
+    /// Path to data files. May be a local path or an object-store URL such as
+    /// `s3://bucket/prefix` (credentials/endpoint via AWS_* env vars or
+    /// `-c s3.*` config overrides).
+    #[structopt(required = true, short = "p", long = "path")]
+    path: String,
 
     /// File format: `csv` or `parquet`
     #[structopt(short = "f", long = "format", default_value = "csv")]
@@ -125,6 +131,11 @@ struct BallistaBenchmarkOpt {
     /// -c datafusion.execution.target_partitions=16
     #[structopt(short = "c", long = "config", number_of_values = 1)]
     config_overrides: Vec<String>,
+
+    /// Verify each Ballista result against single-process DataFusion over the
+    /// same data.
+    #[structopt(long = "verify")]
+    verify: bool,
 }
 
 #[derive(Debug, StructOpt, Clone)]
@@ -296,19 +307,19 @@ async fn benchmark_datafusion(opt: DataFusionBenchmarkOpt) -> Result<Vec<RecordB
     let ctx = SessionContext::new_with_config(config);
 
     // register tables
-    for table in TABLES {
-        let table_provider = {
-            let mut session_state = ctx.state();
-            get_table(
-                &mut session_state,
-                opt.path.to_str().unwrap(),
-                table,
-                opt.file_format.as_str(),
-                opt.partitions,
-            )
-            .await?
-        };
-        if opt.mem_table {
+    if opt.mem_table {
+        for table in TABLES {
+            let table_provider = {
+                let mut session_state = ctx.state();
+                get_table(
+                    &mut session_state,
+                    opt.path.to_str().unwrap(),
+                    table,
+                    opt.file_format.as_str(),
+                    opt.partitions,
+                )
+                .await?
+            };
             println!("Loading table '{table}' into memory");
             let start = Instant::now();
             let memtable =
@@ -320,9 +331,15 @@ async fn benchmark_datafusion(opt: DataFusionBenchmarkOpt) -> Result<Vec<RecordB
                 start.elapsed().as_millis()
             );
             ctx.register_table(*table, Arc::new(memtable))?;
-        } else {
-            ctx.register_table(*table, table_provider)?;
         }
+    } else {
+        register_datafusion_tables(
+            &ctx,
+            opt.path.to_str().unwrap(),
+            opt.file_format.as_str(),
+            opt.partitions,
+        )
+        .await?;
     }
 
     // Determine which queries to run
@@ -347,14 +364,9 @@ async fn benchmark_datafusion(opt: DataFusionBenchmarkOpt) -> Result<Vec<RecordB
         for i in 0..opt.iterations {
             let start = Instant::now();
             // Execute each SQL statement sequentially (required for queries like q15
-            // that create views and then reference them)
-            for sql in &sqls {
-                if opt.debug {
-                    println!("Executing: {sql}");
-                }
-                let df = ctx.sql(sql).await?;
-                result = df.collect().await?;
-            }
+            // that create views and then reference them), keeping the result of
+            // the answer statement, not a trailing DROP VIEW.
+            result = execute_query_capturing_answer(&ctx, &sqls, opt.debug).await?;
             let elapsed = start.elapsed().as_secs_f64();
             if opt.debug {
                 pretty::print_batches(&result)?;
@@ -410,13 +422,30 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> {
         .map(|q| vec![q])
         .unwrap_or_else(|| (1..=22).collect());
 
+    let oracle_ctx = if opt.verify {
+        let cfg = SessionConfig::new()
+            .with_target_partitions(opt.partitions)
+            .with_batch_size(opt.batch_size);
+        let ctx = SessionContext::new_with_config(cfg);
+        register_datafusion_tables(
+            &ctx,
+            opt.path.as_str(),
+            opt.file_format.as_str(),
+            opt.partitions,
+        )
+        .await?;
+        Some(ctx)
+    } else {
+        None
+    };
+
     let mut benchmark_run = BenchmarkRun::new();
     let mut total_elapsed = 0.0;
 
     for query in query_numbers {
         let mut query_run = QueryRun::new(query);
 
-        let mut config = SessionConfig::new_with_ballista()
+        let mut config = session_config_with_s3_support()
             .with_target_partitions(opt.partitions)
             .with_ballista_job_name(&format!("Query derived from TPC-H q{}", query))
             .with_batch_size(opt.batch_size)
@@ -436,14 +465,11 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> {
             }
         }
 
-        let state = SessionStateBuilder::new()
-            .with_default_features()
-            .with_config(config)
-            .build();
+        let state = session_state_with_s3_support(config)?;
         let ctx = SessionContext::remote_with_state(&address, state).await?;
 
         // register tables with Ballista context
-        let path = opt.path.to_str().unwrap();
+        let path = opt.path.as_str();
         let file_format = opt.file_format.as_str();
 
         register_tables(path, file_format, &ctx, opt.debug).await?;
@@ -456,9 +482,10 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> {
             println!("Running benchmark with query {}:\n {:?}", query, sqls);
         }
         let mut batches = vec![];
+        let answer_idx = answer_statement_index(&sqls);
         for i in 0..opt.iterations {
             let start = Instant::now();
-            for sql in &sqls {
+            for (idx, sql) in sqls.iter().enumerate() {
                 let df = ctx
                     .sql(sql)
                     .await
@@ -468,11 +495,14 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> {
                 if opt.debug {
                     println!("=== Optimized logical plan ===\n{plan:?}\n");
                 }
-                batches = df
+                let collected = df
                     .collect()
                     .await
                     .map_err(|e| DataFusionError::Plan(format!("{e:?}")))
                     .unwrap();
+                if idx == answer_idx {
+                    batches = collected;
+                }
             }
             let elapsed = start.elapsed().as_secs_f64();
             secs.push(elapsed);
@@ -495,10 +525,21 @@ async fn benchmark_ballista(opt: BallistaBenchmarkOpt) -> Result<()> {
 
             if let Some(expected_results_path) = opt.expected_results.as_ref() {
                 let expected = get_expected_results(query, expected_results_path).await?;
-                assert_expected_results(&expected, &batches)
+                compare_results(&expected, &batches)?;
             }
         }
 
+        if let Some(oracle_ctx) = &oracle_ctx {
+            let expected =
+                execute_query_capturing_answer(oracle_ctx, &sqls, opt.debug).await?;
+            compare_results(&expected, &batches).map_err(|e| {
+                DataFusionError::Execution(format!(
+                    "query {query} verification failed: {e}"
+                ))
+            })?;
+            println!("Query {query} verified against DataFusion: OK");
+        }
+
         if opt.iterations > 1 {
             let avg = secs.iter().sum::<f64>() / secs.len() as f64;
             println!("Query {} avg time: {:.3} s", query, avg);
@@ -574,7 +615,7 @@ async fn loadtest_ballista(opt: BallistaLoadtestOpt) -> Result<()> {
         .split(',')
         .map(|s| s.parse().unwrap())
         .collect();
-    println!("query list: {:?} ", &query_list);
+    println!("query list: {:?} ", query_list);
 
     let total = Instant::now();
     let mut futures = vec![];
@@ -596,7 +637,7 @@ async fn loadtest_ballista(opt: BallistaLoadtestOpt) -> Result<()> {
                         .unwrap();
                 println!(
                     "Client {} Round {} Query {} started",
-                    &client_id, &i, query_id
+                    client_id, i, query_id
                 );
                 let start = Instant::now();
                 let df = client
@@ -612,7 +653,7 @@ async fn loadtest_ballista(opt: BallistaLoadtestOpt) -> Result<()> {
                 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
                 println!(
                     "Client {} Round {} Query {} took {:.1} ms ",
-                    &client_id, &i, query_id, elapsed
+                    client_id, i, query_id, elapsed
                 );
                 if opt.debug && !batches.is_empty() {
                     pretty::print_batches(&batches).unwrap();
@@ -642,6 +683,24 @@ fn get_query_sql_by_path(query: usize, mut sql_path: String) -> Result<String> {
     }
 }
 
+/// Registers all TPC-H tables on a single-process DataFusion context (the
+/// correctness oracle for `--verify`).
+async fn register_datafusion_tables(
+    ctx: &SessionContext,
+    path: &str,
+    file_format: &str,
+    partitions: usize,
+) -> Result<()> {
+    for table in TABLES {
+        let table_provider = {
+            let mut session_state = ctx.state();
+            get_table(&mut session_state, path, table, file_format, partitions).await?
+        };
+        ctx.register_table(*table, table_provider)?;
+    }
+    Ok(())
+}
+
 async fn register_tables(
     path: &str,
     file_format: &str,
@@ -702,20 +761,6 @@ async fn register_tables(
     Ok(())
 }
 
-fn find_path(path: &str, table: &str, ext: &str) -> Result<String> {
-    let path1 = format!("{path}/{table}.{ext}");
-    let path2 = format!("{path}/{table}");
-    if Path::new(&path1).exists() {
-        Ok(path1)
-    } else if Path::new(&path2).exists() {
-        Ok(path2)
-    } else {
-        Err(DataFusionError::Plan(format!(
-            "Could not find {ext} files at {path1} or {path2}"
-        )))
-    }
-}
-
 /// Get the SQL statements from the specified query file
 fn get_query_sql(query: usize) -> Result<Vec<String>> {
     if query > 0 && query < 23 {
@@ -843,7 +888,7 @@ async fn convert_tbl(opt: ConvertOpt) -> Result<()> {
 
         println!(
             "Converting '{}' to {} files in directory '{}'",
-            &input_path, &opt.file_format, &output_path
+            input_path, opt.file_format, output_path
         );
         match opt.file_format.as_str() {
             "csv" => ctx.write_csv(csv, output_path).await?,
@@ -1116,27 +1161,6 @@ impl BenchmarkRun {
     }
 }
 
-/// Compare actual results against expected results at scale factor 1
-fn assert_expected_results(expected: &[RecordBatch], actual: &[RecordBatch]) {
-    // assert schema equality without comparing nullable values
-    assert_eq!(
-        nullable_schema(expected[0].schema()),
-        nullable_schema(actual[0].schema())
-    );
-
-    // convert both datasets to Vec<Vec<String>> for simple comparison
-    let expected_vec = result_vec(expected);
-    let actual_vec = result_vec(actual);
-
-    // basic result comparison
-    assert_eq!(expected_vec.len(), actual_vec.len());
-
-    // compare each row. this works as all TPC-H queries have deterministically ordered results
-    for i in 0..actual_vec.len() {
-        assert_eq!(expected_vec[i], actual_vec[i]);
-    }
-}
-
 /// Get the expected answer for a specific query at scale factor 1
 async fn get_expected_results(n: usize, path: &str) -> Result<Vec<RecordBatch>> {
     let ctx = SessionContext::new();
@@ -1179,37 +1203,6 @@ async fn get_expected_results(n: usize, path: &str) -> Result<Vec<RecordBatch>>
     df.collect().await
 }
 
-// convert the schema to the same but with all columns set to nullable=true.
-// this allows direct schema comparison ignoring nullable.
-fn nullable_schema(schema: Arc<Schema>) -> Schema {
-    Schema::new(
-        schema
-            .fields()
-            .iter()
-            .map(|field| {
-                Field::new(Field::name(field), Field::data_type(field).to_owned(), true)
-            })
-            .collect::<Vec<Field>>(),
-    )
-}
-
-/// Converts the results into a 2d array of strings, `result[row][column]`
-/// Special cases nulls to NULL for testing
-fn result_vec(results: &[RecordBatch]) -> Vec<Vec<String>> {
-    let mut result = vec![];
-    for batch in results {
-        for row_index in 0..batch.num_rows() {
-            let row_vec = batch
-                .columns()
-                .iter()
-                .map(|column| col_str(column, row_index))
-                .collect();
-            result.push(row_vec);
-        }
-    }
-    result
-}
-
 fn get_answer_schema(n: usize) -> Schema {
     match n {
         1 => Schema::new(vec![
@@ -1376,30 +1369,6 @@ fn string_schema(schema: Schema) -> Schema {
     )
 }
 
-/// Specialised String representation
-fn col_str(column: &ArrayRef, row_index: usize) -> String {
-    if column.is_null(row_index) {
-        return "NULL".to_string();
-    }
-
-    // Special case ListArray as there is no pretty print support for it yet
-    if let DataType::FixedSizeList(_, n) = column.data_type() {
-        let array = column
-            .as_any()
-            .downcast_ref::<FixedSizeListArray>()
-            .unwrap()
-            .value(row_index);
-
-        let mut r = Vec::with_capacity(*n as usize);
-        for i in 0..*n {
-            r.push(col_str(&array, i as usize));
-        }
-        return format!("[{}]", r.join(","));
-    }
-
-    array_value_to_string(column, row_index).unwrap()
-}
-
 #[derive(Debug, Serialize)]
 struct QueryResult {
     elapsed: f64,
@@ -1757,7 +1726,7 @@ mod tests {
             let expected_schema = get_answer_schema(n);
             let normalized = normalize_for_verification(actual, expected_schema).await?;
 
-            assert_expected_results(&expected, &normalized)
+            compare_results(&expected, &normalized)?;
         } else {
             println!("TPCH_DATA environment variable not set, skipping test");
         }
diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs
new file mode 100644
index 0000000000..2f05647dee
--- /dev/null
+++ b/benchmarks/src/lib.rs
@@ -0,0 +1,368 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared helpers for the TPC-H and TPC-DS benchmark/correctness binaries.
+//!
+//! These are benchmark-agnostic: result comparison against a DataFusion oracle,
+//! path resolution, answer-statement selection, and Parquet table registration.
+
+use datafusion::arrow::array::*;
+use datafusion::arrow::datatypes::{DataType, Schema};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::arrow::util::display::array_value_to_string;
+use datafusion::error::{DataFusionError, Result};
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use std::path::Path;
+
+/// Maximum relative-or-absolute difference tolerated between floating-point
+/// cells. Distributed (Ballista) and single-process (DataFusion) execution
+/// aggregate in different orders, and float `sum` is non-associative, so ratio
+/// queries can differ in their last digits.
+const FLOAT_TOLERANCE: f64 = 1e-6;
+
+pub enum Cell {
+    Null,
+    Float(f64),
+    Text(String),
+}
+
+impl std::fmt::Display for Cell {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Cell::Null => f.write_str("NULL"),
+            Cell::Float(x) => write!(f, "{x}"),
+            Cell::Text(s) => f.write_str(s),
+        }
+    }
+}
+
+pub fn cell_at(column: &ArrayRef, row: usize) -> Cell {
+    if column.is_null(row) {
+        return Cell::Null;
+    }
+    match column.data_type() {
+        DataType::Float64 => Cell::Float(
+            column
+                .as_any()
+                .downcast_ref::<Float64Array>()
+                .unwrap()
+                .value(row),
+        ),
+        DataType::Float32 => Cell::Float(
+            column
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .unwrap()
+                .value(row) as f64,
+        ),
+        _ => Cell::Text(col_str(column, row)),
+    }
+}
+
+pub fn rows_as_cells(batches: &[RecordBatch]) -> Vec<Vec<Cell>> {
+    let mut rows = vec![];
+    for batch in batches {
+        for row in 0..batch.num_rows() {
+            rows.push(batch.columns().iter().map(|c| cell_at(c, row)).collect());
+        }
+    }
+    rows
+}
+
+pub fn floats_close(a: f64, b: f64) -> bool {
+    if a == b {
+        return true;
+    }
+    (a - b).abs() <= FLOAT_TOLERANCE * a.abs().max(b.abs()).max(1.0)
+}
+
+pub fn cells_equal(a: &Cell, b: &Cell) -> bool {
+    match (a, b) {
+        (Cell::Null, Cell::Null) => true,
+        (Cell::Float(x), Cell::Float(y)) => floats_close(*x, *y),
+        (Cell::Text(x), Cell::Text(y)) => x == y,
+        _ => false,
+    }
+}
+
+/// Canonicalizes Arrow string/binary representation variants so that physical
+/// differences (e.g. `Utf8` vs `Utf8View`) are not treated as result
+/// differences: single-process DataFusion may infer Parquet strings as
+/// `Utf8View` while Ballista produces `Utf8`, but both stringify identically.
+pub fn canonical_type(dt: &DataType) -> DataType {
+    match dt {
+        DataType::Utf8View | DataType::LargeUtf8 => DataType::Utf8,
+        DataType::BinaryView | DataType::LargeBinary => DataType::Binary,
+        _ => dt.clone(),
+    }
+}
+
+/// Schema reduced to (name, canonical type) pairs, ignoring nullability and
+/// string/binary representation, for result comparison.
+pub fn comparable_schema(schema: &Schema) -> Vec<(String, DataType)> {
+    schema
+        .fields()
+        .iter()
+        .map(|f| (f.name().clone(), canonical_type(f.data_type())))
+        .collect()
+}
+
+/// Compares two result sets, tolerating tiny floating-point differences.
+/// Returns an error describing the first mismatch instead of panicking, so the
+/// benchmark binary exits non-zero with a useful message.
+pub fn compare_results(expected: &[RecordBatch], actual: &[RecordBatch]) -> Result<()> {
+    let expected_rows = rows_as_cells(expected);
+    let actual_rows = rows_as_cells(actual);
+
+    if expected_rows.len() != actual_rows.len() {
+        return Err(DataFusionError::Execution(format!(
+            "result mismatch: expected {} rows, got {} rows",
+            expected_rows.len(),
+            actual_rows.len()
+        )));
+    }
+
+    if let (Some(e), Some(a)) = (expected.first(), actual.first()) {
+        let e_schema = comparable_schema(&e.schema());
+        let a_schema = comparable_schema(&a.schema());
+        if e_schema != a_schema {
+            return Err(DataFusionError::Execution(format!(
+                "schema mismatch:\n expected: {e_schema:?}\n actual:   {a_schema:?}"
+            )));
+        }
+    }
+
+    for (i, (erow, arow)) in expected_rows.iter().zip(actual_rows.iter()).enumerate() {
+        for (j, (ecell, acell)) in erow.iter().zip(arow.iter()).enumerate() {
+            if !cells_equal(ecell, acell) {
+                return Err(DataFusionError::Execution(format!(
+                    "result mismatch at row {i}, column {j}: expected `{ecell}`, got `{acell}`"
+                )));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Specialised String representation
+pub fn col_str(column: &ArrayRef, row_index: usize) -> String {
+    if column.is_null(row_index) {
+        return "NULL".to_string();
+    }
+
+    // Special case ListArray as there is no pretty print support for it yet
+    if let DataType::FixedSizeList(_, n) = column.data_type() {
+        let array = column
+            .as_any()
+            .downcast_ref::<FixedSizeListArray>()
+            .unwrap()
+            .value(row_index);
+
+        let mut r = Vec::with_capacity(*n as usize);
+        for i in 0..*n {
+            r.push(col_str(&array, i as usize));
+        }
+        return format!("[{}]", r.join(","));
+    }
+
+    array_value_to_string(column, row_index).unwrap()
+}
+
+pub fn find_path(path: &str, table: &str, ext: &str) -> Result<String> {
+    // Object-store URLs (e.g. `s3://bucket/prefix`) cannot be probed with the
+    // local filesystem, so register the per-table directory under the base URL
+    // directly. The trailing slash marks it as a directory so the listing table
+    // enumerates the Parquet files inside (e.g. `s3://bucket/prefix/lineitem/`).
+    if path.contains("://") {
+        return Ok(format!("{path}/{table}/"));
+    }
+
+    let path1 = format!("{path}/{table}.{ext}");
+    let path2 = format!("{path}/{table}");
+    if Path::new(&path1).exists() {
+        Ok(path1)
+    } else if Path::new(&path2).exists() {
+        Ok(path2)
+    } else {
+        Err(DataFusionError::Plan(format!(
+            "Could not find {ext} files at {path1} or {path2}"
+        )))
+    }
+}
+
+/// Index of the statement whose result is the query answer: the last `SELECT`
+/// or `WITH` statement, or the last statement if none qualifies. TPC-H query
+/// files may wrap the answer in setup/teardown statements (e.g. q15 creates and
+/// drops a view); only the query statement's result is the answer.
+pub fn answer_statement_index(statements: &[String]) -> usize {
+    statements
+        .iter()
+        .rposition(|s| {
+            let head = s.trim_start().to_ascii_lowercase();
+            head.starts_with("select") || head.starts_with("with")
+        })
+        .unwrap_or_else(|| statements.len().saturating_sub(1))
+}
+
+/// Executes all statements of a query in order and returns the batches of the
+/// answer statement (see `answer_statement_index`).
+pub async fn execute_query_capturing_answer(
+    ctx: &SessionContext,
+    statements: &[String],
+    debug: bool,
+) -> Result<Vec<RecordBatch>> {
+    let answer_idx = answer_statement_index(statements);
+    let mut answer = vec![];
+    for (idx, sql) in statements.iter().enumerate() {
+        if debug {
+            println!("Executing: {sql}");
+        }
+        let df = ctx.sql(sql).await?;
+        let collected = df.collect().await?;
+        if idx == answer_idx {
+            answer = collected;
+        }
+    }
+    Ok(answer)
+}
+
+/// Register each named table as a Parquet source under `path`, inferring the
+/// schema from the files. Works for both a single `<table>.parquet` file and a
+/// partitioned `<table>/` directory (see `find_path`).
+pub async fn register_parquet_tables(
+    ctx: &SessionContext,
+    tables: &[&str],
+    path: &str,
+    debug: bool,
+) -> Result<()> {
+    for &table in tables {
+        let table_path = find_path(path, table, "parquet")?;
+        if debug {
+            println!("Registering table '{table}' from Parquet at {table_path}");
+        }
+        ctx.register_parquet(table, &table_path, ParquetReadOptions::default())
+            .await
+            .map_err(|e| DataFusionError::Plan(format!("{e:?}")))?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::datatypes::Field;
+    use std::sync::Arc;
+
+    fn f64_batch(values: Vec<f64>) -> RecordBatch {
+        let schema =
+            Arc::new(Schema::new(vec![Field::new("x", DataType::Float64, false)]));
+        RecordBatch::try_new(schema, vec![Arc::new(Float64Array::from(values))]).unwrap()
+    }
+
+    fn i64_batch(values: Vec<i64>) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![Field::new("n", DataType::Int64, false)]));
+        RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(values))]).unwrap()
+    }
+
+    #[test]
+    fn compare_results_equal_ok() {
+        assert!(
+            compare_results(&[i64_batch(vec![1, 2])], &[i64_batch(vec![1, 2])]).is_ok()
+        );
+    }
+
+    #[test]
+    fn compare_results_row_count_mismatch() {
+        let err = compare_results(&[i64_batch(vec![1])], &[i64_batch(vec![1, 2])])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("expected 1 rows, got 2 rows"), "{err}");
+    }
+
+    #[test]
+    fn compare_results_value_mismatch() {
+        let err = compare_results(&[i64_batch(vec![1])], &[i64_batch(vec![2])])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("row 0, column 0"), "{err}");
+    }
+
+    #[test]
+    fn compare_results_floats_within_tolerance_ok() {
+        // differ only beyond the tolerance's significant digits
+        assert!(
+            compare_results(
+                &[f64_batch(vec![123.4567890])],
+                &[f64_batch(vec![123.4567891])]
+            )
+            .is_ok()
+        );
+    }
+
+    #[test]
+    fn compare_results_floats_outside_tolerance_err() {
+        assert!(
+            compare_results(&[f64_batch(vec![100.0])], &[f64_batch(vec![100.5])])
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn answer_statement_index_picks_select_not_drop() {
+        let stmts = vec![
+            "create view revenue0 as select 1".to_string(),
+            "select * from revenue0 order by a".to_string(),
+            "drop view revenue0".to_string(),
+        ];
+        assert_eq!(answer_statement_index(&stmts), 1);
+    }
+
+    #[test]
+    fn answer_statement_index_single_select() {
+        let stmts = vec!["select 1".to_string()];
+        assert_eq!(answer_statement_index(&stmts), 0);
+    }
+
+    #[test]
+    fn answer_statement_index_with_cte() {
+        let stmts = vec!["WITH t AS (select 1) select * from t".to_string()];
+        assert_eq!(answer_statement_index(&stmts), 0);
+    }
+
+    #[test]
+    fn compare_results_ignores_utf8view_vs_utf8() {
+        let view_schema = Arc::new(Schema::new(vec![Field::new(
+            "s",
+            DataType::Utf8View,
+            false,
+        )]));
+        let utf8_schema =
+            Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
+        let a = RecordBatch::try_new(
+            view_schema,
+            vec![Arc::new(StringViewArray::from(vec!["x", "y"]))],
+        )
+        .unwrap();
+        let b = RecordBatch::try_new(
+            utf8_schema,
+            vec![Arc::new(StringArray::from(vec!["x", "y"]))],
+        )
+        .unwrap();
+        assert!(compare_results(&[a], &[b]).is_ok());
+    }
+}
diff --git a/benchmarks/tpcds-gen.sh b/benchmarks/tpcds-gen.sh
new file mode 100755
index 0000000000..48098e346a
--- /dev/null
+++ b/benchmarks/tpcds-gen.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generate TPC-DS Parquet using the unified tpcgen-cli
+# (https://github.com/clflushopt/tpchgen-rs). TPC-DS is not yet on crates.io,
+# so tpcgen-cli is installed from git, pinned to a rev. The tpcds subcommand
+# writes one Parquet file per table and has no --parts option.
+#
+# Env overrides:
+#   SCALE_FACTOR   TPC-DS scale factor (default: 1)
+#   OUTPUT_DIR     Where data is written (default: ./data-tpcds)
+#   TPCGEN_REV     tpcgen-cli git rev to install (default: pinned below)
+set -euo pipefail
+
+SCALE_FACTOR="${SCALE_FACTOR:-1}"
+TPCGEN_REV="${TPCGEN_REV:-573fd0018dd1b41e52880162f615e9903418c397}"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT_DIR="${OUTPUT_DIR:-${script_dir}/data-tpcds}"
+
+if ! command -v tpcgen-cli >/dev/null 2>&1; then
+  echo "tpcgen-cli not found; installing from git (rev ${TPCGEN_REV})..."
+  cargo install --git https://github.com/clflushopt/tpchgen-rs --rev "${TPCGEN_REV}" --locked tpcgen-cli
+fi
+
+echo "Generating TPC-DS SF${SCALE_FACTOR} as Parquet into ${OUTPUT_DIR}"
+mkdir -p "${OUTPUT_DIR}"
+rm -rf "${OUTPUT_DIR:?}/"*
+
+tpcgen-cli tpcds parquet \
+  --scale-factor "${SCALE_FACTOR}" \
+  --output-dir "${OUTPUT_DIR}"
+
+echo "Done. Files written to ${OUTPUT_DIR}"
diff --git a/ci/scripts/rust_proto_check.sh b/ci/scripts/rust_proto_check.sh
new file mode 100755
index 0000000000..4347126871
--- /dev/null
+++ b/ci/scripts/rust_proto_check.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fail if the vendored DataFusion proto files drift from the datafusion-proto /
+# datafusion-proto-common versions pinned in Cargo.lock. See the script header of
+# dev/update_datafusion_proto.py for why these files are vendored.
+set -ex
+python3 dev/update_datafusion_proto.py --check
diff --git a/dev/release/README.md b/dev/release/README.md
index 2b43e77b55..3f34ba10e5 100644
--- a/dev/release/README.md
+++ b/dev/release/README.md
@@ -88,48 +88,47 @@ GitHub dependency.
 
 ### Change Log
 
-We maintain a `CHANGELOG.md` so our users know what has been changed between releases.
+We maintain per-release changelogs under
+[`docs/source/changelog/`](../../docs/source/changelog/). They are surfaced
+in the Sphinx site through `docs/source/changelog/index.md`.
 
-You will need a GitHub Personal Access Token for the following steps. Follow
+You will need a GitHub Personal Access Token for the following steps.
+Follow
 [these instructions](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
-to generate one if you do not already have one.
-
-The changelog is generated using a Python script. There is a dependency on `PyGitHub`, which can be installed using pip:
+to generate one if you do not already have one. The changelog script
+depends on `PyGitHub`:
 
 ```bash
 pip3 install PyGitHub
 ```
 
-Run the following command to generate the changelog content.
+Run the generator from the repo root, pointing at the previous release
+tag and the new release tag (or `HEAD`):
 
 ```bash
-$ GITHUB_TOKEN=<TOKEN> ./dev/release/generate-changelog.py apache/datafusion-ballista 0.11.0 HEAD > 0.12.0.md
+GITHUB_TOKEN=<TOKEN> ./dev/release/generate-changelog.py \
+    52.0.0 HEAD 53.0.0 \
+    > docs/source/changelog/53.0.0.md
 ```
 
-This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for
-titles starting with `feat:`, `fix:`, or `docs:` . The script will produce output similar to:
-
-```
-Fetching list of commits between 0.11.0 and HEAD
-Fetching pull requests
-Categorizing pull requests
-Generating changelog content
-```
+The script writes a fully-formed file: ASF header, version title, commit /
+contributor summary, categorized PR list, and a Credits section. The only
+remaining manual step is to prepend the new version to the toctree at the
+top of `docs/source/changelog/index.md`:
 
-This process is not fully automated, so there are some additional manual steps:
-
-- Add the ASF header to the generated file
-- Add the following content (copy from the previous version's changelog and update as appropriate:
-
-```
-## [0.12.0](https://github.com/apache/datafusion-ballista/tree/0.12.0) (2024-01-14)
+````
+```{toctree}
+:maxdepth: 1
 
-[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.11.0...0.12.0)
+53.0.0
+52.0.0
+...
 ```
+````
 
-Send a PR to get these changes merged into the release branch (e.g. `branch-0.12`). If new commits that could change the
-change log content landed in the release branch before you could merge the PR, you need to rerun the changelog update
-script to regenerate the changelog and update the PR accordingly.
+Send a PR with the new file and the updated index to the release branch
+(e.g. `branch-53`). If new commits land in the release branch before
+merge, rerun the generator to refresh the file.
 
 ## Prepare release candidate artifacts
 
diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh
index f1f0bd2972..63d9f4fe4d 100755
--- a/dev/release/create-tarball.sh
+++ b/dev/release/create-tarball.sh
@@ -83,6 +83,7 @@ I would like to propose a release of Apache DataFusion Ballista version ${versio
 This release candidate is based on commit: ${release_hash} [1]
 The proposed release tarball and signatures are hosted at [2].
 The changelog is located at [3].
+The Python wheels built from this release candidate are published to TestPyPI at [4].
 
 Please download, verify checksums and signatures, run the unit tests, and vote
 on the release. The vote will be open for at least 72 hours.
@@ -101,6 +102,7 @@ Here is my vote: +1
 [1]: https://github.com/apache/datafusion-ballista/tree/${release_hash}
 [2]: ${url}
 [3]: https://github.com/apache/datafusion-ballista/blob/${release_hash}/CHANGELOG.md
+[4]: https://test.pypi.org/project/ballista/${version}/
 MAIL
 echo "---------------------------------------------------------"
 
diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py
index 3a33b331af..613c651d07 100755
--- a/dev/release/generate-changelog.py
+++ b/dev/release/generate-changelog.py
@@ -20,7 +20,7 @@
 from github import Github
 import os
 import re
-
+import subprocess
 
 def print_pulls(repo_name, title, pulls):
     if len(pulls)  > 0:
@@ -32,7 +32,7 @@ def print_pulls(repo_name, title, pulls):
         print()
 
 
-def generate_changelog(repo, repo_name, tag1, tag2):
+def generate_changelog(repo, repo_name, tag1, tag2, version):
 
     # get a list of commits between two tags
     print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr)
@@ -52,49 +52,113 @@ def generate_changelog(repo, repo_name, tag1, tag2):
                 all_pulls.append((pull, commit))
 
     # we split the pulls into categories
-    #TODO: make categories configurable
     breaking = []
     bugs = []
     docs = []
     enhancements = []
     performance = []
+    other = []
 
     # categorize the pull requests based on GitHub labels
     print("Categorizing pull requests", file=sys.stderr)
     for (pull, commit) in all_pulls:
 
+        labels = [label.name for label in pull.labels]
+
+        # skip Dependabot dependency-bump PRs (labeled 'auto-dependencies')
+        if 'auto-dependencies' in labels:
+            continue
+
         # see if PR title uses Conventional Commits
         cc_type = ''
         cc_scope = ''
         cc_breaking = ''
-        parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title)
+        parts = re.findall(r'^([a-zA-Z]+)(\([a-zA-Z0-9_-]+\))?(!)?:', pull.title)
         if len(parts) == 1:
             parts_tuple = parts[0]
             cc_type = parts_tuple[0] # fix, feat, docs, chore
             cc_scope = parts_tuple[1] # component within project
             cc_breaking = parts_tuple[2] == '!'
 
-        labels = [label.name for label in pull.labels]
-        #print(pull.number, labels, parts, file=sys.stderr)
         if 'api change' in labels or cc_breaking:
             breaking.append((pull, commit))
-        elif 'bug' in labels or cc_type == 'fix':
-            bugs.append((pull, commit))
         elif 'performance' in labels or cc_type == 'perf':
             performance.append((pull, commit))
+        elif 'bug' in labels or cc_type == 'fix':
+            bugs.append((pull, commit))
         elif 'enhancement' in labels or cc_type == 'feat':
             enhancements.append((pull, commit))
-        elif 'documentation' in labels or cc_type == 'docs':
+        elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc':
             docs.append((pull, commit))
+        else:
+            other.append((pull, commit))
 
     # produce the changelog content
     print("Generating changelog content", file=sys.stderr)
+
+    # ASF header
+    print("""<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->\n""")
+
+    print(f"# Apache DataFusion Ballista {version} Changelog\n")
+
+    # get the number of commits
+    commit_count = subprocess.check_output(["git", "rev-list", "--count", f"{tag1}..{tag2}"], text=True).strip()
+
+    # get number of contributors
+    shortlog_output = subprocess.check_output(["git", "shortlog", "-sn", f"{tag1}..{tag2}"], text=True)
+    contributor_count = len(shortlog_output.strip().splitlines())
+
+    print(f"This release consists of {commit_count} commits from {contributor_count} contributors. "
+          f"See credits at the end of this changelog for more information.\n")
+
     print_pulls(repo_name, "Breaking changes", breaking)
+    print_pulls(repo_name, "Fixed bugs", bugs)
     print_pulls(repo_name, "Performance related", performance)
     print_pulls(repo_name, "Implemented enhancements", enhancements)
-    print_pulls(repo_name, "Fixed bugs", bugs)
     print_pulls(repo_name, "Documentation updates", docs)
-    print_pulls(repo_name, "Merged pull requests", all_pulls)
+    print_pulls(repo_name, "Other", other)
+
+    # show code contributions
+    credits = shortlog_output.rstrip()
+
+    print("## Credits\n")
+    print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) "
+          "per contributor.\n")
+    print("```")
+    print(credits)
+    print("```\n")
+
+    print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing "
+          "PRs, and providing feedback on this release.\n")
+
+def resolve_ref(ref):
+    """Resolve a git ref (e.g. HEAD, branch name) to a full commit SHA."""
+    try:
+        return subprocess.check_output(
+            ["git", "rev-parse", ref], text=True
+        ).strip()
+    except subprocess.CalledProcessError:
+        # If it can't be resolved locally, return as-is (e.g. a remote tag).
+        # The GitHub API will attempt to resolve it.
+        print(f"Note: Could not resolve '{ref}' locally; passing to GitHub API as-is", file=sys.stderr)
+        return ref
 
 
 def cli(args=None):
@@ -103,16 +167,23 @@ def cli(args=None):
         args = sys.argv[1:]
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("project", help="The project name e.g. apache/datafusion-ballista")
-    parser.add_argument("tag1", help="The previous release tag")
-    parser.add_argument("tag2", help="The current release tag")
+    parser.add_argument("tag1", help="The previous commit or tag (e.g. 52.0.0)")
+    parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)")
+    parser.add_argument("version", help="The version number to include in the changelog")
     args = parser.parse_args()
 
+    # Resolve refs to SHAs so the GitHub API compares the same commits
+    # as the local git log. Without this, refs like HEAD get resolved by
+    # the GitHub API to the default branch instead of the current branch.
+    tag1 = resolve_ref(args.tag1)
+    tag2 = resolve_ref(args.tag2)
+
     token = os.getenv("GITHUB_TOKEN")
+    project = "apache/datafusion-ballista"
 
     g = Github(token)
-    repo = g.get_repo(args.project)
-    generate_changelog(repo, args.project, args.tag1, args.tag2)
+    repo = g.get_repo(project)
+    generate_changelog(repo, project, tag1, tag2, args.version)
 
 if __name__ == "__main__":
-    cli()
\ No newline at end of file
+    cli()
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 47a5ee2eb8..83d47821cc 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -34,6 +34,7 @@ requirements.txt
 .gitattributes
 benchmarks/queries/q*.sql
 ballista/scheduler/testdata/*
+ballista/scheduler/tests/tpch_plan_stability/approved/*
 **/yarn.lock
 python/requirements*.txt
 **/testdata/*
diff --git a/.github_changelog_generator b/dev/update-tpch-plan-stability.sh
old mode 100644
new mode 100755
similarity index 63%
rename from .github_changelog_generator
rename to dev/update-tpch-plan-stability.sh
index 45eef2f518..072e2a1b61
--- a/.github_changelog_generator
+++ b/dev/update-tpch-plan-stability.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -17,12 +17,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
-# some issues are just documentation
-add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]},"performance":{"prefix":"**Performance improvements:**","labels":["performance"]}}
-# uncomment to not show PRs. TBD if we shown them or not.
-#pull-requests=false
-# so that the component is shown associated with the issue
-issue-line-labels=sql
-exclude-labels=development-process,invalid
-breaking-labels=api change
+# Regenerate the approved TPC-H distributed plans for the plan-stability suite.
+# Run after an intended planner/plan-shape change, then review the diff.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability
+echo "Regenerated. Review changes under ballista/scheduler/tests/tpch_plan_stability/approved/"
diff --git a/dev/update_datafusion_proto.py b/dev/update_datafusion_proto.py
new file mode 100755
index 0000000000..bc627b5b8c
--- /dev/null
+++ b/dev/update_datafusion_proto.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Sync the vendored DataFusion proto files with the version pinned in Cargo.lock.
+#
+# `ballista/core/proto/ballista.proto` imports `datafusion.proto` and
+# `datafusion_common.proto`. Those two files are NOT compiled into Rust here --
+# `ballista/core/build.rs` maps their packages to the real `datafusion-proto` /
+# `datafusion-proto-common` crates via `extern_path`. They exist only so that
+# `protoc` can resolve `ballista.proto`'s imports at build time. Because of that
+# they must stay compatible with the DataFusion version Ballista depends on.
+#
+# The `datafusion-proto` / `datafusion-proto-common` crates ship their `.proto`
+# files, but expose no `links`/`DEP_*` path for a build script to consume, so we
+# vendor a copy. This script keeps that copy honest: it reads the exact crate
+# source resolved by `cargo metadata` and copies the `.proto` files in, rewriting
+# the one import path that differs from our flat layout.
+#
+# Usage:
+#   dev/update_datafusion_proto.py            # update the vendored files in place
+#   dev/update_datafusion_proto.py --check    # fail if vendored files are stale (CI)
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).parent.parent.absolute()
+VENDOR_DIR = REPO_ROOT / "ballista" / "core" / "proto"
+
+# vendored file name -> (crate name, file name within the crate's `proto/` dir)
+FILES = {
+    "datafusion.proto": ("datafusion-proto", "datafusion.proto"),
+    "datafusion_common.proto": ("datafusion-proto-common", "datafusion_common.proto"),
+}
+
+
+def crate_source_dirs():
+    """Map crate name -> source directory, as resolved in Cargo.lock."""
+    out = subprocess.run(
+        ["cargo", "metadata", "--format-version", "1", "--locked"],
+        cwd=REPO_ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    metadata = json.loads(out.stdout)
+    dirs = {}
+    for pkg in metadata["packages"]:
+        if pkg["name"] in ("datafusion-proto", "datafusion-proto-common"):
+            # manifest_path points at the crate's Cargo.toml; protos sit alongside it.
+            dirs[pkg["name"]] = (Path(pkg["manifest_path"]).parent, pkg["version"])
+    missing = {"datafusion-proto", "datafusion-proto-common"} - dirs.keys()
+    if missing:
+        sys.exit(f"could not resolve crate(s) from cargo metadata: {sorted(missing)}")
+    return dirs
+
+
+def rewrite(content: str) -> str:
+    """Rewrite crate-relative imports to our flat vendored layout.
+
+    Upstream imports the common proto by its repo path, e.g.
+    `import "datafusion/proto-common/proto/datafusion_common.proto";`
+    but we vendor it flat next to `datafusion.proto`.
+    """
+    return re.sub(
+        r'import\s+"[^"]*datafusion_common\.proto";',
+        'import "datafusion_common.proto";',
+        content,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="verify the vendored files are up to date; do not write. Exits 1 on drift.",
+    )
+    args = parser.parse_args()
+
+    dirs = crate_source_dirs()
+    stale = []
+    for vendor_name, (crate, src_name) in FILES.items():
+        crate_dir, version = dirs[crate]
+        src = crate_dir / "proto" / src_name
+        if not src.exists():
+            # Some releases don't publish the `.proto` file (e.g. datafusion-proto-common
+            # only started shipping it in v54). Nothing to sync against, so leave the
+            # vendored copy as-is rather than failing.
+            print(
+                f"note: {crate}@{version} does not ship proto/{src_name}; "
+                f"leaving vendored {vendor_name} unchanged",
+                file=sys.stderr,
+            )
+            continue
+        expected = rewrite(src.read_text())
+        dest = VENDOR_DIR / vendor_name
+
+        if args.check:
+            current = dest.read_text() if dest.exists() else ""
+            if current != expected:
+                stale.append(vendor_name)
+                diff = difflib.unified_diff(
+                    current.splitlines(keepends=True),
+                    expected.splitlines(keepends=True),
+                    fromfile=f"vendored/{vendor_name}",
+                    tofile=f"{crate}@{version}/proto/{src_name}",
+                )
+                sys.stderr.writelines(diff)
+        else:
+            dest.write_text(expected)
+            print(f"updated {dest.relative_to(REPO_ROOT)} from {crate}@{version}")
+
+    if args.check and stale:
+        sys.exit(
+            "\nVendored DataFusion proto file(s) out of date: "
+            + ", ".join(stale)
+            + "\nRun `dev/update_datafusion_proto.py` and commit the result."
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/dev/update_datafusion_versions.py b/dev/update_datafusion_versions.py
index aed2cee3a2..d21096ea13 100755
--- a/dev/update_datafusion_versions.py
+++ b/dev/update_datafusion_versions.py
@@ -133,6 +133,14 @@ def main():
     update_docs("README.md", new_version)
     update_pyproject_toml(os.path.join(repo_root, "python", "pyproject.toml"), new_version)
 
+    print(
+        '\nNext steps:\n'
+        '  1. Run `cargo update` (or `cargo build`) so Cargo.lock resolves the new '
+        'datafusion crates.\n'
+        '  2. Run `dev/update_datafusion_proto.py` to re-sync the vendored '
+        'DataFusion proto files, then commit the result.'
+    )
+
 
 if __name__ == "__main__":
     main()
diff --git a/dev/vendor-tpcds-queries.sh b/dev/vendor-tpcds-queries.sh
new file mode 100755
index 0000000000..9f381d66c2
--- /dev/null
+++ b/dev/vendor-tpcds-queries.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fetch the 99 DataFusion-materialized TPC-DS queries (branch-54) into
+# benchmarks/queries-tpcds/qN.sql. Re-run to refresh when the DataFusion pin
+# changes. Source files are named N.sql upstream; we prefix with `q` to match
+# the TPC-H convention.
+set -euo pipefail
+
+BRANCH="${DATAFUSION_BRANCH:-branch-54}"
+BASE="https://raw.githubusercontent.com/apache/datafusion/${BRANCH}/datafusion/core/tests/tpc-ds"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+out_dir="${script_dir}/../benchmarks/queries-tpcds"
+
+mkdir -p "$out_dir"
+for n in $(seq 1 99); do
+  echo "Fetching q${n}.sql"
+  curl -fsSL "${BASE}/${n}.sql" -o "${out_dir}/q${n}.sql"
+done
+echo "Vendored 99 TPC-DS queries from ${BRANCH} into ${out_dir}"
diff --git a/docker-compose.yml b/docker-compose.yml
index bb2ffaf9c1..99845507b8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,7 +39,7 @@ services:
     build:
       dockerfile: dev/docker/ballista-executor.Dockerfile
       context: .
-    command: "--bind-host 0.0.0.0 --scheduler-host ballista-scheduler --concurrent-tasks 4 --memory-pool-size 8GB --work-dir /work"
+    command: "--bind-host 0.0.0.0 --scheduler-host ballista-scheduler --vcores 4 --memory-pool-size 8GB --work-dir /work"
     deploy:
       replicas: 2
       resources:
diff --git a/docs/developer/architecture.md b/docs/developer/architecture.md
index 3e00346b58..c3e4ffb2b0 100644
--- a/docs/developer/architecture.md
+++ b/docs/developer/architecture.md
@@ -48,6 +48,48 @@ processes.
 
 ![Query Execution Flow](images/query-execution.png)
 
+## Multi-Partition Tasks
+
+Ballista dispatches at the _slice_ level, not the partition level. Each executor advertises a fixed number of
+virtual cores (`vcores`), and the scheduler packs up to that many of a stage's output partitions into a single
+task. All partitions in the slice execute concurrently under one DataFusion plan invocation: scans and shuffle
+readers are rewritten to see only the assigned partition ids, and DataFusion's per-partition `execute(N)`
+contract fans the work across the executor's threads. Slice size is bounded by the executor's free vcore count
+and by `ballista.scheduler.max_partitions_per_task` (`1` = one task per partition — the default and the
+pre-multi-partition-tasks model, preserved on merge for backwards compatibility; raise to opt into multi-partition
+tasks; `0` = unbounded — fills each task up to the executor's free vcore count).
+
+Compared to Apache Spark, whose unit of dispatch is one task per partition, Ballista's unit is one task per
+slice of partitions bound to a single executor. Spark achieves cluster-scale parallelism the same way — many
+tasks running concurrently across cores — but each task is single-threaded and doesn't share state with its
+neighbours. Ballista's slice model preserves cluster-scale parallelism _and_ adds intra-task shared-memory
+parallelism: partitions inside one slice share DataFusion's per-task memory pool budget, share the collect-left
+build side of a broadcast hash join (one hash table probed by every partition, instead of one materialization
+per task), share segment-tree indices needed for degenerate window aggregates (non-invertible aggregates like
+MIN/MAX, or wide/data-dependent frames where a sliding accumulator degrades to O(n × frame)), and can cooperate
+on shared-memory algorithms like PSRS parallel sort that a shuffle-based system can't express within a stage.
+It also unlocks pipelines whose intra-task state must be global-per-slice — e.g.
+`SELECT sum(v2) OVER (ORDER BY v2 RANGE 3 PRECEDING) FROM large`, which today collapses onto a single-partition
+sort+window and OOMs at h2o 10 GB scale; with the KLL-adaptive range-repartition rewrite that builds on this
+model, one slice-task per executor holds the sketch, buffered input, and per-partition halo state inside a
+single plan.
+
+### Composition with in-flight DataFusion AQE
+
+Two upstream DataFusion PoCs are converging on the same primitives at the single-plan level:
+[apache/datafusion#23026](https://github.com/apache/datafusion/pull/23026) adds `RangeRepartitionExec`,
+`HaloDropExec`, and a `runtime_partition_extremes` trait method to parallelize `RANGE`-frame windows inside one
+plan; [apache/datafusion#23167](https://github.com/apache/datafusion/pull/23167) adds `PipelineBreakerBuffer` +
+`RuntimeOptimizerExec` + a `RuntimeRule` trait so a plan-root coordinator can observe post-pipeline-breaker
+runtime stats and mutate adaptive operators — build-side swaps, split points, skew fixes — in place,
+streaming-native, no disk materialization. Multi-partition tasks widens the plan a single coordinator sees: one
+`RuntimeOptimizerExec` now observes the full slice's pipeline-breaker state, so `RangeRepartitionExec`'s
+halo-aware routing and any `RuntimeRule`'s adaptive decisions cover an executor's whole vcore budget instead of
+one core. Ballista's AQE stage barriers are the cluster-scale analog of that plan-root coordinator, and the same
+rule library lifts unchanged: sketches and row counts collected inside each slice-task get reported at the
+shuffle boundary, and the scheduler applies the same rules cluster-wide. Three levels, one rule library —
+intra-plan (DataFusion), intra-executor slice (this PR), inter-executor stage boundary (Ballista AQE).
+
 ## Scheduler Process
 
 The scheduler process implements a gRPC interface (defined in
diff --git a/docs/source/_static/images/tpch_allqueries.png b/docs/source/_static/images/tpch_allqueries.png
deleted file mode 100644
index 5e30bde02f..0000000000
Binary files a/docs/source/_static/images/tpch_allqueries.png and /dev/null differ
diff --git a/docs/source/_static/images/tpch_queries_compare.png b/docs/source/_static/images/tpch_queries_compare.png
deleted file mode 100644
index 969043f93a..0000000000
Binary files a/docs/source/_static/images/tpch_queries_compare.png and /dev/null differ
diff --git a/docs/source/_static/images/tpch_queries_speedup_abs.png b/docs/source/_static/images/tpch_queries_speedup_abs.png
deleted file mode 100644
index 251c8bb459..0000000000
Binary files a/docs/source/_static/images/tpch_queries_speedup_abs.png and /dev/null differ
diff --git a/docs/source/_static/images/tpch_queries_speedup_rel.png b/docs/source/_static/images/tpch_queries_speedup_rel.png
deleted file mode 100644
index 04f044c008..0000000000
Binary files a/docs/source/_static/images/tpch_queries_speedup_rel.png and /dev/null differ
diff --git a/docs/source/changelog/0.10.0.md b/docs/source/changelog/0.10.0.md
new file mode 100644
index 0000000000..2984a207bb
--- /dev/null
+++ b/docs/source/changelog/0.10.0.md
@@ -0,0 +1,95 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.10.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.9.0...0.10.0)
+
+**Implemented enhancements:**
+
+- Add user guide section on prometheus metrics [\#507](https://github.com/apache/datafusion-ballista/issues/507)
+- Don't throw error when job path not exist in remove_job_data [\#502](https://github.com/apache/datafusion-ballista/issues/502)
+- Fix clippy warning [\#494](https://github.com/apache/datafusion-ballista/issues/494)
+- Use job_data_clean_up_interval_seconds == 0 to indicate executor_cleanup_enable [\#488](https://github.com/apache/datafusion-ballista/issues/488)
+- Add a config for tracing log rolling policy for both scheduler and executor [\#486](https://github.com/apache/datafusion-ballista/issues/486)
+- Set up repo where we can push benchmark results [\#473](https://github.com/apache/datafusion-ballista/issues/473)
+- Make the delayed time interval for cleanup job data in both scheduler and executor configurable [\#469](https://github.com/apache/datafusion-ballista/issues/469)
+- Add some validation for the remove_job_data grpc service [\#467](https://github.com/apache/datafusion-ballista/issues/467)
+- Add ability to build docker images using `release-lto` profile [\#463](https://github.com/apache/datafusion-ballista/issues/463)
+- Suggest users download \(rather than build\) the FlightSQL JDBC Driver [\#460](https://github.com/apache/datafusion-ballista/issues/460)
+- Clean up legacy job shuffle data [\#459](https://github.com/apache/datafusion-ballista/issues/459)
+- Add grpc service for the scheduler to make it able to be triggered by client explicitly [\#458](https://github.com/apache/datafusion-ballista/issues/458)
+- Replace Mutex\<HashMap\> by using DashMap [\#448](https://github.com/apache/datafusion-ballista/issues/448)
+- Refine log level [\#446](https://github.com/apache/datafusion-ballista/issues/446)
+- Upgrade to DataFusion 14.0.0 [\#445](https://github.com/apache/datafusion-ballista/issues/445)
+- Add a feature for hdfs3 [\#419](https://github.com/apache/datafusion-ballista/issues/419)
+- Add optional flag which advertises host for Arrow Flight SQL [\#418](https://github.com/apache/datafusion-ballista/issues/418)
+- Partitioning reasoning in DataFusion and Ballista [\#284](https://github.com/apache/datafusion-ballista/issues/284)
+- Stop wasting time in CI on MIRI runs [\#283](https://github.com/apache/datafusion-ballista/issues/283)
+- Publish Docker images as part of each release [\#236](https://github.com/apache/datafusion-ballista/issues/236)
+- Cleanup job/stage status from TaskManager and clean up shuffle data after a period after JobFinished [\#185](https://github.com/apache/datafusion-ballista/issues/185)
+
+**Fixed bugs:**
+
+- build broken: configure_me_codegen retroactively reserved `bind_host` [\#519](https://github.com/apache/datafusion-ballista/issues/519)
+- Return empty results for SQLs with order by [\#451](https://github.com/apache/datafusion-ballista/issues/451)
+- ballista scheduler is not taken inline parameters into account [\#443](https://github.com/apache/datafusion-ballista/issues/443)
+- \[FlightSQL\] Cannot connect with Tableau Desktop [\#428](https://github.com/apache/datafusion-ballista/issues/428)
+- Benchmark q15 fails [\#372](https://github.com/apache/datafusion-ballista/issues/372)
+- Incorrect documentation for building Ballista on Linux when using docker-compose [\#362](https://github.com/apache/datafusion-ballista/issues/362)
+- Scheduler silently replaces `ParquetExec` with `EmptyExec` if data path is not correctly mounted in container [\#353](https://github.com/apache/datafusion-ballista/issues/353)
+- SQL with order by limit returns nothing [\#334](https://github.com/apache/datafusion-ballista/issues/334)
+
+**Documentation updates:**
+
+- README updates [\#433](https://github.com/apache/datafusion-ballista/pull/433) ([andygrove](https://github.com/andygrove))
+
+**Merged pull requests:**
+
+- configure_me_codegen retroactively reserved on our `bind_host` parame… [\#520](https://github.com/apache/datafusion-ballista/pull/520) ([avantgardnerio](https://github.com/avantgardnerio))
+- Bump actions/cache from 2 to 3 [\#517](https://github.com/apache/datafusion-ballista/pull/517) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update graphviz-rust requirement from 0.3.0 to 0.4.0 [\#515](https://github.com/apache/datafusion-ballista/pull/515) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Add Prometheus metrics endpoint [\#511](https://github.com/apache/datafusion-ballista/pull/511) ([thinkharderdev](https://github.com/thinkharderdev))
+- Enable tests that work since upgrading to DataFusion 14 [\#510](https://github.com/apache/datafusion-ballista/pull/510) ([andygrove](https://github.com/andygrove))
+- Update hashbrown requirement from 0.12 to 0.13 [\#506](https://github.com/apache/datafusion-ballista/pull/506) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Don't throw error when job shuffle data path not exist in executor [\#503](https://github.com/apache/datafusion-ballista/pull/503) ([yahoNanJing](https://github.com/yahoNanJing))
+- Upgrade to DataFusion 14.0.0 and Arrow 26.0.0 [\#499](https://github.com/apache/datafusion-ballista/pull/499) ([andygrove](https://github.com/andygrove))
+- Fix clippy warning [\#495](https://github.com/apache/datafusion-ballista/pull/495) ([yahoNanJing](https://github.com/yahoNanJing))
+- Stop wasting time in CI on MIRI runs [\#491](https://github.com/apache/datafusion-ballista/pull/491) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Remove executor config executor_cleanup_enable and make the configuation name for executor cleanup more intuitive [\#489](https://github.com/apache/datafusion-ballista/pull/489) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add a config for tracing log rolling policy for both scheduler and executor [\#487](https://github.com/apache/datafusion-ballista/pull/487) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add grpc service of cleaning up job shuffle data for the scheduler to make it able to be triggered by client explicitly [\#485](https://github.com/apache/datafusion-ballista/pull/485) ([yahoNanJing](https://github.com/yahoNanJing))
+- \[Minor\] Bump DataFusion [\#480](https://github.com/apache/datafusion-ballista/pull/480) ([Dandandan](https://github.com/Dandandan))
+- Remove benchmark results from README [\#478](https://github.com/apache/datafusion-ballista/pull/478) ([andygrove](https://github.com/andygrove))
+- Update `flightsql.md` to provide correct instruction [\#476](https://github.com/apache/datafusion-ballista/pull/476) ([iajoiner](https://github.com/iajoiner))
+- Add support for Tableau [\#475](https://github.com/apache/datafusion-ballista/pull/475) ([avantgardnerio](https://github.com/avantgardnerio))
+- Add SchedulerConfig for the scheduler configurations, like event_loop_buffer_size, finished_job_data_clean_up_interval_seconds, finished_job_state_clean_up_interval_seconds [\#472](https://github.com/apache/datafusion-ballista/pull/472) ([yahoNanJing](https://github.com/yahoNanJing))
+- Bump DataFusion [\#471](https://github.com/apache/datafusion-ballista/pull/471) ([Dandandan](https://github.com/Dandandan))
+- Add some validation for remove_job_data in the executor server [\#468](https://github.com/apache/datafusion-ballista/pull/468) ([yahoNanJing](https://github.com/yahoNanJing))
+- Update documentation to reflect the release of the FlightSQL JDBC Driver [\#461](https://github.com/apache/datafusion-ballista/pull/461) ([avantgardnerio](https://github.com/avantgardnerio))
+- Bump DataFusion version [\#453](https://github.com/apache/datafusion-ballista/pull/453) ([andygrove](https://github.com/andygrove))
+- Add shuffle for SortPreservingMergeExec physical operator [\#452](https://github.com/apache/datafusion-ballista/pull/452) ([yahoNanJing](https://github.com/yahoNanJing))
+- Replace Mutex\<HashMap\> by using DashMap [\#449](https://github.com/apache/datafusion-ballista/pull/449) ([yahoNanJing](https://github.com/yahoNanJing))
+- Refine log level for trial info and periodically invoked places [\#447](https://github.com/apache/datafusion-ballista/pull/447) ([yahoNanJing](https://github.com/yahoNanJing))
+- MINOR: Add `set -e` to scripts, fix a typo [\#444](https://github.com/apache/datafusion-ballista/pull/444) ([andygrove](https://github.com/andygrove))
+- Add optional flag which advertises host for Arrow Flight SQL \#418 [\#442](https://github.com/apache/datafusion-ballista/pull/442) ([DaltonModlin](https://github.com/DaltonModlin))
+- Reorder joins after resolving stage inputs [\#441](https://github.com/apache/datafusion-ballista/pull/441) ([Dandandan](https://github.com/Dandandan))
+- Add a feature for hdfs3 [\#439](https://github.com/apache/datafusion-ballista/pull/439) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add Spark benchmarks [\#438](https://github.com/apache/datafusion-ballista/pull/438) ([andygrove](https://github.com/andygrove))
+- scheduler now verifies that `file://` ListingTable URLs are accessible [\#414](https://github.com/apache/datafusion-ballista/pull/414) ([andygrove](https://github.com/andygrove))
diff --git a/docs/source/changelog/0.11.0.md b/docs/source/changelog/0.11.0.md
new file mode 100644
index 0000000000..7dc454b048
--- /dev/null
+++ b/docs/source/changelog/0.11.0.md
@@ -0,0 +1,99 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.11.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.10.0...0.11.0)
+
+**Implemented enhancements:**
+
+- Remove `python` since it has been moved to its own repo, `datafusion-ballista-python` [\#653](https://github.com/apache/datafusion-ballista/issues/653)
+- Add executor self-registration mechanism in the heartbeat service [\#648](https://github.com/apache/datafusion-ballista/issues/648)
+- Upgrade to DataFusion 17 [\#638](https://github.com/apache/datafusion-ballista/issues/638)
+- Move Python bindings to separate repo? [\#635](https://github.com/apache/datafusion-ballista/issues/635)
+- Implement new release process [\#622](https://github.com/apache/datafusion-ballista/issues/622)
+- Change default branch name from master to main [\#618](https://github.com/apache/datafusion-ballista/issues/618)
+- Update latest datafusion dependency [\#610](https://github.com/apache/datafusion-ballista/issues/610)
+- Implement optimizer rule to remove redundant repartitioning [\#608](https://github.com/apache/datafusion-ballista/issues/608)
+- ballista-cli as \(docker\) images [\#600](https://github.com/apache/datafusion-ballista/issues/600)
+- Update contributor guide [\#598](https://github.com/apache/datafusion-ballista/issues/598)
+- Fix cargo clippy [\#570](https://github.com/apache/datafusion-ballista/issues/570)
+- Support Alibaba Cloud OSS with ObjectStore [\#566](https://github.com/apache/datafusion-ballista/issues/566)
+- Refactor `StateBackendClient` to be a higher-level interface [\#554](https://github.com/apache/datafusion-ballista/issues/554)
+- Make it concurrently to launch tasks to executors [\#544](https://github.com/apache/datafusion-ballista/issues/544)
+- Simplify docs [\#531](https://github.com/apache/datafusion-ballista/issues/531)
+- Provide an in-memory StateBackend [\#505](https://github.com/apache/datafusion-ballista/issues/505)
+- Add support for Azure blob storage [\#294](https://github.com/apache/datafusion-ballista/issues/294)
+- Add a workflow to build the image and publish it to the package [\#71](https://github.com/apache/datafusion-ballista/issues/71)
+
+**Fixed bugs:**
+
+- Rust / Check Cargo.toml formatting \(amd64, stable\) \(pull_request\) Failing [\#662](https://github.com/apache/datafusion-ballista/issues/662)
+- Protobuf parsing error [\#646](https://github.com/apache/datafusion-ballista/issues/646)
+- jobs from python client not showing up in Scheduler UI [\#625](https://github.com/apache/datafusion-ballista/issues/625)
+- ballista ui fails to build [\#594](https://github.com/apache/datafusion-ballista/issues/594)
+- cargo build --release fails for ballista-scheduler [\#590](https://github.com/apache/datafusion-ballista/issues/590)
+- docker build fails [\#589](https://github.com/apache/datafusion-ballista/issues/589)
+- Multi-scheduler Job Starvation [\#585](https://github.com/apache/datafusion-ballista/issues/585)
+- Cannot query file from S3 [\#559](https://github.com/apache/datafusion-ballista/issues/559)
+- Benchmark q16 fails [\#373](https://github.com/apache/datafusion-ballista/issues/373)
+
+**Documentation updates:**
+
+- Check in benchmark image [\#647](https://github.com/apache/datafusion-ballista/pull/647) ([andygrove](https://github.com/andygrove))
+- MINOR: Fix benchmark image link [\#596](https://github.com/apache/datafusion-ballista/pull/596) ([andygrove](https://github.com/andygrove))
+
+**Merged pull requests:**
+
+- Upgrade to DataFusion 18 [\#668](https://github.com/apache/datafusion-ballista/pull/668) ([andygrove](https://github.com/andygrove))
+- Enable physical plan round-trip tests [\#666](https://github.com/apache/datafusion-ballista/pull/666) ([andygrove](https://github.com/andygrove))
+- Upgrade to DataFusion 18.0.0-rc1 [\#664](https://github.com/apache/datafusion-ballista/pull/664) ([andygrove](https://github.com/andygrove))
+- add test_util to make examples work well [\#661](https://github.com/apache/datafusion-ballista/pull/661) ([jiangzhx](https://github.com/jiangzhx))
+- Minor refactor to reduce duplicate code [\#659](https://github.com/apache/datafusion-ballista/pull/659) ([andygrove](https://github.com/andygrove))
+- Cluster state refactor Part 2 [\#658](https://github.com/apache/datafusion-ballista/pull/658) ([thinkharderdev](https://github.com/thinkharderdev))
+- Remove `python` dir & python-related workflows [\#654](https://github.com/apache/datafusion-ballista/pull/654) ([iajoiner](https://github.com/iajoiner))
+- Add executor self-registration mechanism in the heartbeat service [\#649](https://github.com/apache/datafusion-ballista/pull/649) ([yahoNanJing](https://github.com/yahoNanJing))
+- Upgrade to DataFusion 17 [\#639](https://github.com/apache/datafusion-ballista/pull/639) ([avantgardnerio](https://github.com/avantgardnerio))
+- Upgrade to DataFusion 16 \(again\) [\#636](https://github.com/apache/datafusion-ballista/pull/636) ([avantgardnerio](https://github.com/avantgardnerio))
+- Update release process documentation [\#632](https://github.com/apache/datafusion-ballista/pull/632) ([andygrove](https://github.com/andygrove))
+- Implement new release process [\#623](https://github.com/apache/datafusion-ballista/pull/623) ([andygrove](https://github.com/andygrove))
+- Update contributor guide [\#617](https://github.com/apache/datafusion-ballista/pull/617) ([andygrove](https://github.com/andygrove))
+- Fix Cargo.toml format issue [\#616](https://github.com/apache/datafusion-ballista/pull/616) ([andygrove](https://github.com/andygrove))
+- Refactor scheduler main [\#615](https://github.com/apache/datafusion-ballista/pull/615) ([andygrove](https://github.com/andygrove))
+- Refactor executor main [\#614](https://github.com/apache/datafusion-ballista/pull/614) ([andygrove](https://github.com/andygrove))
+- Update datafusion dependency to the latest version [\#612](https://github.com/apache/datafusion-ballista/pull/612) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add support for Azure Blob Storage [\#599](https://github.com/apache/datafusion-ballista/pull/599) ([aidankovacic-8451](https://github.com/aidankovacic-8451))
+- Python: add method to get explain output as a string [\#593](https://github.com/apache/datafusion-ballista/pull/593) ([andygrove](https://github.com/andygrove))
+- Handle job resubmission [\#586](https://github.com/apache/datafusion-ballista/pull/586) ([thinkharderdev](https://github.com/thinkharderdev))
+- updated readme to contain correct versions of dependencies. [\#580](https://github.com/apache/datafusion-ballista/pull/580) ([saikrishna1-bidgely](https://github.com/saikrishna1-bidgely))
+- Update graphviz-rust requirement from 0.4.0 to 0.5.0 [\#574](https://github.com/apache/datafusion-ballista/pull/574) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Super minor spelling error [\#573](https://github.com/apache/datafusion-ballista/pull/573) ([jdye64](https://github.com/jdye64))
+- Fix cargo clippy [\#571](https://github.com/apache/datafusion-ballista/pull/571) ([yahoNanJing](https://github.com/yahoNanJing))
+- Support Alibaba Cloud OSS with ObjectStore [\#567](https://github.com/apache/datafusion-ballista/pull/567) ([r4ntix](https://github.com/r4ntix))
+- fix\(ui\): fix last seen [\#562](https://github.com/apache/datafusion-ballista/pull/562) ([duyet](https://github.com/duyet))
+- Cluster state refactor part 1 [\#560](https://github.com/apache/datafusion-ballista/pull/560) ([thinkharderdev](https://github.com/thinkharderdev))
+- Make it concurrently to launch tasks to executors [\#557](https://github.com/apache/datafusion-ballista/pull/557) ([yahoNanJing](https://github.com/yahoNanJing))
+- Update datafusion requirement from 14.0.0 to 15.0.0 [\#552](https://github.com/apache/datafusion-ballista/pull/552) ([yahoNanJing](https://github.com/yahoNanJing))
+- docs: fix style in the Helm readme [\#551](https://github.com/apache/datafusion-ballista/pull/551) ([haoxins](https://github.com/haoxins))
+- Fix Helm chart's image format [\#550](https://github.com/apache/datafusion-ballista/pull/550) ([haoxins](https://github.com/haoxins))
+- Update env_logger requirement from 0.9 to 0.10 [\#539](https://github.com/apache/datafusion-ballista/pull/539) ([dependabot[bot]](https://github.com/apps/dependabot))
+- only build docker images on rc tags [\#535](https://github.com/apache/datafusion-ballista/pull/535) ([andygrove](https://github.com/andygrove))
+- Remove `--locked` when building Python wheels [\#533](https://github.com/apache/datafusion-ballista/pull/533) ([andygrove](https://github.com/andygrove))
+- Bump actions/labeler from 4.0.2 to 4.1.0 [\#525](https://github.com/apache/datafusion-ballista/pull/525) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Provide a memory StateBackendClient [\#523](https://github.com/apache/datafusion-ballista/pull/523) ([yahoNanJing](https://github.com/yahoNanJing))
diff --git a/docs/source/changelog/0.12.0.md b/docs/source/changelog/0.12.0.md
new file mode 100644
index 0000000000..8c271e04dc
--- /dev/null
+++ b/docs/source/changelog/0.12.0.md
@@ -0,0 +1,119 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.12.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.11.0...0.12.0)
+
+**Documentation updates:**
+
+- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins)
+
+**Merged pull requests:**
+
+- [minor] remove outdate todo [#683](https://github.com/apache/datafusion-ballista/pull/683) (Ted-Jiang)
+- Add executor terminating status for graceful shutdown [#667](https://github.com/apache/datafusion-ballista/pull/667) (thinkharderdev)
+- Allow `BallistaContext::read_*` methods to read multiple paths. [#679](https://github.com/apache/datafusion-ballista/pull/679) (luckylsk34)
+- Update scheduler.md [#657](https://github.com/apache/datafusion-ballista/pull/657) (psvri)
+- Mark `SchedulerState` as pub [#688](https://github.com/apache/datafusion-ballista/pull/688) (Dandandan)
+- Update graphviz-rust requirement from 0.5.0 to 0.6.1 [#651](https://github.com/apache/datafusion-ballista/pull/651) (dependabot[bot])
+- Upgrade DataFusion to 19.0.0 [#691](https://github.com/apache/datafusion-ballista/pull/691) (r4ntix)
+- Update release docs [#692](https://github.com/apache/datafusion-ballista/pull/692) (andygrove)
+- Mark `SchedulerServer::with_task_launcher` as pub [#695](https://github.com/apache/datafusion-ballista/pull/695) (Dandandan)
+- Make task_manager pub [#698](https://github.com/apache/datafusion-ballista/pull/698) (Dandandan)
+- Add ExecutionEngine abstraction [#687](https://github.com/apache/datafusion-ballista/pull/687) (andygrove)
+- Allow accessing s3 locations in client mode [#700](https://github.com/apache/datafusion-ballista/pull/700) (luckylsk34)
+- git clone branch incorrect [#699](https://github.com/apache/datafusion-ballista/pull/699) (BubbaJoe)
+- Fix for error message during testing [#707](https://github.com/apache/datafusion-ballista/pull/707) (yahoNanJing)
+- Upgrade datafusion to 20.0.0 & sqlparser to to 0.32.0 [#711](https://github.com/apache/datafusion-ballista/pull/711) (r4ntix)
+- Update README.md [#729](https://github.com/apache/datafusion-ballista/pull/729) (jiangzhx)
+- Update link to scheduler proto file in dev docs [#713](https://github.com/apache/datafusion-ballista/pull/713) (JAicewizard)
+- Fix `show tables` fails [#715](https://github.com/apache/datafusion-ballista/pull/715) (r4ntix)
+- Remove redundant fields in ExecutorManager [#728](https://github.com/apache/datafusion-ballista/pull/728) (yahoNanJing)
+- Fix parameter '--config-backend' to '--cluster-backend' [#720](https://github.com/apache/datafusion-ballista/pull/720) (paolorechia)
+- Upgrade DataFusion to 21.0.0 [#727](https://github.com/apache/datafusion-ballista/pull/727) (r4ntix)
+- [minor] remove useless brackets [#739](https://github.com/apache/datafusion-ballista/pull/739) (Ted-Jiang)
+- Only decode plan in `LaunchMultiTaskParams` once [#743](https://github.com/apache/datafusion-ballista/pull/743) (Dandandan)
+- Upgrade DataFusion to 22.0.0 [#740](https://github.com/apache/datafusion-ballista/pull/740) (r4ntix)
+- [feature] support shuffle read with retry when facing IO error. [#738](https://github.com/apache/datafusion-ballista/pull/738) (Ted-Jiang)
+- [log] Print long running task status. [#750](https://github.com/apache/datafusion-ballista/pull/750) (Ted-Jiang)
+- Upgrade DataFusion to 23.0.0 [#755](https://github.com/apache/datafusion-ballista/pull/755) (yahoNanJing)
+- Fix plan metrics length and stage metrics length not match [#764](https://github.com/apache/datafusion-ballista/pull/764) (yahoNanJing)
+- added match arms to create ClusterStorageConfig [#766](https://github.com/apache/datafusion-ballista/pull/766) (BokarevNik)
+- [Improve] refactor the offer_reservation avoid wait result [#760](https://github.com/apache/datafusion-ballista/pull/760) (Ted-Jiang)
+- [fea] Avoid multithreaded write lock conflicts in event queue. [#754](https://github.com/apache/datafusion-ballista/pull/754) (Ted-Jiang)
+- Upgrade DataFusion to 24.0.0, Object_Store to 0.5.6 [#769](https://github.com/apache/datafusion-ballista/pull/769) (r4ntix)
+- Refine create_datafusion_context() [#778](https://github.com/apache/datafusion-ballista/pull/778) (yahoNanJing)
+- Remove output_partitioning for task definition [#776](https://github.com/apache/datafusion-ballista/pull/776) (yahoNanJing)
+- Upgrade DataFusion to 25.0.0 [#779](https://github.com/apache/datafusion-ballista/pull/779) (r4ntix)
+- Disable the ansi feature of tracing-subscriber [#784](https://github.com/apache/datafusion-ballista/pull/784) (yahoNanJing)
+- Add config grpc_server_max_decoding_message_size to make the maximum size of a decoded message at the grpc server side configurable [#782](https://github.com/apache/datafusion-ballista/pull/782) (yahoNanJing)
+- Fix nodejs issues in Docker build [#731](https://github.com/apache/datafusion-ballista/pull/731) (jnaous)
+- Upgrade node version to fix build in `main` [#794](https://github.com/apache/datafusion-ballista/pull/794) (avantgardnerio)
+- Remove redundant mod session_registry [#792](https://github.com/apache/datafusion-ballista/pull/792) (yahoNanJing)
+- Make last_seen_ts_threshold for getting alive executor at the scheduler side larger than the heartbeat time interval [#786](https://github.com/apache/datafusion-ballista/pull/786) (yahoNanJing)
+- Remove the prometheus-metrics from the default feature [#788](https://github.com/apache/datafusion-ballista/pull/788) (yahoNanJing)
+- Refine the ExecuteQuery grpc interface [#790](https://github.com/apache/datafusion-ballista/pull/790) (yahoNanJing)
+- Add config to collect statistics, enable in TPC-H benchmark [#796](https://github.com/apache/datafusion-ballista/pull/796) (Dandandan)
+- Add support for GCS data sources [#805](https://github.com/apache/datafusion-ballista/pull/805) (haoxins)
+- Update DataFusion to 26 [#798](https://github.com/apache/datafusion-ballista/pull/798) (Dandandan)
+- Issue 162 build docker image in ci [#716](https://github.com/apache/datafusion-ballista/pull/716) (paolorechia)
+- Fix index out of bounds panic [#819](https://github.com/apache/datafusion-ballista/pull/819) (yahoNanJing)
+- Refactor the TaskDefinition by changing encoding execution plan to the decoded one [#817](https://github.com/apache/datafusion-ballista/pull/817) (yahoNanJing)
+- Fix ballista-cli docs [#800](https://github.com/apache/datafusion-ballista/pull/800) (jonahgao)
+- docs: fix link [#799](https://github.com/apache/datafusion-ballista/pull/799) (haoxins)
+- Implement the with_new_children for ShuffleReaderExec [#821](https://github.com/apache/datafusion-ballista/pull/821) (yahoNanJing)
+- Update to point to the correct documentation [#838](https://github.com/apache/datafusion-ballista/pull/838) (dadepo)
+- Remove ExecutorReservation and change the task assignment philosophy from executor first to task first [#823](https://github.com/apache/datafusion-ballista/pull/823) (yahoNanJing)
+- Upgrade DataFusion to 27.0.0 [#834](https://github.com/apache/datafusion-ballista/pull/834) (r4ntix)
+- Reduce the number of calls to `create_logical_plan` [#842](https://github.com/apache/datafusion-ballista/pull/842) (jonahgao)
+- Bump semver from 5.7.1 to 5.7.2 in /ballista/scheduler/ui [#843](https://github.com/apache/datafusion-ballista/pull/843) (dependabot[bot])
+- Bump actions/labeler from 4.1.0 to 4.3.0 [#841](https://github.com/apache/datafusion-ballista/pull/841) (dependabot[bot])
+- Bump tough-cookie from 4.1.2 to 4.1.3 in /ballista/scheduler/ui [#840](https://github.com/apache/datafusion-ballista/pull/840) (dependabot[bot])
+- Update flatbuffers requirement from 22.9.29 to 23.5.26 [#801](https://github.com/apache/datafusion-ballista/pull/801) (dependabot[bot])
+- Update dirs requirement from 4.0.0 to 5.0.1 [#767](https://github.com/apache/datafusion-ballista/pull/767) (dependabot[bot])
+- Update libloading requirement from 0.7.3 to 0.8.0 [#761](https://github.com/apache/datafusion-ballista/pull/761) (dependabot[bot])
+- Introduce a cache crate supporting concurrent cache value loading [#825](https://github.com/apache/datafusion-ballista/pull/825) (yahoNanJing)
+- Fix cargo clippy for latest rust version [#848](https://github.com/apache/datafusion-ballista/pull/848) (yahoNanJing)
+- Introduce CachedBasedObjectStoreRegistry to use data source cache transparently [#827](https://github.com/apache/datafusion-ballista/pull/827) (yahoNanJing)
+- Add ConsistentHash for node topology management [#830](https://github.com/apache/datafusion-ballista/pull/830) (yahoNanJing)
+- Implement 3-phase consistent hash based task assignment policy [#833](https://github.com/apache/datafusion-ballista/pull/833) (yahoNanJing)
+- Update tonic requirement from 0.8 to 0.9 [#733](https://github.com/apache/datafusion-ballista/pull/733) (dependabot[bot])
+- Update itertools requirement from 0.10 to 0.11 [#844](https://github.com/apache/datafusion-ballista/pull/844) (dependabot[bot])
+- Update etcd-client requirement from 0.10 to 0.11 [#845](https://github.com/apache/datafusion-ballista/pull/845) (dependabot[bot])
+- Update hashbrown requirement from 0.13 to 0.14 [#846](https://github.com/apache/datafusion-ballista/pull/846) (dependabot[bot])
+- Bump word-wrap from 1.2.3 to 1.2.4 in /ballista/scheduler/ui [#849](https://github.com/apache/datafusion-ballista/pull/849) (dependabot[bot])
+- Update hdfs requirement from 0.1.1 to 0.1.4 [#856](https://github.com/apache/datafusion-ballista/pull/856) (yahoNanJing)
+- Update to DataFusion 28 [#858](https://github.com/apache/datafusion-ballista/pull/858) (Dandandan)
+- Upgrade datafusion to 30.0.0 [#866](https://github.com/apache/datafusion-ballista/pull/866) (r4ntix)
+- refactor: port get_scan_files to Ballista [#877](https://github.com/apache/datafusion-ballista/pull/877) (alamb)
+- Upgrade datafusion to 31.0.0 [#878](https://github.com/apache/datafusion-ballista/pull/878) (r4ntix)
+- Upgrade datafusion to 32.0.0 [#899](https://github.com/apache/datafusion-ballista/pull/899) (r4ntix)
+- Update to DataFusion 33 [#900](https://github.com/apache/datafusion-ballista/pull/900) (Dandandan)
+- Refactor lru mod, remove linked_hash_map [#918](https://github.com/apache/datafusion-ballista/pull/918) (PsiACE)
+- Dynamically optimize aggregate (count) based on shuffle stats [#919](https://github.com/apache/datafusion-ballista/pull/919) (Dandandan)
+- Use lz4 compression for shuffle files & flight stream, refactoring / improvements [#920](https://github.com/apache/datafusion-ballista/pull/920) (Dandandan)
+- Make max encoding message size configurable [#928](https://github.com/apache/datafusion-ballista/pull/928) (andygrove)
+- Set max message size to 16MB in gRPC clients [#931](https://github.com/apache/datafusion-ballista/pull/931) (andygrove)
+- Upgrade to DataFusion 34.0.0-rc1 [#927](https://github.com/apache/datafusion-ballista/pull/927) (andygrove)
+- Use official DF 34 release [#939](https://github.com/apache/datafusion-ballista/pull/939) (andygrove)
+- Use StreamWriter instead of FileWriter [#943](https://github.com/apache/datafusion-ballista/pull/943) (avantgardnerio)
+- Remove some TODO comments related to context fetching schemas from scheduler [#946](https://github.com/apache/datafusion-ballista/pull/946) (andygrove)
+- Fix Docker build [#947](https://github.com/apache/datafusion-ballista/pull/947) (andygrove)
+- Fix regression in DataFrame.write_xxx [#945](https://github.com/apache/datafusion-ballista/pull/945) (andygrove)
diff --git a/docs/source/changelog/0.5.0.md b/docs/source/changelog/0.5.0.md
new file mode 100644
index 0000000000..38d0355dae
--- /dev/null
+++ b/docs/source/changelog/0.5.0.md
@@ -0,0 +1,174 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.5.0 Changelog
+
+[Full Changelog](https://github.com/apache/arrow-datafusion/compare/4.0.0...ballista-0.5.0)
+
+**Breaking changes:**
+
+- \[ballista\] support date_part and date_turnc ser/de, pass tpch 7 [\#840](https://github.com/apache/arrow-datafusion/pull/840) ([houqp](https://github.com/houqp))
+- Box ScalarValue:Lists, reduce size by half size [\#788](https://github.com/apache/arrow-datafusion/pull/788) ([alamb](https://github.com/alamb))
+- Support DataFrame.collect for Ballista DataFrames [\#785](https://github.com/apache/arrow-datafusion/pull/785) ([andygrove](https://github.com/andygrove))
+- JOIN conditions are order dependent [\#778](https://github.com/apache/arrow-datafusion/pull/778) ([seddonm1](https://github.com/seddonm1))
+- UnresolvedShuffleExec should represent a single shuffle [\#727](https://github.com/apache/arrow-datafusion/pull/727) ([andygrove](https://github.com/andygrove))
+- Ballista: Make shuffle partitions configurable in benchmarks [\#702](https://github.com/apache/arrow-datafusion/pull/702) ([andygrove](https://github.com/andygrove))
+- Rename MergeExec to CoalescePartitionsExec [\#635](https://github.com/apache/arrow-datafusion/pull/635) ([andygrove](https://github.com/andygrove))
+- Ballista: Rename QueryStageExec to ShuffleWriterExec [\#633](https://github.com/apache/arrow-datafusion/pull/633) ([andygrove](https://github.com/andygrove))
+- fix 593, reduce cloning by taking ownership in logical planner's `from` fn [\#610](https://github.com/apache/arrow-datafusion/pull/610) ([Jimexist](https://github.com/Jimexist))
+- fix join column handling logic for `On` and `Using` constraints [\#605](https://github.com/apache/arrow-datafusion/pull/605) ([houqp](https://github.com/houqp))
+- Move ballista standalone mode to client [\#589](https://github.com/apache/arrow-datafusion/pull/589) ([edrevo](https://github.com/edrevo))
+- Ballista: Implement map-side shuffle [\#543](https://github.com/apache/arrow-datafusion/pull/543) ([andygrove](https://github.com/andygrove))
+- ShuffleReaderExec now supports multiple locations per partition [\#541](https://github.com/apache/arrow-datafusion/pull/541) ([andygrove](https://github.com/andygrove))
+- Make external hostname in executor optional [\#232](https://github.com/apache/arrow-datafusion/pull/232) ([edrevo](https://github.com/edrevo))
+- Remove namespace from executors [\#75](https://github.com/apache/arrow-datafusion/pull/75) ([edrevo](https://github.com/edrevo))
+- Support qualified columns in queries [\#55](https://github.com/apache/arrow-datafusion/pull/55) ([houqp](https://github.com/houqp))
+- Read CSV format text from stdin or memory [\#54](https://github.com/apache/arrow-datafusion/pull/54) ([heymind](https://github.com/heymind))
+- Remove Ballista DataFrame [\#48](https://github.com/apache/arrow-datafusion/pull/48) ([andygrove](https://github.com/andygrove))
+- Use atomics for SQLMetric implementation, remove unused name field [\#25](https://github.com/apache/arrow-datafusion/pull/25) ([returnString](https://github.com/returnString))
+
+**Implemented enhancements:**
+
+- Add crate documentation for Ballista crates [\#830](https://github.com/apache/arrow-datafusion/issues/830)
+- Support DataFrame.collect for Ballista DataFrames [\#787](https://github.com/apache/arrow-datafusion/issues/787)
+- Ballista: Prep for supporting shuffle correctly, part one [\#736](https://github.com/apache/arrow-datafusion/issues/736)
+- Ballista: Implement physical plan serde for ShuffleWriterExec [\#710](https://github.com/apache/arrow-datafusion/issues/710)
+- Ballista: Finish implementing shuffle mechanism [\#707](https://github.com/apache/arrow-datafusion/issues/707)
+- Rename QueryStageExec to ShuffleWriterExec [\#542](https://github.com/apache/arrow-datafusion/issues/542)
+- Ballista ShuffleReaderExec should be able to read from multiple locations per partition [\#540](https://github.com/apache/arrow-datafusion/issues/540)
+- \[Ballista\] Use deployments in k8s user guide [\#473](https://github.com/apache/arrow-datafusion/issues/473)
+- Ballista refactor QueryStageExec in preparation for map-side shuffle [\#458](https://github.com/apache/arrow-datafusion/issues/458)
+- Ballista: Implement map-side of shuffle [\#456](https://github.com/apache/arrow-datafusion/issues/456)
+- Refactor Ballista to separate Flight logic from execution logic [\#449](https://github.com/apache/arrow-datafusion/issues/449)
+- Use published versions of arrow rather than github shas [\#393](https://github.com/apache/arrow-datafusion/issues/393)
+- BallistaContext::collect\(\) logging is too noisy [\#352](https://github.com/apache/arrow-datafusion/issues/352)
+- Update Ballista to use new physical plan formatter utility [\#343](https://github.com/apache/arrow-datafusion/issues/343)
+- Add Ballista Getting Started documentation [\#329](https://github.com/apache/arrow-datafusion/issues/329)
+- Remove references to ballistacompute Docker Hub repo [\#325](https://github.com/apache/arrow-datafusion/issues/325)
+- Implement scalable distributed joins [\#63](https://github.com/apache/arrow-datafusion/issues/63)
+- Remove hard-coded Ballista version from scripts [\#32](https://github.com/apache/arrow-datafusion/issues/32)
+- Implement streaming versions of Dataframe.collect methods [\#789](https://github.com/apache/arrow-datafusion/pull/789) ([andygrove](https://github.com/andygrove))
+- Ballista shuffle is finally working as intended, providing scalable distributed joins [\#750](https://github.com/apache/arrow-datafusion/pull/750) ([andygrove](https://github.com/andygrove))
+- Update to use arrow 5.0 [\#721](https://github.com/apache/arrow-datafusion/pull/721) ([alamb](https://github.com/alamb))
+- Implement serde for ShuffleWriterExec [\#712](https://github.com/apache/arrow-datafusion/pull/712) ([andygrove](https://github.com/andygrove))
+- dedup using join column in wildcard expansion [\#678](https://github.com/apache/arrow-datafusion/pull/678) ([houqp](https://github.com/houqp))
+- Implement metrics for shuffle read and write [\#676](https://github.com/apache/arrow-datafusion/pull/676) ([andygrove](https://github.com/andygrove))
+- Remove hard-coded PartitionMode from Ballista serde [\#637](https://github.com/apache/arrow-datafusion/pull/637) ([andygrove](https://github.com/andygrove))
+- Ballista: Implement scalable distributed joins [\#634](https://github.com/apache/arrow-datafusion/pull/634) ([andygrove](https://github.com/andygrove))
+- Add Keda autoscaling for ballista in k8s [\#586](https://github.com/apache/arrow-datafusion/pull/586) ([edrevo](https://github.com/edrevo))
+- Add some resiliency to lost executors [\#568](https://github.com/apache/arrow-datafusion/pull/568) ([edrevo](https://github.com/edrevo))
+- Add `partition by` constructs in window functions and modify logical planning [\#501](https://github.com/apache/arrow-datafusion/pull/501) ([Jimexist](https://github.com/Jimexist))
+- Support anti join [\#482](https://github.com/apache/arrow-datafusion/pull/482) ([Dandandan](https://github.com/Dandandan))
+- add `order by` construct in window function and logical plans [\#463](https://github.com/apache/arrow-datafusion/pull/463) ([Jimexist](https://github.com/Jimexist))
+- Refactor Ballista executor so that FlightService delegates to an Executor struct [\#450](https://github.com/apache/arrow-datafusion/pull/450) ([andygrove](https://github.com/andygrove))
+- implement lead and lag built-in window function [\#429](https://github.com/apache/arrow-datafusion/pull/429) ([Jimexist](https://github.com/Jimexist))
+- Implement fmt_as for ShuffleReaderExec [\#400](https://github.com/apache/arrow-datafusion/pull/400) ([andygrove](https://github.com/andygrove))
+- Add window expression part 1 - logical and physical planning, structure, to/from proto, and explain, for empty over clause only [\#334](https://github.com/apache/arrow-datafusion/pull/334) ([Jimexist](https://github.com/Jimexist))
+- \[breaking change\] fix 265, log should be log10, and add ln [\#271](https://github.com/apache/arrow-datafusion/pull/271) ([Jimexist](https://github.com/Jimexist))
+- Allow table providers to indicate their type for catalog metadata [\#205](https://github.com/apache/arrow-datafusion/pull/205) ([returnString](https://github.com/returnString))
+- Add query 19 to TPC-H regression tests [\#59](https://github.com/apache/arrow-datafusion/pull/59) ([Dandandan](https://github.com/Dandandan))
+- Use arrow eq kernels in CaseWhen expression evaluation [\#52](https://github.com/apache/arrow-datafusion/pull/52) ([Dandandan](https://github.com/Dandandan))
+- Add option param for standalone mode [\#42](https://github.com/apache/arrow-datafusion/pull/42) ([djKooks](https://github.com/djKooks))
+- \[DataFusion\] Optimize hash join inner workings, null handling fix [\#24](https://github.com/apache/arrow-datafusion/pull/24) ([Dandandan](https://github.com/Dandandan))
+- \[Ballista\] Docker files for ui [\#22](https://github.com/apache/arrow-datafusion/pull/22) ([msathis](https://github.com/msathis))
+
+**Fixed bugs:**
+
+- Ballista: TPC-H q3 @ SF=1000 never completes [\#835](https://github.com/apache/arrow-datafusion/issues/835)
+- Ballista does not support MIN/MAX aggregate functions [\#832](https://github.com/apache/arrow-datafusion/issues/832)
+- Ballista docker images fail to build [\#828](https://github.com/apache/arrow-datafusion/issues/828)
+- Ballista: UnresolvedShuffleExec should only have a single stage_id [\#726](https://github.com/apache/arrow-datafusion/issues/726)
+- Ballista integration tests are failing [\#623](https://github.com/apache/arrow-datafusion/issues/623)
+- Integration test build failure due to arrow-rs using unstable feature [\#596](https://github.com/apache/arrow-datafusion/issues/596)
+- `cargo build` cannot build the project [\#531](https://github.com/apache/arrow-datafusion/issues/531)
+- ShuffleReaderExec does not get formatted correctly in displayable physical plan [\#399](https://github.com/apache/arrow-datafusion/issues/399)
+- Implement serde for MIN and MAX [\#833](https://github.com/apache/arrow-datafusion/pull/833) ([andygrove](https://github.com/andygrove))
+- Ballista: Prep for fixing shuffle mechansim, part 1 [\#738](https://github.com/apache/arrow-datafusion/pull/738) ([andygrove](https://github.com/andygrove))
+- Ballista: Shuffle write bug fix [\#714](https://github.com/apache/arrow-datafusion/pull/714) ([andygrove](https://github.com/andygrove))
+- honor table name for csv/parquet scan in ballista plan serde [\#629](https://github.com/apache/arrow-datafusion/pull/629) ([houqp](https://github.com/houqp))
+- MINOR: Fix integration tests by adding datafusion-cli module to docker image [\#322](https://github.com/apache/arrow-datafusion/pull/322) ([andygrove](https://github.com/andygrove))
+
+**Documentation updates:**
+
+- Add minimal crate documentation for Ballista crates [\#831](https://github.com/apache/arrow-datafusion/pull/831) ([andygrove](https://github.com/andygrove))
+- Add Ballista examples [\#775](https://github.com/apache/arrow-datafusion/pull/775) ([andygrove](https://github.com/andygrove))
+- Update ballista.proto link in architecture doc [\#502](https://github.com/apache/arrow-datafusion/pull/502) ([terrycorley](https://github.com/terrycorley))
+- Update k8s user guide to use deployments [\#474](https://github.com/apache/arrow-datafusion/pull/474) ([edrevo](https://github.com/edrevo))
+- use prettier to format md files [\#367](https://github.com/apache/arrow-datafusion/pull/367) ([Jimexist](https://github.com/Jimexist))
+- Make it easier for developers to find Ballista documentation [\#330](https://github.com/apache/arrow-datafusion/pull/330) ([andygrove](https://github.com/andygrove))
+- Instructions for cross-compiling Ballista to the Raspberry Pi [\#263](https://github.com/apache/arrow-datafusion/pull/263) ([andygrove](https://github.com/andygrove))
+- Add install guide in README [\#236](https://github.com/apache/arrow-datafusion/pull/236) ([djKooks](https://github.com/djKooks))
+
+**Performance improvements:**
+
+- Ballista: Avoid sleeping between polling for tasks [\#698](https://github.com/apache/arrow-datafusion/pull/698) ([Dandandan](https://github.com/Dandandan))
+- Make BallistaContext::collect streaming [\#535](https://github.com/apache/arrow-datafusion/pull/535) ([edrevo](https://github.com/edrevo))
+
+**Closed issues:**
+
+- Confirm git tagging strategy for releases [\#770](https://github.com/apache/arrow-datafusion/issues/770)
+- arrow::util::pretty::pretty_format_batches missing [\#769](https://github.com/apache/arrow-datafusion/issues/769)
+- move the `assert_batches_eq!` macros to a non part of datafusion [\#745](https://github.com/apache/arrow-datafusion/issues/745)
+- fix an issue where aliases are not respected in generating downstream schemas in window expr [\#592](https://github.com/apache/arrow-datafusion/issues/592)
+- make the planner to print more succinct and useful information in window function explain clause [\#526](https://github.com/apache/arrow-datafusion/issues/526)
+- move window frame module to be in `logical_plan` [\#517](https://github.com/apache/arrow-datafusion/issues/517)
+- use a more rust idiomatic way of handling nth_value [\#448](https://github.com/apache/arrow-datafusion/issues/448)
+- Make Ballista not depend on arrow directly [\#446](https://github.com/apache/arrow-datafusion/issues/446)
+- create a test with more than one partition for window functions [\#435](https://github.com/apache/arrow-datafusion/issues/435)
+- Implement hash-partitioned hash aggregate [\#27](https://github.com/apache/arrow-datafusion/issues/27)
+- Consider using GitHub pages for DataFusion/Ballista documentation [\#18](https://github.com/apache/arrow-datafusion/issues/18)
+- Add Ballista to default cargo workspace [\#17](https://github.com/apache/arrow-datafusion/issues/17)
+- Update "repository" in Cargo.toml [\#16](https://github.com/apache/arrow-datafusion/issues/16)
+- Consolidate TPC-H benchmarks [\#6](https://github.com/apache/arrow-datafusion/issues/6)
+- \[Ballista\] Fix integration test script [\#4](https://github.com/apache/arrow-datafusion/issues/4)
+- Ballista should not have separate DataFrame implementation [\#2](https://github.com/apache/arrow-datafusion/issues/2)
+
+**Merged pull requests:**
+
+- Change datatype of tpch keys from Int32 to UInt64 to support sf=1000 [\#836](https://github.com/apache/arrow-datafusion/pull/836) ([andygrove](https://github.com/andygrove))
+- Add ballista-examples to docker build [\#829](https://github.com/apache/arrow-datafusion/pull/829) ([andygrove](https://github.com/andygrove))
+- Update dependencies: prost to 0.8 and tonic to 0.5 [\#818](https://github.com/apache/arrow-datafusion/pull/818) ([alamb](https://github.com/alamb))
+- Move `hash_array` into hash_utils.rs [\#807](https://github.com/apache/arrow-datafusion/pull/807) ([alamb](https://github.com/alamb))
+- Fix: Update clippy lints for Rust 1.54 [\#794](https://github.com/apache/arrow-datafusion/pull/794) ([alamb](https://github.com/alamb))
+- MINOR: Remove unused Ballista query execution code path [\#732](https://github.com/apache/arrow-datafusion/pull/732) ([andygrove](https://github.com/andygrove))
+- \[fix\] benchmark run with compose [\#666](https://github.com/apache/arrow-datafusion/pull/666) ([rdettai](https://github.com/rdettai))
+- bring back dev scripts for ballista [\#648](https://github.com/apache/arrow-datafusion/pull/648) ([Jimexist](https://github.com/Jimexist))
+- Remove unnecessary mutex [\#639](https://github.com/apache/arrow-datafusion/pull/639) ([edrevo](https://github.com/edrevo))
+- round trip TPCH queries in tests [\#630](https://github.com/apache/arrow-datafusion/pull/630) ([houqp](https://github.com/houqp))
+- Fix build [\#627](https://github.com/apache/arrow-datafusion/pull/627) ([andygrove](https://github.com/andygrove))
+- in ballista also check for UI prettier changes [\#578](https://github.com/apache/arrow-datafusion/pull/578) ([Jimexist](https://github.com/Jimexist))
+- turn on clippy rule for needless borrow [\#545](https://github.com/apache/arrow-datafusion/pull/545) ([Jimexist](https://github.com/Jimexist))
+- reuse datafusion physical planner in ballista building from protobuf [\#532](https://github.com/apache/arrow-datafusion/pull/532) ([Jimexist](https://github.com/Jimexist))
+- update cargo.toml in python crate and fix unit test due to hash joins [\#483](https://github.com/apache/arrow-datafusion/pull/483) ([Jimexist](https://github.com/Jimexist))
+- make `VOLUME` declaration in tpch datagen docker absolute [\#466](https://github.com/apache/arrow-datafusion/pull/466) ([crepererum](https://github.com/crepererum))
+- Refactor QueryStageExec in preparation for implementing map-side shuffle [\#459](https://github.com/apache/arrow-datafusion/pull/459) ([andygrove](https://github.com/andygrove))
+- Simplified usage of `use arrow` in ballista. [\#447](https://github.com/apache/arrow-datafusion/pull/447) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Benchmark subcommand to distinguish between DataFusion and Ballista [\#402](https://github.com/apache/arrow-datafusion/pull/402) ([jgoday](https://github.com/jgoday))
+- \#352: BallistaContext::collect\(\) logging is too noisy [\#394](https://github.com/apache/arrow-datafusion/pull/394) ([jgoday](https://github.com/jgoday))
+- cleanup function return type fn [\#350](https://github.com/apache/arrow-datafusion/pull/350) ([Jimexist](https://github.com/Jimexist))
+- Update Ballista to use new physical plan formatter utility [\#344](https://github.com/apache/arrow-datafusion/pull/344) ([andygrove](https://github.com/andygrove))
+- Update arrow dependencies again [\#341](https://github.com/apache/arrow-datafusion/pull/341) ([alamb](https://github.com/alamb))
+- Remove references to Ballista Docker images published to ballistacompute Docker Hub repo [\#326](https://github.com/apache/arrow-datafusion/pull/326) ([andygrove](https://github.com/andygrove))
+- Update arrow-rs deps [\#317](https://github.com/apache/arrow-datafusion/pull/317) ([alamb](https://github.com/alamb))
+- Update arrow deps [\#269](https://github.com/apache/arrow-datafusion/pull/269) ([alamb](https://github.com/alamb))
+- Enable redundant_field_names clippy lint [\#261](https://github.com/apache/arrow-datafusion/pull/261) ([Dandandan](https://github.com/Dandandan))
+- Update arrow-rs deps \(to fix build due to flatbuffers update\) [\#224](https://github.com/apache/arrow-datafusion/pull/224) ([alamb](https://github.com/alamb))
+- update arrow-rs deps to latest master [\#216](https://github.com/apache/arrow-datafusion/pull/216) ([alamb](https://github.com/alamb))
+
+\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_
diff --git a/docs/source/changelog/0.6.0.md b/docs/source/changelog/0.6.0.md
new file mode 100644
index 0000000000..9657586118
--- /dev/null
+++ b/docs/source/changelog/0.6.0.md
@@ -0,0 +1,105 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.6.0 Changelog
+
+[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.5.0...ballista-0.6.0)
+
+**Breaking changes:**
+
+- File partitioning for ListingTable [\#1141](https://github.com/apache/arrow-datafusion/pull/1141) ([rdettai](https://github.com/rdettai))
+- Register tables in BallistaContext using TableProviders instead of Dataframe [\#1028](https://github.com/apache/arrow-datafusion/pull/1028) ([rdettai](https://github.com/rdettai))
+- Make TableProvider.scan\(\) and PhysicalPlanner::create_physical_plan\(\) async [\#1013](https://github.com/apache/arrow-datafusion/pull/1013) ([rdettai](https://github.com/rdettai))
+- Reorganize table providers by table format [\#1010](https://github.com/apache/arrow-datafusion/pull/1010) ([rdettai](https://github.com/rdettai))
+- Move CBOs and Statistics to physical plan [\#965](https://github.com/apache/arrow-datafusion/pull/965) ([rdettai](https://github.com/rdettai))
+- Update to sqlparser v 0.10.0 [\#934](https://github.com/apache/arrow-datafusion/pull/934) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
+- FilePartition and PartitionedFile for scanning flexibility [\#932](https://github.com/apache/arrow-datafusion/pull/932) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([yjshen](https://github.com/yjshen))
+- Improve SQLMetric APIs, port existing metrics [\#908](https://github.com/apache/arrow-datafusion/pull/908) ([alamb](https://github.com/alamb))
+- Add support for EXPLAIN ANALYZE [\#858](https://github.com/apache/arrow-datafusion/pull/858) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
+- Rename concurrency to target_partitions [\#706](https://github.com/apache/arrow-datafusion/pull/706) ([andygrove](https://github.com/andygrove))
+
+**Implemented enhancements:**
+
+- Update datafusion-cli to support Ballista, or implement new ballista-cli [\#886](https://github.com/apache/arrow-datafusion/issues/886)
+- Prepare Ballista crates for publishing [\#509](https://github.com/apache/arrow-datafusion/issues/509)
+- Add drop table support [\#1266](https://github.com/apache/arrow-datafusion/pull/1266) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([viirya](https://github.com/viirya))
+- use arrow 6.1.0 [\#1255](https://github.com/apache/arrow-datafusion/pull/1255) ([Jimexist](https://github.com/Jimexist))
+- Add support for `create table as` via MemTable [\#1243](https://github.com/apache/arrow-datafusion/pull/1243) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan))
+- add values list expression [\#1165](https://github.com/apache/arrow-datafusion/pull/1165) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist))
+- Multiple files per partitions for CSV Avro Json [\#1138](https://github.com/apache/arrow-datafusion/pull/1138) ([rdettai](https://github.com/rdettai))
+- Implement INTERSECT & INTERSECT DISTINCT [\#1135](https://github.com/apache/arrow-datafusion/pull/1135) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
+- Simplify file struct abstractions [\#1120](https://github.com/apache/arrow-datafusion/pull/1120) ([rdettai](https://github.com/rdettai))
+- Implement `is [not] distinct from` [\#1117](https://github.com/apache/arrow-datafusion/pull/1117) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan))
+- add digest\(utf8, method\) function and refactor all current hash digest functions [\#1090](https://github.com/apache/arrow-datafusion/pull/1090) ([Jimexist](https://github.com/Jimexist))
+- \[crypto\] add `blake3` algorithm to `digest` function [\#1086](https://github.com/apache/arrow-datafusion/pull/1086) ([Jimexist](https://github.com/Jimexist))
+- \[crypto\] add blake2b and blake2s functions [\#1081](https://github.com/apache/arrow-datafusion/pull/1081) ([Jimexist](https://github.com/Jimexist))
+- Update sqlparser-rs to 0.11 [\#1052](https://github.com/apache/arrow-datafusion/pull/1052) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
+- remove hard coded partition count in ballista logicalplan deserialization [\#1044](https://github.com/apache/arrow-datafusion/pull/1044) ([xudong963](https://github.com/xudong963))
+- Indexed field access for List [\#1006](https://github.com/apache/arrow-datafusion/pull/1006) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki))
+- Update DataFusion to arrow 6.0 [\#984](https://github.com/apache/arrow-datafusion/pull/984) ([alamb](https://github.com/alamb))
+- Implement Display for Expr, improve operator display [\#971](https://github.com/apache/arrow-datafusion/pull/971) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
+- ObjectStore API to read from remote storage systems [\#950](https://github.com/apache/arrow-datafusion/pull/950) ([yjshen](https://github.com/yjshen))
+- fixes \#933 replace placeholder fmt_as fr ExecutionPlan impls [\#939](https://github.com/apache/arrow-datafusion/pull/939) ([tiphaineruy](https://github.com/tiphaineruy))
+- Support `NotLike` in Ballista [\#916](https://github.com/apache/arrow-datafusion/pull/916) ([Dandandan](https://github.com/Dandandan))
+- Avro Table Provider [\#910](https://github.com/apache/arrow-datafusion/pull/910) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki))
+- Add BaselineMetrics, Timestamp metrics, add for `CoalescePartitionsExec`, rename output_time -\> elapsed_compute [\#909](https://github.com/apache/arrow-datafusion/pull/909) ([alamb](https://github.com/alamb))
+- \[Ballista\] Add executor last seen info to the ui [\#895](https://github.com/apache/arrow-datafusion/pull/895) ([msathis](https://github.com/msathis))
+- add cross join support to ballista [\#891](https://github.com/apache/arrow-datafusion/pull/891) ([houqp](https://github.com/houqp))
+- Add Ballista support to DataFusion CLI [\#889](https://github.com/apache/arrow-datafusion/pull/889) ([andygrove](https://github.com/andygrove))
+- Add support for PostgreSQL regex match [\#870](https://github.com/apache/arrow-datafusion/pull/870) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([b41sh](https://github.com/b41sh))
+
+**Fixed bugs:**
+
+- Test execution_plans::shuffle_writer::tests::test Fail [\#1040](https://github.com/apache/arrow-datafusion/issues/1040)
+- Integration test fails to build docker images [\#918](https://github.com/apache/arrow-datafusion/issues/918)
+- Ballista: Remove hard-coded concurrency from logical plan serde code [\#708](https://github.com/apache/arrow-datafusion/issues/708)
+- How can I make ballista distributed compute work? [\#327](https://github.com/apache/arrow-datafusion/issues/327)
+- fix subquery alias [\#1067](https://github.com/apache/arrow-datafusion/pull/1067) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
+- Fix compilation for ballista in stand-alone mode [\#1008](https://github.com/apache/arrow-datafusion/pull/1008) ([Igosuki](https://github.com/Igosuki))
+
+**Documentation updates:**
+
+- Add Ballista roadmap [\#1166](https://github.com/apache/arrow-datafusion/pull/1166) ([andygrove](https://github.com/andygrove))
+- Adds note on compatible rust version [\#1097](https://github.com/apache/arrow-datafusion/pull/1097) ([1nF0rmed](https://github.com/1nF0rmed))
+- implement `approx_distinct` function using HyperLogLog [\#1087](https://github.com/apache/arrow-datafusion/pull/1087) ([Jimexist](https://github.com/Jimexist))
+- Improve User Guide [\#954](https://github.com/apache/arrow-datafusion/pull/954) ([andygrove](https://github.com/andygrove))
+- Update plan_query_stages doc [\#951](https://github.com/apache/arrow-datafusion/pull/951) ([rdettai](https://github.com/rdettai))
+- \[DataFusion\] - Add show and show_limit function for DataFrame [\#923](https://github.com/apache/arrow-datafusion/pull/923) ([francis-du](https://github.com/francis-du))
+- update docs related to protoc and optional syntax [\#902](https://github.com/apache/arrow-datafusion/pull/902) ([Jimexist](https://github.com/Jimexist))
+- Improve Ballista crate README content [\#878](https://github.com/apache/arrow-datafusion/pull/878) ([andygrove](https://github.com/andygrove))
+
+**Performance improvements:**
+
+- optimize build profile for datafusion python binding, cli and ballista [\#1137](https://github.com/apache/arrow-datafusion/pull/1137) ([houqp](https://github.com/houqp))
+
+**Closed issues:**
+
+- InList expr with NULL literals do not work [\#1190](https://github.com/apache/arrow-datafusion/issues/1190)
+- update the homepage README to include values, `approx_distinct`, etc. [\#1171](https://github.com/apache/arrow-datafusion/issues/1171)
+- \[Python\]: Inconsistencies with Python package name [\#1011](https://github.com/apache/arrow-datafusion/issues/1011)
+- Wanting to contribute to project where to start? [\#983](https://github.com/apache/arrow-datafusion/issues/983)
+- delete redundant code [\#973](https://github.com/apache/arrow-datafusion/issues/973)
+- How to build DataFusion python wheel [\#853](https://github.com/apache/arrow-datafusion/issues/853)
+- Produce a design for a metrics framework [\#21](https://github.com/apache/arrow-datafusion/issues/21)
+
+**Merged pull requests:**
+
+- \[nit\] simplify ballista executor `CollectExec` impl codes [\#1140](https://github.com/apache/arrow-datafusion/pull/1140) ([panarch](https://github.com/panarch))
+
+For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)
diff --git a/docs/source/changelog/0.7.0.md b/docs/source/changelog/0.7.0.md
new file mode 100644
index 0000000000..7fffd781f9
--- /dev/null
+++ b/docs/source/changelog/0.7.0.md
@@ -0,0 +1,165 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.7.0 Changelog
+
+[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.1.0-rc1...ballista-0.7.0)
+
+**Breaking changes:**
+
+- Make `ExecutionPlan::execute` Sync [\#2434](https://github.com/apache/arrow-datafusion/pull/2434) ([tustvold](https://github.com/tustvold))
+- Add `Expr::Exists` to represent EXISTS subquery expression [\#2339](https://github.com/apache/arrow-datafusion/pull/2339) ([andygrove](https://github.com/andygrove))
+- Remove dependency from `LogicalPlan::TableScan` to `ExecutionPlan` [\#2284](https://github.com/apache/arrow-datafusion/pull/2284) ([andygrove](https://github.com/andygrove))
+- Move logical expression type-coercion code from `physical-expr` crate to `expr` crate [\#2257](https://github.com/apache/arrow-datafusion/pull/2257) ([andygrove](https://github.com/andygrove))
+- feat: 2061 create external table ddl table partition cols [\#2099](https://github.com/apache/arrow-datafusion/pull/2099) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jychen7](https://github.com/jychen7))
+- Reorganize the project folders [\#2081](https://github.com/apache/arrow-datafusion/pull/2081) ([yahoNanJing](https://github.com/yahoNanJing))
+- Support more ScalarFunction in Ballista [\#2008](https://github.com/apache/arrow-datafusion/pull/2008) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Merge dataframe and dataframe imp [\#1998](https://github.com/apache/arrow-datafusion/pull/1998) ([vchag](https://github.com/vchag))
+- Rename `ExecutionContext` to `SessionContext`, `ExecutionContextState` to `SessionState`, add `TaskContext` to support multi-tenancy configurations - Part 1 [\#1987](https://github.com/apache/arrow-datafusion/pull/1987) ([mingmwang](https://github.com/mingmwang))
+- Add Coalesce function [\#1969](https://github.com/apache/arrow-datafusion/pull/1969) ([msathis](https://github.com/msathis))
+- Add Create Schema functionality in SQL [\#1959](https://github.com/apache/arrow-datafusion/pull/1959) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
+- remove sync constraint of SendableRecordBatchStream [\#1884](https://github.com/apache/arrow-datafusion/pull/1884) ([doki23](https://github.com/doki23))
+
+**Implemented enhancements:**
+
+- Add `CREATE VIEW` [\#2279](https://github.com/apache/arrow-datafusion/pull/2279) ([matthewmturner](https://github.com/matthewmturner))
+- \[Ballista\] Support Union in ballista. [\#2098](https://github.com/apache/arrow-datafusion/pull/2098) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Add missing aggr_expr to PhysicalExprNode for Ballista. [\#1989](https://github.com/apache/arrow-datafusion/pull/1989) ([Ted-Jiang](https://github.com/Ted-Jiang))
+
+**Fixed bugs:**
+
+- Ballista integration tests no longer work [\#2440](https://github.com/apache/arrow-datafusion/issues/2440)
+- Ballista crates cannot be released from DafaFusion 7.0.0 source release [\#1980](https://github.com/apache/arrow-datafusion/issues/1980)
+- protobuf OctetLength should be deserialized as octet_length, not length [\#1834](https://github.com/apache/arrow-datafusion/pull/1834) ([carols10cents](https://github.com/carols10cents))
+
+**Documentation updates:**
+
+- MINOR: Make crate READMEs consistent [\#2437](https://github.com/apache/arrow-datafusion/pull/2437) ([andygrove](https://github.com/andygrove))
+- docs: Update the Ballista dev env instructions [\#2419](https://github.com/apache/arrow-datafusion/pull/2419) ([haoxins](https://github.com/haoxins))
+- Revise document of installing ballista pinned to specified version [\#2034](https://github.com/apache/arrow-datafusion/pull/2034) ([WinkerDu](https://github.com/WinkerDu))
+- Fix typos \(Datafusion -\> DataFusion\) [\#1993](https://github.com/apache/arrow-datafusion/pull/1993) ([andygrove](https://github.com/andygrove))
+
+**Performance improvements:**
+
+- Introduce StageManager for managing tasks stage by stage [\#1983](https://github.com/apache/arrow-datafusion/pull/1983) ([yahoNanJing](https://github.com/yahoNanJing))
+
+**Closed issues:**
+
+- Make expected result string in unit tests more readable [\#2412](https://github.com/apache/arrow-datafusion/issues/2412)
+- remove duplicated `fn aggregate()` in aggregate expression tests [\#2399](https://github.com/apache/arrow-datafusion/issues/2399)
+- split `distinct_expression.rs` into `count_distinct.rs` and `array_agg_distinct.rs` [\#2385](https://github.com/apache/arrow-datafusion/issues/2385)
+- move sql tests in `context.rs` to corresponding test files in `datafustion/core/tests/sql` [\#2328](https://github.com/apache/arrow-datafusion/issues/2328)
+- Date32/Date64 as join keys for merge join [\#2314](https://github.com/apache/arrow-datafusion/issues/2314)
+- Error precision and scale for decimal coercion in logic comparison [\#2232](https://github.com/apache/arrow-datafusion/issues/2232)
+- Support Multiple row layout [\#2188](https://github.com/apache/arrow-datafusion/issues/2188)
+- Discussion: Is Ballista a standalone system or framework [\#1916](https://github.com/apache/arrow-datafusion/issues/1916)
+
+**Merged pull requests:**
+
+- MINOR: Enable multi-statement benchmark queries [\#2507](https://github.com/apache/arrow-datafusion/pull/2507) ([andygrove](https://github.com/andygrove))
+- Persist session configs in scheduler [\#2501](https://github.com/apache/arrow-datafusion/pull/2501) ([thinkharderdev](https://github.com/thinkharderdev))
+- Update to `sqlparser` `0.17.0` [\#2500](https://github.com/apache/arrow-datafusion/pull/2500) ([alamb](https://github.com/alamb))
+- Limit cpu cores used when generating changelog [\#2494](https://github.com/apache/arrow-datafusion/pull/2494) ([andygrove](https://github.com/andygrove))
+- MINOR: Parameterize changelog script [\#2484](https://github.com/apache/arrow-datafusion/pull/2484) ([jychen7](https://github.com/jychen7))
+- Fix stage key extraction [\#2472](https://github.com/apache/arrow-datafusion/pull/2472) ([thinkharderdev](https://github.com/thinkharderdev))
+- Add support for list_dir\(\) on local fs [\#2467](https://github.com/apache/arrow-datafusion/pull/2467) ([wjones127](https://github.com/wjones127))
+- minor: update versions and paths in changelog scripts [\#2429](https://github.com/apache/arrow-datafusion/pull/2429) ([andygrove](https://github.com/andygrove))
+- Fix Ballista executing during plan [\#2428](https://github.com/apache/arrow-datafusion/pull/2428) ([tustvold](https://github.com/tustvold))
+- Re-organize and rename aggregates physical plan [\#2388](https://github.com/apache/arrow-datafusion/pull/2388) ([yjshen](https://github.com/yjshen))
+- Upgrade to arrow 13 [\#2382](https://github.com/apache/arrow-datafusion/pull/2382) ([alamb](https://github.com/alamb))
+- Grouped Aggregate in row format [\#2375](https://github.com/apache/arrow-datafusion/pull/2375) ([yjshen](https://github.com/yjshen))
+- Stop optimizing queries twice [\#2369](https://github.com/apache/arrow-datafusion/pull/2369) ([andygrove](https://github.com/andygrove))
+- Bump follow-redirects from 1.13.2 to 1.14.9 in /ballista/ui/scheduler [\#2325](https://github.com/apache/arrow-datafusion/pull/2325) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Move FileType enum from sql module to logical_plan module [\#2290](https://github.com/apache/arrow-datafusion/pull/2290) ([andygrove](https://github.com/andygrove))
+- Add BatchPartitioner \(\#2285\) [\#2287](https://github.com/apache/arrow-datafusion/pull/2287) ([tustvold](https://github.com/tustvold))
+- Update uuid requirement from 0.8 to 1.0 [\#2280](https://github.com/apache/arrow-datafusion/pull/2280) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump async from 2.6.3 to 2.6.4 in /ballista/ui/scheduler [\#2277](https://github.com/apache/arrow-datafusion/pull/2277) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump minimist from 1.2.5 to 1.2.6 in /ballista/ui/scheduler [\#2276](https://github.com/apache/arrow-datafusion/pull/2276) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump url-parse from 1.5.1 to 1.5.10 in /ballista/ui/scheduler [\#2275](https://github.com/apache/arrow-datafusion/pull/2275) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump nanoid from 3.1.20 to 3.3.3 in /ballista/ui/scheduler [\#2274](https://github.com/apache/arrow-datafusion/pull/2274) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Update to Arrow 12.0.0, update tonic and prost [\#2253](https://github.com/apache/arrow-datafusion/pull/2253) ([alamb](https://github.com/alamb))
+- Add ExecutorMetricsCollector interface [\#2234](https://github.com/apache/arrow-datafusion/pull/2234) ([thinkharderdev](https://github.com/thinkharderdev))
+- minor: add editor config file [\#2224](https://github.com/apache/arrow-datafusion/pull/2224) ([jackwener](https://github.com/jackwener))
+- \[Ballista\] Enable ApproxPercentileWithWeight in Ballista and fill UT [\#2192](https://github.com/apache/arrow-datafusion/pull/2192) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- make nightly clippy happy [\#2186](https://github.com/apache/arrow-datafusion/pull/2186) ([xudong963](https://github.com/xudong963))
+- \[Ballista\]Make PhysicalAggregateExprNode has repeated PhysicalExprNode [\#2184](https://github.com/apache/arrow-datafusion/pull/2184) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Add LogicalPlan::SubqueryAlias [\#2172](https://github.com/apache/arrow-datafusion/pull/2172) ([andygrove](https://github.com/andygrove))
+- Implement fast path of with_new_children\(\) in ExecutionPlan [\#2168](https://github.com/apache/arrow-datafusion/pull/2168) ([mingmwang](https://github.com/mingmwang))
+- \[MINOR\] ignore suspicious slow test in Ballista [\#2167](https://github.com/apache/arrow-datafusion/pull/2167) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- enable explain for ballista [\#2163](https://github.com/apache/arrow-datafusion/pull/2163) ([doki23](https://github.com/doki23))
+- Add delimiter for create external table [\#2162](https://github.com/apache/arrow-datafusion/pull/2162) ([matthewmturner](https://github.com/matthewmturner))
+- Update sqlparser requirement from 0.15 to 0.16 [\#2152](https://github.com/apache/arrow-datafusion/pull/2152) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Add IF NOT EXISTS to `CREATE TABLE` and `CREATE EXTERNAL TABLE` [\#2143](https://github.com/apache/arrow-datafusion/pull/2143) ([matthewmturner](https://github.com/matthewmturner))
+- Update quarterly roadmap for Q2 [\#2133](https://github.com/apache/arrow-datafusion/pull/2133) ([matthewmturner](https://github.com/matthewmturner))
+- \[Ballista\] Add ballista plugin manager and UDF plugin [\#2131](https://github.com/apache/arrow-datafusion/pull/2131) ([gaojun2048](https://github.com/gaojun2048))
+- Serialize scalar UDFs in physical plan [\#2130](https://github.com/apache/arrow-datafusion/pull/2130) ([thinkharderdev](https://github.com/thinkharderdev))
+- doc: update release schedule [\#2110](https://github.com/apache/arrow-datafusion/pull/2110) ([jychen7](https://github.com/jychen7))
+- Reduce repetition in Decimal binary kernels, upgrade to arrow 11.1 [\#2107](https://github.com/apache/arrow-datafusion/pull/2107) ([alamb](https://github.com/alamb))
+- update zlib version to 1.2.12 [\#2106](https://github.com/apache/arrow-datafusion/pull/2106) ([waitingkuo](https://github.com/waitingkuo))
+- Add CREATE DATABASE command to SQL [\#2094](https://github.com/apache/arrow-datafusion/pull/2094) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
+- Refactor SessionContext, BallistaContext to support multi-tenancy configurations - Part 3 [\#2091](https://github.com/apache/arrow-datafusion/pull/2091) ([mingmwang](https://github.com/mingmwang))
+- Remove dependency of common for the storage crate [\#2076](https://github.com/apache/arrow-datafusion/pull/2076) ([yahoNanJing](https://github.com/yahoNanJing))
+- [MINOR] fix doc in `EXTRACT\(field FROM source\) [\#2074](https://github.com/apache/arrow-datafusion/pull/2074) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- \[Bug\]\[Datafusion\] fix TaskContext session_config bug [\#2070](https://github.com/apache/arrow-datafusion/pull/2070) ([gaojun2048](https://github.com/gaojun2048))
+- Short-circuit evaluation for `CaseWhen` [\#2068](https://github.com/apache/arrow-datafusion/pull/2068) ([yjshen](https://github.com/yjshen))
+- split datafusion-object-store module [\#2065](https://github.com/apache/arrow-datafusion/pull/2065) ([yahoNanJing](https://github.com/yahoNanJing))
+- Change log level for noisy logs [\#2060](https://github.com/apache/arrow-datafusion/pull/2060) ([thinkharderdev](https://github.com/thinkharderdev))
+- Update to arrow/parquet 11.0 [\#2048](https://github.com/apache/arrow-datafusion/pull/2048) ([alamb](https://github.com/alamb))
+- minor: format comments \(`//` to `//`\) [\#2047](https://github.com/apache/arrow-datafusion/pull/2047) ([jackwener](https://github.com/jackwener))
+- use cargo-tomlfmt to check Cargo.toml formatting in CI [\#2033](https://github.com/apache/arrow-datafusion/pull/2033) ([WinkerDu](https://github.com/WinkerDu))
+- Refactor SessionContext, SessionState and SessionConfig to support multi-tenancy configurations - Part 2 [\#2029](https://github.com/apache/arrow-datafusion/pull/2029) ([mingmwang](https://github.com/mingmwang))
+- Simplify prerequisites for running examples [\#2028](https://github.com/apache/arrow-datafusion/pull/2028) ([doki23](https://github.com/doki23))
+- Use SessionContext to parse Expr protobuf [\#2024](https://github.com/apache/arrow-datafusion/pull/2024) ([thinkharderdev](https://github.com/thinkharderdev))
+- Fix stuck issue for the load testing of Push-based task scheduling [\#2006](https://github.com/apache/arrow-datafusion/pull/2006) ([yahoNanJing](https://github.com/yahoNanJing))
+- Fixing a typo in documentation [\#1997](https://github.com/apache/arrow-datafusion/pull/1997) ([psvri](https://github.com/psvri))
+- Fix minor clippy issue [\#1995](https://github.com/apache/arrow-datafusion/pull/1995) ([alamb](https://github.com/alamb))
+- Make it possible to only scan part of a parquet file in a partition [\#1990](https://github.com/apache/arrow-datafusion/pull/1990) ([yjshen](https://github.com/yjshen))
+- Update Dockerfile to fix integration tests [\#1982](https://github.com/apache/arrow-datafusion/pull/1982) ([andygrove](https://github.com/andygrove))
+- Update sqlparser requirement from 0.14 to 0.15 [\#1966](https://github.com/apache/arrow-datafusion/pull/1966) ([dependabot[bot]](https://github.com/apps/dependabot))
+- fix logical conflict with protobuf [\#1958](https://github.com/apache/arrow-datafusion/pull/1958) ([alamb](https://github.com/alamb))
+- Update to arrow 10.0.0, pyo3 0.16 [\#1957](https://github.com/apache/arrow-datafusion/pull/1957) ([alamb](https://github.com/alamb))
+- update jit-related dependencies [\#1953](https://github.com/apache/arrow-datafusion/pull/1953) ([xudong963](https://github.com/xudong963))
+- Allow different types of query variables \(`@@var`\) rather than just string [\#1943](https://github.com/apache/arrow-datafusion/pull/1943) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([maxburke](https://github.com/maxburke))
+- Pruning serialization [\#1941](https://github.com/apache/arrow-datafusion/pull/1941) ([thinkharderdev](https://github.com/thinkharderdev))
+- Fix select from EmptyExec always return 0 row after optimizer passes [\#1938](https://github.com/apache/arrow-datafusion/pull/1938) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Introduce Ballista query stage scheduler [\#1935](https://github.com/apache/arrow-datafusion/pull/1935) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add db benchmark script [\#1928](https://github.com/apache/arrow-datafusion/pull/1928) ([matthewmturner](https://github.com/matthewmturner))
+- fix a typo [\#1919](https://github.com/apache/arrow-datafusion/pull/1919) ([vchag](https://github.com/vchag))
+- \[MINOR\] Update copyright year in Docs [\#1918](https://github.com/apache/arrow-datafusion/pull/1918) ([alamb](https://github.com/alamb))
+- add metadata to DFSchema, close \#1806. [\#1914](https://github.com/apache/arrow-datafusion/pull/1914) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jiacai2050](https://github.com/jiacai2050))
+- Refactor scheduler state mod [\#1913](https://github.com/apache/arrow-datafusion/pull/1913) ([yahoNanJing](https://github.com/yahoNanJing))
+- Refactor the event channel [\#1912](https://github.com/apache/arrow-datafusion/pull/1912) ([yahoNanJing](https://github.com/yahoNanJing))
+- Refactor scheduler server [\#1911](https://github.com/apache/arrow-datafusion/pull/1911) ([yahoNanJing](https://github.com/yahoNanJing))
+- Clippy fix on nightly [\#1907](https://github.com/apache/arrow-datafusion/pull/1907) ([yjshen](https://github.com/yjshen))
+- Updated Rust version to 1.59 in all the files [\#1903](https://github.com/apache/arrow-datafusion/pull/1903) ([NaincyKumariKnoldus](https://github.com/NaincyKumariKnoldus))
+- Remove uneeded Mutex in Ballista Client [\#1898](https://github.com/apache/arrow-datafusion/pull/1898) ([alamb](https://github.com/alamb))
+- Create a `datafusion-proto` crate for datafusion protobuf serialization [\#1887](https://github.com/apache/arrow-datafusion/pull/1887) ([carols10cents](https://github.com/carols10cents))
+- Fix clippy lints [\#1885](https://github.com/apache/arrow-datafusion/pull/1885) ([HaoYang670](https://github.com/HaoYang670))
+- Separate cpu-bound \(query-execution\) and IO-bound\(heartbeat\) to … [\#1883](https://github.com/apache/arrow-datafusion/pull/1883) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- \[Minor\] Clean up DecimalArray API Usage [\#1869](https://github.com/apache/arrow-datafusion/pull/1869) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
+- Changes after went through "Datafusion as a library section" [\#1868](https://github.com/apache/arrow-datafusion/pull/1868) ([nonontb](https://github.com/nonontb))
+- Remove allow unused imports from ballista-core, then fix all warnings [\#1853](https://github.com/apache/arrow-datafusion/pull/1853) ([carols10cents](https://github.com/carols10cents))
+- Update to arrow 9.1.0 [\#1851](https://github.com/apache/arrow-datafusion/pull/1851) ([alamb](https://github.com/alamb))
+- move some tests out of context and into sql [\#1846](https://github.com/apache/arrow-datafusion/pull/1846) ([alamb](https://github.com/alamb))
+- Fix compiling ballista in standalone mode, add build to CI [\#1839](https://github.com/apache/arrow-datafusion/pull/1839) ([alamb](https://github.com/alamb))
+- Update documentation example for change in API [\#1812](https://github.com/apache/arrow-datafusion/pull/1812) ([alamb](https://github.com/alamb))
+- Refactor scheduler state with different management policy for volatile and stable states [\#1810](https://github.com/apache/arrow-datafusion/pull/1810) ([yahoNanJing](https://github.com/yahoNanJing))
+- DataFusion + Conbench Integration [\#1791](https://github.com/apache/arrow-datafusion/pull/1791) ([dianaclarke](https://github.com/dianaclarke))
+- Enable periodic cleanup of work_dir directories in ballista executor [\#1783](https://github.com/apache/arrow-datafusion/pull/1783) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Use`eq_dyn`, `neq_dyn`, `lt_dyn`, `lt_eq_dyn`, `gt_dyn`, `gt_eq_dyn` kernels from arrow [\#1475](https://github.com/apache/arrow-datafusion/pull/1475) ([alamb](https://github.com/alamb))
diff --git a/docs/source/changelog/0.8.0.md b/docs/source/changelog/0.8.0.md
new file mode 100644
index 0000000000..b8e5b73828
--- /dev/null
+++ b/docs/source/changelog/0.8.0.md
@@ -0,0 +1,64 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.8.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.7.0...0.8.0)
+
+**Implemented enhancements:**
+
+- Executor should use all available cores by default [\#218](https://github.com/apache/datafusion-ballista/issues/218)
+- Update task status to the task job curator scheduler [\#179](https://github.com/apache/datafusion-ballista/issues/179)
+- update datafusion and arrow to 20.0.0 [\#176](https://github.com/apache/datafusion-ballista/issues/176)
+- No scheduler logs when deployed to k8s [\#165](https://github.com/apache/datafusion-ballista/issues/165)
+- Upgrade to DataFusion 11.0.0 [\#163](https://github.com/apache/datafusion-ballista/issues/163)
+- Better encapsulation for ExecutionGraph [\#149](https://github.com/apache/datafusion-ballista/issues/149)
+- A stage may act as the input of multiple stages [\#144](https://github.com/apache/datafusion-ballista/issues/144)
+- Executor Lost handling [\#143](https://github.com/apache/datafusion-ballista/issues/143)
+- Cancel a running query. [\#139](https://github.com/apache/datafusion-ballista/issues/139)
+- Ignore the previous job_id inside fill_reservations\(\) [\#138](https://github.com/apache/datafusion-ballista/issues/138)
+- Normalize the serialization and deserialization places of protobuf structs [\#137](https://github.com/apache/datafusion-ballista/issues/137)
+- Remove revive offer event loop [\#136](https://github.com/apache/datafusion-ballista/issues/136)
+- Remove Keyspace::QueuedJobs [\#133](https://github.com/apache/datafusion-ballista/issues/133)
+- Spawn a thread for execution plan generation [\#131](https://github.com/apache/datafusion-ballista/issues/131)
+- Introduce CuratorTaskManager for make an active job be curated by only one scheduler [\#130](https://github.com/apache/datafusion-ballista/issues/130)
+- Using tokio tracing for log file [\#122](https://github.com/apache/datafusion-ballista/issues/122)
+- Ballista Executor report plan/operators metrics to Ballista Scheduler when task finish [\#116](https://github.com/apache/datafusion-ballista/issues/116)
+- Add timeout settings for Grpc Client [\#114](https://github.com/apache/datafusion-ballista/issues/114)
+- Add log level config in ballista [\#102](https://github.com/apache/datafusion-ballista/issues/102)
+- Use another channel to update the status of a task set for executor [\#96](https://github.com/apache/datafusion-ballista/issues/96)
+- Add config for concurrent_task in executor [\#94](https://github.com/apache/datafusion-ballista/issues/94)
+- Ballista should support Arrow FlightSQL [\#92](https://github.com/apache/datafusion-ballista/issues/92)
+- Why not include the `ballista-cli` in the member of workspace [\#88](https://github.com/apache/datafusion-ballista/issues/88)
+- Upgrade dependency of arrow-datafusion to commit d0d5564b8f689a01e542b8c1df829d74d0fab2b0 [\#84](https://github.com/apache/datafusion-ballista/issues/84)
+- Support sled path in config file. [\#79](https://github.com/apache/datafusion-ballista/issues/79)
+- Support for multi-scheduler deployments [\#39](https://github.com/apache/datafusion-ballista/issues/39)
+- Ballista 0.7.0 Release [\#126](https://github.com/apache/datafusion-ballista/issues/126)
+- Improvements to Ballista extensibility [\#8](https://github.com/apache/datafusion-ballista/issues/8)
+- Implement Python bindings for BallistaContext [\#15](https://github.com/apache/datafusion-ballista/issues/15)
+
+**Fixed bugs:**
+
+- Run example fails via PushStaged mode [\#214](https://github.com/apache/datafusion-ballista/issues/214)
+- Config settings in BallistaContext do not get passed to DataFusion context [\#213](https://github.com/apache/datafusion-ballista/issues/213)
+- Start scheduler fails with arguments "-s PushStaged" [\#207](https://github.com/apache/datafusion-ballista/issues/207)
+- FlightSQL is broken and CI isn't catching it [\#190](https://github.com/apache/datafusion-ballista/issues/190)
+- Query fails with "NULL is invalid as a DataFusion scalar value" [\#180](https://github.com/apache/datafusion-ballista/issues/180)
+- Executor doesn't compile, missing `tokio::signal` [\#171](https://github.com/apache/datafusion-ballista/issues/171)
+- Unable to build master [\#76](https://github.com/apache/datafusion-ballista/issues/76)
diff --git a/docs/source/changelog/0.9.0.md b/docs/source/changelog/0.9.0.md
new file mode 100644
index 0000000000..7f3ba4832a
--- /dev/null
+++ b/docs/source/changelog/0.9.0.md
@@ -0,0 +1,184 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 0.9.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.8.0...0.9.0)
+
+**Implemented enhancements:**
+
+- Support count distinct aggregation function [\#411](https://github.com/apache/datafusion-ballista/issues/411)
+- Use multi-task definition in pull-based execution loop [\#400](https://github.com/apache/datafusion-ballista/issues/400)
+- Make the scheduler event loop buffer size configurable [\#397](https://github.com/apache/datafusion-ballista/issues/397)
+- Remove active execution graph when the related job is successful or failed. [\#391](https://github.com/apache/datafusion-ballista/issues/391)
+- Improve launch task efficiency by calling LaunchMultiTask [\#389](https://github.com/apache/datafusion-ballista/issues/389)
+- Use `tokio::sync::Semaphore` to wait for available task slots [\#388](https://github.com/apache/datafusion-ballista/issues/388)
+- stdout and file log level settings are inconsistent [\#385](https://github.com/apache/datafusion-ballista/issues/385)
+- Use dedicated executor in pull based loop [\#383](https://github.com/apache/datafusion-ballista/issues/383)
+- Avoid calling scheduler when the executor cannot accept new tasks [\#377](https://github.com/apache/datafusion-ballista/issues/377)
+- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#371](https://github.com/apache/datafusion-ballista/issues/371)
+- Switch to mimalloc and enable by default [\#369](https://github.com/apache/datafusion-ballista/issues/369)
+- Integration test script should use docker-compose [\#364](https://github.com/apache/datafusion-ballista/issues/364)
+- Use local shuffle reader in containerized environments [\#356](https://github.com/apache/datafusion-ballista/issues/356)
+- Add `--ext` option to benchmark [\#352](https://github.com/apache/datafusion-ballista/issues/352)
+- Add job cancel in the UI [\#350](https://github.com/apache/datafusion-ballista/issues/350)
+- Using local shuffle reader avoid flight rpc call. [\#346](https://github.com/apache/datafusion-ballista/issues/346)
+- Add a Helm Chart [\#321](https://github.com/apache/datafusion-ballista/issues/321)
+- \[UI\] Show list of query stages with metrics [\#306](https://github.com/apache/datafusion-ballista/issues/306)
+- \[UI\] Add ability to specify job name and have it show in the job listing page in the UI [\#277](https://github.com/apache/datafusion-ballista/issues/277)
+- \[UI\] Add ability to download query plans in dot format [\#276](https://github.com/apache/datafusion-ballista/issues/276)
+- \[UI\] Add ability to render query plans [\#275](https://github.com/apache/datafusion-ballista/issues/275)
+- Add REST API documentation to User Guide [\#272](https://github.com/apache/datafusion-ballista/issues/272)
+- Graceful shutdown: Handle `SIGTERM` [\#266](https://github.com/apache/datafusion-ballista/issues/266)
+- \[EPIC\] Scheduler UI [\#265](https://github.com/apache/datafusion-ballista/issues/265)
+- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#259](https://github.com/apache/datafusion-ballista/issues/259)
+- Add a feature based object store provider [\#257](https://github.com/apache/datafusion-ballista/issues/257)
+- Add docker build files [\#248](https://github.com/apache/datafusion-ballista/issues/248)
+- Allow IDEs to recognize generated code [\#246](https://github.com/apache/datafusion-ballista/issues/246)
+- Add user guide section on Flight SQL support [\#230](https://github.com/apache/datafusion-ballista/issues/230)
+- `dev/release/README.md` is outdated [\#228](https://github.com/apache/datafusion-ballista/issues/228)
+- Make ShuffleReaderExec output less verbose [\#211](https://github.com/apache/datafusion-ballista/issues/211)
+- Add LaunchMultiTask rpc interface for executor [\#209](https://github.com/apache/datafusion-ballista/issues/209)
+- Make executor fetch shuffle partition data in parallel [\#208](https://github.com/apache/datafusion-ballista/issues/208)
+- Concurrency control and rate limit during shuffle reader [\#195](https://github.com/apache/datafusion-ballista/issues/195)
+- Update User Guide [\#160](https://github.com/apache/datafusion-ballista/issues/160)
+- Ballista 0.8.0 Release [\#159](https://github.com/apache/datafusion-ballista/issues/159)
+- Save encoded execution plan in the ExecutionStage to reduce cost of task serialization and deserialization [\#142](https://github.com/apache/datafusion-ballista/issues/142)
+- Failed task retry [\#140](https://github.com/apache/datafusion-ballista/issues/140)
+- Redefine the executor task slots [\#132](https://github.com/apache/datafusion-ballista/issues/132)
+- Use ArrowFlight bearer token auth to create a session key for FlightSql clients [\#112](https://github.com/apache/datafusion-ballista/issues/112)
+- Leverage Atomic for the in-memory states in Scheduler [\#101](https://github.com/apache/datafusion-ballista/issues/101)
+- Introduce the object stores in datafusion-contrib as optional features [\#87](https://github.com/apache/datafusion-ballista/issues/87)
+- Support multiple paths for ListingTableScanNode [\#75](https://github.com/apache/datafusion-ballista/issues/75)
+- Need clean up intermediate data in Ballista [\#9](https://github.com/apache/datafusion-ballista/issues/9)
+- Ballista does not support external file systems [\#10](https://github.com/apache/datafusion-ballista/issues/10)
+
+**Fixed bugs:**
+
+- Build errors in ./dev/build-ballista-rust.sh [\#407](https://github.com/apache/datafusion-ballista/issues/407)
+- The Ballista Scheduler Dockerfile copies a file that no longer exists [\#402](https://github.com/apache/datafusion-ballista/issues/402)
+- Benchmark q20 fails [\#374](https://github.com/apache/datafusion-ballista/issues/374)
+- Integration tests fail [\#360](https://github.com/apache/datafusion-ballista/issues/360)
+- Helm deploy fails [\#344](https://github.com/apache/datafusion-ballista/issues/344)
+- Executor get stopped unexpected [\#333](https://github.com/apache/datafusion-ballista/issues/333)
+- Executor poll work loop failure [\#311](https://github.com/apache/datafusion-ballista/issues/311)
+- Queries with `LIMIT` are failing with "PhysicalExtensionCodec is not provided" [\#300](https://github.com/apache/datafusion-ballista/issues/300)
+- Schema inference does not work in Ballista-cli with a remote context [\#287](https://github.com/apache/datafusion-ballista/issues/287)
+- There are bugs in the yarn build github misses but break our internal build [\#270](https://github.com/apache/datafusion-ballista/issues/270)
+- Race condition running docker-compose [\#267](https://github.com/apache/datafusion-ballista/issues/267)
+- Scheduler UI not working in Docker image [\#250](https://github.com/apache/datafusion-ballista/issues/250)
+- Use bind host rather than the external host for starting a local executor service [\#244](https://github.com/apache/datafusion-ballista/issues/244)
+- Initial query stages read parquet files and repartition them needlessly [\#243](https://github.com/apache/datafusion-ballista/issues/243)
+- Cannot build Docker images on macOS 12.5.1 with M1 chip [\#234](https://github.com/apache/datafusion-ballista/issues/234)
+- CLI uses DataFusion context if no host or port are provided [\#219](https://github.com/apache/datafusion-ballista/issues/219)
+- Unsupported binary operator `StringConcat` [\#201](https://github.com/apache/datafusion-ballista/issues/201)
+- Ballista assumes all aggregate expressions are not DISTINCT [\#5](https://github.com/apache/datafusion-ballista/issues/5)
+- Start ballista ui with docker, but it can not found ballista scheduler [\#11](https://github.com/apache/datafusion-ballista/issues/11)
+- Cannot build Ballista docker images on Apple silicon [\#17](https://github.com/apache/datafusion-ballista/issues/17)
+
+**Documentation updates:**
+
+- Fixup links in README.md [\#366](https://github.com/apache/datafusion-ballista/pull/366) ([romanz](https://github.com/romanz))
+- Update README in preparation for 0.9.0 release [\#318](https://github.com/apache/datafusion-ballista/pull/318) ([andygrove](https://github.com/andygrove))
+- User Guide improvements [\#274](https://github.com/apache/datafusion-ballista/pull/274) ([andygrove](https://github.com/andygrove))
+
+**Closed issues:**
+
+- Automatic version updates for github actions with dependabot [\#127](https://github.com/apache/datafusion-ballista/issues/127)
+
+**Merged pull requests:**
+
+- Return multiple tasks in poll_work based on free slots [\#429](https://github.com/apache/datafusion-ballista/pull/429) ([Dandandan](https://github.com/Dandandan))
+- Run integration tests as part of release verification script [\#426](https://github.com/apache/datafusion-ballista/pull/426) ([andygrove](https://github.com/andygrove))
+- Bump actions/setup-node from 2 to 3 [\#424](https://github.com/apache/datafusion-ballista/pull/424) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/setup-python from 2 to 4 [\#423](https://github.com/apache/datafusion-ballista/pull/423) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/checkout from 2 to 3 [\#422](https://github.com/apache/datafusion-ballista/pull/422) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/download-artifact from 2 to 3 [\#421](https://github.com/apache/datafusion-ballista/pull/421) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump actions/upload-artifact from 2 to 3 [\#420](https://github.com/apache/datafusion-ballista/pull/420) ([dependabot[bot]](https://github.com/apps/dependabot))
+- MINOR: Fix yarn warnings [\#415](https://github.com/apache/datafusion-ballista/pull/415) ([andygrove](https://github.com/andygrove))
+- Fix q20 sql typo in benchmarks [\#409](https://github.com/apache/datafusion-ballista/pull/409) ([r4ntix](https://github.com/r4ntix))
+- MINOR: Add notes on Apache Reporter [\#401](https://github.com/apache/datafusion-ballista/pull/401) ([andygrove](https://github.com/andygrove))
+- Use local shuffle reader in containerized environments and some impro… [\#399](https://github.com/apache/datafusion-ballista/pull/399) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Make the scheduler event loop buffer size configurable [\#398](https://github.com/apache/datafusion-ballista/pull/398) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add RoundRobinLocal slots policy for caching executor data to avoid seld persistency [\#396](https://github.com/apache/datafusion-ballista/pull/396) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add round robin executor slots reservation policy for the scheduler to evenly assign tasks to executors [\#395](https://github.com/apache/datafusion-ballista/pull/395) ([yahoNanJing](https://github.com/yahoNanJing))
+- Improve launch task efficiency by calling LaunchMultiTask [\#394](https://github.com/apache/datafusion-ballista/pull/394) ([yahoNanJing](https://github.com/yahoNanJing))
+- Cache encoded stage plan [\#393](https://github.com/apache/datafusion-ballista/pull/393) ([yahoNanJing](https://github.com/yahoNanJing))
+- Remove active execution graph when the related job is successful or failed [\#392](https://github.com/apache/datafusion-ballista/pull/392) ([yahoNanJing](https://github.com/yahoNanJing))
+- Update flatbuffers requirement from 2.1.2 to 22.9.29 [\#390](https://github.com/apache/datafusion-ballista/pull/390) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Unified the log level configuration behavior [\#386](https://github.com/apache/datafusion-ballista/pull/386) ([r4ntix](https://github.com/r4ntix))
+- Add DistinctCount support [\#384](https://github.com/apache/datafusion-ballista/pull/384) ([r4ntix](https://github.com/r4ntix))
+- Pull-based execution loop improvements [\#380](https://github.com/apache/datafusion-ballista/pull/380) ([Dandandan](https://github.com/Dandandan))
+- Fix latest commit [\#379](https://github.com/apache/datafusion-ballista/pull/379) ([Dandandan](https://github.com/Dandandan))
+- Avoid calling scheduler when the executor cannot accept new tasks [\#378](https://github.com/apache/datafusion-ballista/pull/378) ([Dandandan](https://github.com/Dandandan))
+- Switch to mimalloc and enable by default in executor [\#370](https://github.com/apache/datafusion-ballista/pull/370) ([Dandandan](https://github.com/Dandandan))
+- Benchmark looks for path with and without extension [\#354](https://github.com/apache/datafusion-ballista/pull/354) ([andygrove](https://github.com/andygrove))
+- Implement job cancellation in UI [\#349](https://github.com/apache/datafusion-ballista/pull/349) ([Dandandan](https://github.com/Dandandan))
+- Using local shuffle reader avoid flight rpc call. [\#347](https://github.com/apache/datafusion-ballista/pull/347) ([Ted-Jiang](https://github.com/Ted-Jiang))
+- Make helm deployable [\#345](https://github.com/apache/datafusion-ballista/pull/345) ([avantgardnerio](https://github.com/avantgardnerio))
+- Benchmark & UI improvements [\#343](https://github.com/apache/datafusion-ballista/pull/343) ([andygrove](https://github.com/andygrove))
+- Add `cancel_job` REST API [\#340](https://github.com/apache/datafusion-ballista/pull/340) ([tfeda](https://github.com/tfeda))
+- Fix labeler [\#337](https://github.com/apache/datafusion-ballista/pull/337) ([andygrove](https://github.com/andygrove))
+- Upgrade to DataFusion 13.0.0 [\#336](https://github.com/apache/datafusion-ballista/pull/336) ([andygrove](https://github.com/andygrove))
+- Check executor id consistency when receive stop executor request [\#335](https://github.com/apache/datafusion-ballista/pull/335) ([yahoNanJing](https://github.com/yahoNanJing))
+- Enable more benchmark serde tests [\#331](https://github.com/apache/datafusion-ballista/pull/331) ([andygrove](https://github.com/andygrove))
+- Downgrade `docker-compose.yaml` to version 3.3 so that we can support Ubuntu 20.04.4 LTS [\#329](https://github.com/apache/datafusion-ballista/pull/329) ([andygrove](https://github.com/andygrove))
+- update labeler [\#326](https://github.com/apache/datafusion-ballista/pull/326) ([andygrove](https://github.com/andygrove))
+- Upgrade to DataFusion 13.0.0-rc1 [\#325](https://github.com/apache/datafusion-ballista/pull/325) ([andygrove](https://github.com/andygrove))
+- Dependabot stop suggesting arrow and datafusion updates [\#324](https://github.com/apache/datafusion-ballista/pull/324) ([andygrove](https://github.com/andygrove))
+- Show job stages metrics [\#323](https://github.com/apache/datafusion-ballista/pull/323) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
+- Add helm chart [\#322](https://github.com/apache/datafusion-ballista/pull/322) ([avantgardnerio](https://github.com/avantgardnerio))
+- Atomic support for enhancement [\#319](https://github.com/apache/datafusion-ballista/pull/319) ([metesynnada](https://github.com/metesynnada))
+- Allow automatic schema inference when registering csv [\#313](https://github.com/apache/datafusion-ballista/pull/313) ([r4ntix](https://github.com/r4ntix))
+- Add ability to specify job name and have it show in the job listing page in the UI [\#312](https://github.com/apache/datafusion-ballista/pull/312) ([andygrove](https://github.com/andygrove))
+- Add REST API to generate DOT graph for individual query stage [\#310](https://github.com/apache/datafusion-ballista/pull/310) ([andygrove](https://github.com/andygrove))
+- \[UI\] Use tabbed pane with Queries and Executors tabs [\#309](https://github.com/apache/datafusion-ballista/pull/309) ([andygrove](https://github.com/andygrove))
+- REST API to get query stages [\#305](https://github.com/apache/datafusion-ballista/pull/305) ([andygrove](https://github.com/andygrove))
+- Add support for SortPreservingMergeExec; fix LIMIT bug [\#304](https://github.com/apache/datafusion-ballista/pull/304) ([andygrove](https://github.com/andygrove))
+- Add Python script to run benchmarks [\#302](https://github.com/apache/datafusion-ballista/pull/302) ([andygrove](https://github.com/andygrove))
+- \[UI\] Add ability to view query plans directly in the UI [\#301](https://github.com/apache/datafusion-ballista/pull/301) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
+- Update datafusion.proto [\#299](https://github.com/apache/datafusion-ballista/pull/299) ([andygrove](https://github.com/andygrove))
+- Replace function `from_proto_binary_op` from upstream [\#298](https://github.com/apache/datafusion-ballista/pull/298) ([askoa](https://github.com/askoa))
+- Fix dead link in contribution guideline readme file [\#297](https://github.com/apache/datafusion-ballista/pull/297) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
+- UI code cleanup [\#291](https://github.com/apache/datafusion-ballista/pull/291) ([KenSuenobu](https://github.com/KenSuenobu))
+- Add support for S3 data sources [\#290](https://github.com/apache/datafusion-ballista/pull/290) ([andygrove](https://github.com/andygrove))
+- Use latest datafusion [\#289](https://github.com/apache/datafusion-ballista/pull/289) ([andygrove](https://github.com/andygrove))
+- Fix documentation example [\#288](https://github.com/apache/datafusion-ballista/pull/288) ([onthebridgetonowhere](https://github.com/onthebridgetonowhere))
+- Improve formatting of job status in UI [\#286](https://github.com/apache/datafusion-ballista/pull/286) ([andygrove](https://github.com/andygrove))
+- Enabled download of dot files from Download icon [\#279](https://github.com/apache/datafusion-ballista/pull/279) ([KenSuenobu](https://github.com/KenSuenobu))
+- Executor graceful shutdown: Handle SIGTERM [\#278](https://github.com/apache/datafusion-ballista/pull/278) ([mingmwang](https://github.com/mingmwang))
+- Also run yarn build to catch JavaScript errors in CI [\#271](https://github.com/apache/datafusion-ballista/pull/271) ([avantgardnerio](https://github.com/avantgardnerio))
+- Store sessions so users can register tables and query them through flight [\#269](https://github.com/apache/datafusion-ballista/pull/269) ([avantgardnerio](https://github.com/avantgardnerio))
+- Fix compose for Ian [\#268](https://github.com/apache/datafusion-ballista/pull/268) ([avantgardnerio](https://github.com/avantgardnerio))
+- Task level retry and Stage level retry [\#261](https://github.com/apache/datafusion-ballista/pull/261) ([mingmwang](https://github.com/mingmwang))
+- Introduce the datafusion-objectstore-hdfs in datafusion-contrib as an object store feature [\#260](https://github.com/apache/datafusion-ballista/pull/260) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add a feature based object store provider [\#258](https://github.com/apache/datafusion-ballista/pull/258) ([yahoNanJing](https://github.com/yahoNanJing))
+- Make fetch shuffle partition data in parallel [\#256](https://github.com/apache/datafusion-ballista/pull/256) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add LaunchMultiTask rpc interface for executor [\#255](https://github.com/apache/datafusion-ballista/pull/255) ([yahoNanJing](https://github.com/yahoNanJing))
+- CLI uses ballista context instead of datafusion context in local mode [\#252](https://github.com/apache/datafusion-ballista/pull/252) ([r4ntix](https://github.com/r4ntix))
+- Fix Scheduler UI in Docker image [\#251](https://github.com/apache/datafusion-ballista/pull/251) ([andygrove](https://github.com/andygrove))
+- Generate into source folder to make IDEs happy [\#247](https://github.com/apache/datafusion-ballista/pull/247) ([avantgardnerio](https://github.com/avantgardnerio))
+- Use bind host rather than the external host for starting a local executor service [\#245](https://github.com/apache/datafusion-ballista/pull/245) ([yahoNanJing](https://github.com/yahoNanJing))
+- Add REST endpoint to get DOT graph of a job [\#242](https://github.com/apache/datafusion-ballista/pull/242) ([andygrove](https://github.com/andygrove))
+- Add list of jobs to scheduler UI [\#241](https://github.com/apache/datafusion-ballista/pull/241) ([andygrove](https://github.com/andygrove))
+- Clean up job data on both Scheduler and Executor [\#188](https://github.com/apache/datafusion-ballista/pull/188) ([mingmwang](https://github.com/mingmwang))
+- Update etcd-client requirement from 0.9 to 0.10 [\#111](https://github.com/apache/datafusion-ballista/pull/111) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump terser from 4.8.0 to 4.8.1 in /ballista/ui/scheduler [\#91](https://github.com/apache/datafusion-ballista/pull/91) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump jsdom from 16.4.0 to 16.7.0 in /ballista/ui/scheduler [\#74](https://github.com/apache/datafusion-ballista/pull/74) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump numpy from 1.21.3 to 1.22.0 in /python [\#72](https://github.com/apache/datafusion-ballista/pull/72) ([dependabot[bot]](https://github.com/apps/dependabot))
diff --git a/docs/source/changelog/43.0.0.md b/docs/source/changelog/43.0.0.md
new file mode 100644
index 0000000000..a206322d3a
--- /dev/null
+++ b/docs/source/changelog/43.0.0.md
@@ -0,0 +1,146 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 43.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/0.12.0...43.0.0-rc2)
+
+**Implemented enhancements:**
+
+- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov)
+- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove)
+- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove)
+- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm)
+- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm)
+- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm)
+
+**Documentation updates:**
+
+- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd)
+- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove)
+- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins)
+- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov)
+- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins)
+- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski)
+- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove)
+- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove)
+- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove)
+- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove)
+- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm)
+- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm)
+- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm)
+- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4)
+- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm)
+- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove)
+
+**Merged pull requests:**
+
+- PyBallista - Python SQL client for Ballista [#970](https://github.com/apache/datafusion-ballista/pull/970) (andygrove)
+- docs: Add ASF attribution [#973](https://github.com/apache/datafusion-ballista/pull/973) (simicd)
+- [Python] Add `read_csv` and `read_parquet` methods [#976](https://github.com/apache/datafusion-ballista/pull/976) (andygrove)
+- Architecture guide [#977](https://github.com/apache/datafusion-ballista/pull/977) (andygrove)
+- [Python] Add more methods to SessionContext [#978](https://github.com/apache/datafusion-ballista/pull/978) (andygrove)
+- [Python] Add `execute_logical_plan` to context [#972](https://github.com/apache/datafusion-ballista/pull/972) (andygrove)
+- Use correct product name in docs [#975](https://github.com/apache/datafusion-ballista/pull/975) (andygrove)
+- docs: enhance the ballista-cli docs [#979](https://github.com/apache/datafusion-ballista/pull/979) (haoxins)
+- docs: update user guide with docker image information [#980](https://github.com/apache/datafusion-ballista/pull/980) (etolbakov)
+- Upgrade Rust version to 1.72 to keep the same as DataFusion v35 [#982](https://github.com/apache/datafusion-ballista/pull/982) (haoxins)
+- build: Fix the ballista-cli Dockerfile [#981](https://github.com/apache/datafusion-ballista/pull/981) (haoxins)
+- feat: make max message size configurable for gRPC clients [#983](https://github.com/apache/datafusion-ballista/pull/983) (etolbakov)
+- Remove some hard-coded gRPC max message sizes [#984](https://github.com/apache/datafusion-ballista/pull/984) (andygrove)
+- docs: enhance the docs of Ballista client [#985](https://github.com/apache/datafusion-ballista/pull/985) (haoxins)
+- docs: list protoc dependency [#989](https://github.com/apache/datafusion-ballista/pull/989) (davidwilemski)
+- Fix ExecutorLost event debug info [#988](https://github.com/apache/datafusion-ballista/pull/988) (lewiszlw)
+- Fix shuffle writer test [#998](https://github.com/apache/datafusion-ballista/pull/998) (Jefffrey)
+- Bump graphviz-rust from 0.6.1 to 0.8.0 [#999](https://github.com/apache/datafusion-ballista/pull/999) (Jefffrey)
+- Add rust-toolchain.toml for clarity [#1014](https://github.com/apache/datafusion-ballista/pull/1014) (scnerd)
+- Fix executor metadata decode bug [#1004](https://github.com/apache/datafusion-ballista/pull/1004) (lewiszlw)
+- update asf yaml [#1007](https://github.com/apache/datafusion-ballista/pull/1007) (andygrove)
+- Fix Ballista rust.yml github workflow [#1026](https://github.com/apache/datafusion-ballista/pull/1026) (RaphaelMarinier)
+- Bump datafusion to 36.0.0 and make ballista compatible with it. [#1027](https://github.com/apache/datafusion-ballista/pull/1027) (RaphaelMarinier)
+- Make Ballista compatible with Datafusion 37.0.0 (from 36.0.0) [#1031](https://github.com/apache/datafusion-ballista/pull/1031) (RaphaelMarinier)
+- Fixes Setting Job Name Not Reflected in Ballista UI [#1039](https://github.com/apache/datafusion-ballista/pull/1039) (athultr1997)
+- docs: Add workflow to publish documentation [#1040](https://github.com/apache/datafusion-ballista/pull/1040) (andygrove)
+- [Docs] fix good_first_issue link in the contribution md doc [#1022](https://github.com/apache/datafusion-ballista/pull/1022) (Almaz-KG)
+- docs: Replace Arrow Ballista with DataFusion Ballista [#1041](https://github.com/apache/datafusion-ballista/pull/1041) (andygrove)
+- Fix job hangs when partition count of plan is zero [#1024](https://github.com/apache/datafusion-ballista/pull/1024) (lewiszlw)
+- Add maintenance status note [#1043](https://github.com/apache/datafusion-ballista/pull/1043) (andygrove)
+- Fix cargo build [#1045](https://github.com/apache/datafusion-ballista/pull/1045) (andygrove)
+- fix docker build in CI [#1046](https://github.com/apache/datafusion-ballista/pull/1046) (andygrove)
+- feat: Upgrade to DataFusion 38 [#1048](https://github.com/apache/datafusion-ballista/pull/1048) (andygrove)
+- Bump actions/setup-node from 3 to 4 [#909](https://github.com/apache/datafusion-ballista/pull/909) (dependabot[bot])
+- Bump actions/cache from 3 to 4 [#958](https://github.com/apache/datafusion-ballista/pull/958) (dependabot[bot])
+- feat: Upgrade to DataFusion 39.0.0 [#1052](https://github.com/apache/datafusion-ballista/pull/1052) (andygrove)
+- Update datafusion protobuf definitions [#1057](https://github.com/apache/datafusion-ballista/pull/1057) (palaska)
+- Fix regression with TPC-H benchmark [#1060](https://github.com/apache/datafusion-ballista/pull/1060) (andygrove)
+- Upgrade to Datafusion 41 [#1062](https://github.com/apache/datafusion-ballista/pull/1062) (palaska)
+- Remove helm from supported code [#1071](https://github.com/apache/datafusion-ballista/pull/1071) (milenkovicm)
+- Remove plugin subsystem [#1070](https://github.com/apache/datafusion-ballista/pull/1070) (milenkovicm)
+- Remove CI folder [#1074](https://github.com/apache/datafusion-ballista/pull/1074) (milenkovicm)
+- Code cleanup, move examples, remove unused files [#1075](https://github.com/apache/datafusion-ballista/pull/1075) (milenkovicm)
+- Remove UI [#1072](https://github.com/apache/datafusion-ballista/pull/1072) (milenkovicm)
+- Remove key-value stores for scheduler persistence [#1077](https://github.com/apache/datafusion-ballista/pull/1077) (milenkovicm)
+- Remove cache functionality [#1076](https://github.com/apache/datafusion-ballista/pull/1076) (milenkovicm)
+- Remove HDFS support ... [#1073](https://github.com/apache/datafusion-ballista/pull/1073) (milenkovicm)
+- Reorganise and remove dependencies [#1078](https://github.com/apache/datafusion-ballista/pull/1078) (milenkovicm)
+- Promote keda and flight-sql to optional features [#1079](https://github.com/apache/datafusion-ballista/pull/1079) (milenkovicm)
+- Update to datafusion 42 ... [#1080](https://github.com/apache/datafusion-ballista/pull/1080) (milenkovicm)
+- #1086 solve examples errors [#1087](https://github.com/apache/datafusion-ballista/pull/1087) (tbar4)
+- fix issue with not building python package ... [#1085](https://github.com/apache/datafusion-ballista/pull/1085) (milenkovicm)
+- another round of code cleanup ... [#1089](https://github.com/apache/datafusion-ballista/pull/1089) (milenkovicm)
+- Make rest-api optional feature ... [#1084](https://github.com/apache/datafusion-ballista/pull/1084) (milenkovicm)
+- fix clippy issues after updating to rust 1.82 [#1090](https://github.com/apache/datafusion-ballista/pull/1090) (milenkovicm)
+- Replace BallistaContext with SessionContext [#1088](https://github.com/apache/datafusion-ballista/pull/1088) (milenkovicm)
+- Removed Maintenance Notice [#1094](https://github.com/apache/datafusion-ballista/pull/1094) (tbar4)
+- Ergonomic way to setup/configure `SessionContextExt` [#1096](https://github.com/apache/datafusion-ballista/pull/1096) (milenkovicm)
+- Executor configuration extended .. [#1099](https://github.com/apache/datafusion-ballista/pull/1099) (milenkovicm)
+- fix issue with executor registration ... [#1101](https://github.com/apache/datafusion-ballista/pull/1101) (milenkovicm)
+- Deprecate `BallistaContext` [#1103](https://github.com/apache/datafusion-ballista/pull/1103) (milenkovicm)
+- fix imports after un-rebased PR [#1106](https://github.com/apache/datafusion-ballista/pull/1106) (milenkovicm)
+- Ballista proto cleanup [#1110](https://github.com/apache/datafusion-ballista/pull/1110) (milenkovicm)
+- Update and move deps to workspace [#1109](https://github.com/apache/datafusion-ballista/pull/1109) (milenkovicm)
+- Trim down `BallistaConfig` [#1108](https://github.com/apache/datafusion-ballista/pull/1108) (milenkovicm)
+- Remove build-in object store registry [#1114](https://github.com/apache/datafusion-ballista/pull/1114) (milenkovicm)
+- Update root `README.md` and other documentation with latest changes [#1113](https://github.com/apache/datafusion-ballista/pull/1113) (milenkovicm)
+- support window functions [#1112](https://github.com/apache/datafusion-ballista/pull/1112) (onursatici)
+- added a BallistaContext to ballista to allow for Remote or standalone [#1100](https://github.com/apache/datafusion-ballista/pull/1100) (tbar4)
+- Decommission `BallistaContext` [#1119](https://github.com/apache/datafusion-ballista/pull/1119) (milenkovicm)
+- docs: Update benchmarks [#1121](https://github.com/apache/datafusion-ballista/pull/1121) (andygrove)
+- Make easier to create custom schedulers and executors [#1118](https://github.com/apache/datafusion-ballista/pull/1118) (milenkovicm)
+- refactor: Move BallistaRegistry to better location [#1126](https://github.com/apache/datafusion-ballista/pull/1126) (milenkovicm)
+- refactor: BallistaLogicalExtensionCodec refactoring and improvements [#1127](https://github.com/apache/datafusion-ballista/pull/1127) (milenkovicm)
+- refactor: consolidate ballista tests [#1129](https://github.com/apache/datafusion-ballista/pull/1129) (milenkovicm)
+- refactor: SessionStateExt and SessionConfigExt [#1130](https://github.com/apache/datafusion-ballista/pull/1130) (milenkovicm)
+- chore: dependancy updates [#1131](https://github.com/apache/datafusion-ballista/pull/1131) (milenkovicm)
+- chore: fix warning mimaloc warning when building [#1137](https://github.com/apache/datafusion-ballista/pull/1137) (milenkovicm)
+- refactor: SessionBuilder to return Result<\_> [#1138](https://github.com/apache/datafusion-ballista/pull/1138) (milenkovicm)
+- chore: remove unused cache\_ options from executor [#1140](https://github.com/apache/datafusion-ballista/pull/1140) (milenkovicm)
+- updated maturin version and ccargo build to build yml [#1136](https://github.com/apache/datafusion-ballista/pull/1136) (tbar4)
+- chore: Fix clippy issues after rust update (1.83.0) [#1143](https://github.com/apache/datafusion-ballista/pull/1143) (milenkovicm)
+- Fix documentation example which still uses BallistaContext [#1145](https://github.com/apache/datafusion-ballista/pull/1145) (milenkovicm)
+- Ballista proto cleanup [#1146](https://github.com/apache/datafusion-ballista/pull/1146) (milenkovicm)
+- feat: default instance for executor configuration [#1147](https://github.com/apache/datafusion-ballista/pull/1147) (milenkovicm)
+- feat: Expose Ballista Scheduler and Executor in Python [#1148](https://github.com/apache/datafusion-ballista/pull/1148) (milenkovicm)
+- chore: dependency cleanup [#1150](https://github.com/apache/datafusion-ballista/pull/1150) (milenkovicm)
+- Update DataFusion to 43 [#1125](https://github.com/apache/datafusion-ballista/pull/1125) (Dandandan)
+- Reinstantiate join order optimization [#1122](https://github.com/apache/datafusion-ballista/pull/1122) (Dandandan)
+- add partitioning scheme for unresolved shuffle and shuffle reader exec [#1144](https://github.com/apache/datafusion-ballista/pull/1144) (onursatici)
+- chore: update py-df to 43.1 [#1152](https://github.com/apache/datafusion-ballista/pull/1152) (milenkovicm)
+- chore: no need to run python test in rust [#1154](https://github.com/apache/datafusion-ballista/pull/1154) (milenkovicm)
+- feat: add test to check for `ctx.enable_url_table()` [#1155](https://github.com/apache/datafusion-ballista/pull/1155) (milenkovicm)
diff --git a/docs/source/changelog/44.0.0.md b/docs/source/changelog/44.0.0.md
new file mode 100644
index 0000000000..409717d6ce
--- /dev/null
+++ b/docs/source/changelog/44.0.0.md
@@ -0,0 +1,56 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 44.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/43.0.0...44.0.0)
+
+**Implemented enhancements:**
+
+- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm)
+- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm)
+- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm)
+- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm)
+
+**Fixed bugs:**
+
+- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm)
+- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm)
+
+**Documentation updates:**
+
+- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm)
+
+**Merged pull requests:**
+
+- chore: planner cleanup and refactor [#1160](https://github.com/apache/datafusion-ballista/pull/1160) (milenkovicm)
+- feat: Job notification should emit job status for successful and failed jobs [#1165](https://github.com/apache/datafusion-ballista/pull/1165) (milenkovicm)
+- chore: fix executor build issue on release [#1167](https://github.com/apache/datafusion-ballista/pull/1167) (milenkovicm)
+- fix: do not compile `keda.proto` if feature not used. [#1168](https://github.com/apache/datafusion-ballista/pull/1168) (milenkovicm)
+- chore: update to DF.44 [#1153](https://github.com/apache/datafusion-ballista/pull/1153) (milenkovicm)
+- chore: publicly expose datafusion in ballista client [#1169](https://github.com/apache/datafusion-ballista/pull/1169) (milenkovicm)
+- feat: executor supports pluggable arrow flight server [#1170](https://github.com/apache/datafusion-ballista/pull/1170) (milenkovicm)
+- doc: update ballista client front page [#1171](https://github.com/apache/datafusion-ballista/pull/1171) (milenkovicm)
+- fix: rest api `/api/executors` does not show executors if `TaskSchedulingPolicy::PullStaged` [#1175](https://github.com/apache/datafusion-ballista/pull/1175) (milenkovicm)
+- chore: generate change log for 44.0.0 [#1173](https://github.com/apache/datafusion-ballista/pull/1173) (milenkovicm)
+- feat: notify scheduler when a task fails [#1181](https://github.com/apache/datafusion-ballista/pull/1181) (milenkovicm)
+- feat: configure max grpc message size and disable view types in ballista [#1185](https://github.com/apache/datafusion-ballista/pull/1185) (milenkovicm)
+- chore: fix tpch data generator [#1186](https://github.com/apache/datafusion-ballista/pull/1186) (milenkovicm)
+- chore: fix clippy after rust 1.85 update [#1188](https://github.com/apache/datafusion-ballista/pull/1188) (milenkovicm)
+- chore: commit `Cargo.lock` file to make builds more predictable [#1190](https://github.com/apache/datafusion-ballista/pull/1190) (milenkovicm)
diff --git a/docs/source/changelog/45.0.0.md b/docs/source/changelog/45.0.0.md
new file mode 100644
index 0000000000..241144aa1a
--- /dev/null
+++ b/docs/source/changelog/45.0.0.md
@@ -0,0 +1,45 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 45.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/44.0.0...45.0.0)
+
+**Implemented enhancements:**
+
+- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm)
+- feat: publish docker containers for executor and scheduler [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm)
+
+**Merged pull requests:**
+
+- chore: minor release script fix [#1192](https://github.com/apache/datafusion-ballista/pull/1192) (andygrove)
+- feat: improve executor logs [#1187](https://github.com/apache/datafusion-ballista/pull/1187) (milenkovicm)
+- chore: update datafusion to v45 [#1176](https://github.com/apache/datafusion-ballista/pull/1176) (milenkovicm)
+- chore: Update changelog for 44.0.0 [#1191](https://github.com/apache/datafusion-ballista/pull/1191) (andygrove)
+- minor: fix repo and homepage url in `cargo.toml` [#1196](https://github.com/apache/datafusion-ballista/pull/1196) (milenkovicm)
+- chore(deps): bump ring from 0.17.11 to 0.17.13 [#1199](https://github.com/apache/datafusion-ballista/pull/1199) (dependabot[bot])
+- feat: publish docker containers for executor and scheduler [#1200](https://github.com/apache/datafusion-ballista/pull/1200) (milenkovicm)
+- minor: make `graphviz-rust` dependency optional [#1203](https://github.com/apache/datafusion-ballista/pull/1203) (milenkovicm)
+- doc: update docker related documentation [#1204](https://github.com/apache/datafusion-ballista/pull/1204) (milenkovicm)
+- chore: update python dependencies [#1197](https://github.com/apache/datafusion-ballista/pull/1197) (milenkovicm)
+- chore(deps): bump ring from 0.17.8 to 0.17.14 in /python [#1206](https://github.com/apache/datafusion-ballista/pull/1206) (dependabot[bot])
+- doc: remove arrow from doc title [#1207](https://github.com/apache/datafusion-ballista/pull/1207) (milenkovicm)
+- Fix unit tests in tpch.rs [#1195](https://github.com/apache/datafusion-ballista/pull/1195) (vmingchen)
+- documentation :: quick-start.md sample source code correction [#1213](https://github.com/apache/datafusion-ballista/pull/1213) (nj7)
+- doc: fix quick-start executor command [#1217](https://github.com/apache/datafusion-ballista/pull/1217) (westhide)
diff --git a/docs/source/changelog/46.0.0.md b/docs/source/changelog/46.0.0.md
new file mode 100644
index 0000000000..f15227474d
--- /dev/null
+++ b/docs/source/changelog/46.0.0.md
@@ -0,0 +1,58 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 46.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/45.0.0...46.0.0)
+
+**Implemented enhancements:**
+
+- feat: make task distribution policies pluggable [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm)
+- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm)
+- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm)
+- feat: make execution_graph.stages() public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm)
+
+**Fixed bugs:**
+
+- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii)
+
+**Merged pull requests:**
+
+- chore: Remove some arrow references [#1232](https://github.com/apache/datafusion-ballista/pull/1232) (andygrove)
+- chore: remove unused executor configuration option [#1229](https://github.com/apache/datafusion-ballista/pull/1229) (milenkovicm)
+- chore: return `404` for api requests if path does not exist [#1224](https://github.com/apache/datafusion-ballista/pull/1224) (milenkovicm)
+- chore(ci): replace `actions-rs` which are deprecated [#1222](https://github.com/apache/datafusion-ballista/pull/1222) (milenkovicm)
+- minor: Decouple `ExecutionGraph` and `DistributedPlanner` [#1221](https://github.com/apache/datafusion-ballista/pull/1221) (milenkovicm)
+- chore: update datafusion to 46 [#1201](https://github.com/apache/datafusion-ballista/pull/1201) (milenkovicm)
+- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 [#1233](https://github.com/apache/datafusion-ballista/pull/1233) (dependabot[bot])
+- chore(deps): bump tokio from 1.44.1 to 1.44.2 [#1234](https://github.com/apache/datafusion-ballista/pull/1234) (dependabot[bot])
+- fix: executor can't read s3 config in push-staged mode [#1236](https://github.com/apache/datafusion-ballista/pull/1236) (mmooyyii)
+- chore: update python deps to 45 [#1240](https://github.com/apache/datafusion-ballista/pull/1240) (milenkovicm)
+- Add S3 object store support to executor and scheduler [#1230](https://github.com/apache/datafusion-ballista/pull/1230) (milenkovicm)
+- feat: make task distribution policies pluggable [#1243](https://github.com/apache/datafusion-ballista/pull/1243) (milenkovicm)
+- chore: reduce log levels for few log statements [#1237](https://github.com/apache/datafusion-ballista/pull/1237) (milenkovicm)
+- chore(deps): bump crossbeam-channel from 0.5.14 to 0.5.15 in /python [#1244](https://github.com/apache/datafusion-ballista/pull/1244) (dependabot[bot])
+- feat: remove flight-sql from scheduler [#1228](https://github.com/apache/datafusion-ballista/pull/1228) (milenkovicm)
+- minor: `executor_shutdown_while_running` test has race condition [#1248](https://github.com/apache/datafusion-ballista/pull/1248) (milenkovicm)
+- bug: build fails with `--no-default-features` [#1255](https://github.com/apache/datafusion-ballista/pull/1255) (milenkovicm)
+- doc: update architectural diagram [#1253](https://github.com/apache/datafusion-ballista/pull/1253) (milenkovicm)
+- feat: ballista client collects (few) metrics [#1251](https://github.com/apache/datafusion-ballista/pull/1251) (milenkovicm)
+- chore: add read/write roundtrip tests [#1249](https://github.com/apache/datafusion-ballista/pull/1249) (milenkovicm)
+- minor: change log level for object store creation [#1247](https://github.com/apache/datafusion-ballista/pull/1247) (milenkovicm)
+- feat: make execution_graph.stages() public [#1256](https://github.com/apache/datafusion-ballista/pull/1256) (milenkovicm)
diff --git a/docs/source/changelog/47.0.0.md b/docs/source/changelog/47.0.0.md
new file mode 100644
index 0000000000..413b235d79
--- /dev/null
+++ b/docs/source/changelog/47.0.0.md
@@ -0,0 +1,48 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 47.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/46.0.0...47.0.0)
+
+**Implemented enhancements:**
+
+- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm)
+- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm)
+- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm)
+- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm)
+- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm)
+- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm)
+
+**Fixed bugs:**
+
+- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm)
+- fix: fix tests failing on windows [#1273](https://github.com/apache/datafusion-ballista/pull/1273) (Huy1Ng)
+
+**Merged pull requests:**
+
+- fix: clippy issue after rust update to 1.87 [#1262](https://github.com/apache/datafusion-ballista/pull/1262) (milenkovicm)
+- feat: expose submit and cancel job methods as public in scheduler [#1260](https://github.com/apache/datafusion-ballista/pull/1260) (milenkovicm)
+- feat: split scheduler gprc creation [#1261](https://github.com/apache/datafusion-ballista/pull/1261) (milenkovicm)
+- feat: expose cluster state notifications [#1263](https://github.com/apache/datafusion-ballista/pull/1263) (milenkovicm)
+- minor: release docker on when release has been tagged [#1264](https://github.com/apache/datafusion-ballista/pull/1264) (milenkovicm)
+- feat: remove `ClusterStorageConfig` as it is redundant [#1265](https://github.com/apache/datafusion-ballista/pull/1265) (milenkovicm)
+- feat: disable task stage plan binary cache [#1266](https://github.com/apache/datafusion-ballista/pull/1266) (milenkovicm)
+- feat: `ClusterState` does not cache session contexts [#1226](https://github.com/apache/datafusion-ballista/pull/1226) (milenkovicm)
+- chore(deps): update to datafusion 47.0.0 [#1250](https://github.com/apache/datafusion-ballista/pull/1250) (milenkovicm)
diff --git a/docs/source/changelog/48.0.0.md b/docs/source/changelog/48.0.0.md
new file mode 100644
index 0000000000..4e1e1ce6bc
--- /dev/null
+++ b/docs/source/changelog/48.0.0.md
@@ -0,0 +1,44 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 48.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/47.0.0...48.0.0)
+
+**Fixed bugs:**
+
+- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG)
+- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm)
+- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm)
+
+**Documentation updates:**
+
+- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm)
+- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone)
+
+**Merged pull requests:**
+
+- chore: update datafusion to 48 [#1270](https://github.com/apache/datafusion-ballista/pull/1270) (milenkovicm)
+- docs: Apply method chaining in example [#1276](https://github.com/apache/datafusion-ballista/pull/1276) (0ne-stone)
+- fix: devcontainer protoc:1 feature url [#1278](https://github.com/apache/datafusion-ballista/pull/1278) (Almaz-KG)
+- improve rust workflows without cache [#1275](https://github.com/apache/datafusion-ballista/pull/1275) (Huy1Ng)
+- fix: change code to support rust 2024 [#1283](https://github.com/apache/datafusion-ballista/pull/1283) (milenkovicm)
+- fix: remove configure_me [#1282](https://github.com/apache/datafusion-ballista/pull/1282) (milenkovicm)
+- chore: update python module to latest ballista release (v.47) [#1279](https://github.com/apache/datafusion-ballista/pull/1279) (milenkovicm)
+- chore: update datafusion to 48.0.1 [#1284](https://github.com/apache/datafusion-ballista/pull/1284) (milenkovicm)
diff --git a/docs/source/changelog/49.0.0.md b/docs/source/changelog/49.0.0.md
new file mode 100644
index 0000000000..770f5f9f69
--- /dev/null
+++ b/docs/source/changelog/49.0.0.md
@@ -0,0 +1,58 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 49.0.0 Changelog
+
+[Full Changelog](https://github.com/apache/datafusion-ballista/compare/48.0.0...49.0.0)
+
+**Implemented enhancements:**
+
+- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm)
+- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay)
+
+**Fixed bugs:**
+
+- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm)
+- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez)
+- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm)
+- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm)
+- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm)
+- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm)
+
+**Merged pull requests:**
+
+- chore: update datafusion to 49 [#1285](https://github.com/apache/datafusion-ballista/pull/1285) (milenkovicm)
+- chore: Improve GitHub actions/python workflows [#1289](https://github.com/apache/datafusion-ballista/pull/1289) (Huy1Ng)
+- chore: update datafusion to 49.0.2 [#1298](https://github.com/apache/datafusion-ballista/pull/1298) (milenkovicm)
+- minor: make shuffle exec display consistent [#1299](https://github.com/apache/datafusion-ballista/pull/1299) (milenkovicm)
+- fix: fail job in case of serde error (pull-mode) [#1297](https://github.com/apache/datafusion-ballista/pull/1297) (milenkovicm)
+- fix: Ensure stage-level sort requirements are enforced in distributed planning [#1306](https://github.com/apache/datafusion-ballista/pull/1306) (metegenez)
+- fix: `UnresolvedShuffleExec` should support `with_new_children` [#1300](https://github.com/apache/datafusion-ballista/pull/1300) (milenkovicm)
+- fix: `ShuffleReader` should return statistics [#1302](https://github.com/apache/datafusion-ballista/pull/1302) (milenkovicm)
+- fix: Disable CollectLeft join as it is broken in ballista [#1301](https://github.com/apache/datafusion-ballista/pull/1301) (milenkovicm)
+- chore: notice and cargo deps cleanup [#1295](https://github.com/apache/datafusion-ballista/pull/1295) (milenkovicm)
+- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 [#1303](https://github.com/apache/datafusion-ballista/pull/1303) (dependabot[bot])
+- chore(deps): bump tracing-subscriber from 0.3.19 to 0.3.20 in /python [#1308](https://github.com/apache/datafusion-ballista/pull/1308) (dependabot[bot])
+- refactor: expand ClusterEventSender visibility to public [#1312](https://github.com/apache/datafusion-ballista/pull/1312) (Th824)
+- fix: Issue with JoinSelection and NestedLoopJoin::swap_inputs when stage is resolved [#1307](https://github.com/apache/datafusion-ballista/pull/1307) (milenkovicm)
+- minor: enable json write test [#1311](https://github.com/apache/datafusion-ballista/pull/1311) (milenkovicm)
+- feat: Feat force shuffle reader to read all files using arrow flight client [#1310](https://github.com/apache/datafusion-ballista/pull/1310) (milenkovicm)
+- feat: implement job data cleanup in pull-staged strategy #1219 [#1314](https://github.com/apache/datafusion-ballista/pull/1314) (KR-bluejay)
+- feat: Improve Remote Shuffle Read Speed and Resource Utilisation [#1318](https://github.com/apache/datafusion-ballista/pull/1318) (milenkovicm)
+- fix: Issue with `JoinSelection` and `CrossJoinExec` when stages have been resoled [#1322](https://github.com/apache/datafusion-ballista/pull/1322) (ZihuanLing)
diff --git a/docs/source/changelog/50.0.0.md b/docs/source/changelog/50.0.0.md
new file mode 100644
index 0000000000..cbae10092e
--- /dev/null
+++ b/docs/source/changelog/50.0.0.md
@@ -0,0 +1,46 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 50.0.0 Changelog
+
+**Implemented enhancements:**
+
+- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan)
+
+**Fixed bugs:**
+
+- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear)
+- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel)
+- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm)
+
+**Merged pull requests:**
+
+- infra: macos-13 is deprecated [#1324](https://github.com/apache/datafusion-ballista/pull/1324) (kevinjqliu)
+- chore: update to datafusion v50 [#1320](https://github.com/apache/datafusion-ballista/pull/1320) (milenkovicm)
+- chore: Update datafusion to 50.2 [#1326](https://github.com/apache/datafusion-ballista/pull/1326) (milenkovicm)
+- document private items and update docs CI [#1327](https://github.com/apache/datafusion-ballista/pull/1327) (killzoner)
+- fix: update REST API route syntax for axum 0.8 compatibility [#1330](https://github.com/apache/datafusion-ballista/pull/1330) (tomsanbear)
+- add msrvcheck [#1328](https://github.com/apache/datafusion-ballista/pull/1328) (killzoner)
+- fix: Executor does not panic if using unwritable work dir [#1332](https://github.com/apache/datafusion-ballista/pull/1332) (mach-kernel)
+- chore: Pinning versions of external actions. [#1334](https://github.com/apache/datafusion-ballista/pull/1334) (samueleresca)
+- chore: update python deps to 49 [#1335](https://github.com/apache/datafusion-ballista/pull/1335) (milenkovicm)
+- chore: update datafusion to 50.3 [#1336](https://github.com/apache/datafusion-ballista/pull/1336) (milenkovicm)
+- fix: failing documentation [#1339](https://github.com/apache/datafusion-ballista/pull/1339) (milenkovicm)
+- feat: make gRPC timeout configurations user-configurable [#1337](https://github.com/apache/datafusion-ballista/pull/1337) (CuteChuanChuan)
+- minor: change log level [#1340](https://github.com/apache/datafusion-ballista/pull/1340) (milenkovicm)
diff --git a/docs/source/changelog/51.0.0.md b/docs/source/changelog/51.0.0.md
new file mode 100644
index 0000000000..a6b125d48d
--- /dev/null
+++ b/docs/source/changelog/51.0.0.md
@@ -0,0 +1,45 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 51.0.0 Changelog
+
+**Implemented enhancements:**
+
+- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod)
+- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner)
+- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang)
+- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento)
+- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento)
+
+**Merged pull requests:**
+
+- minor: minor changes to release script/docs [#1342](https://github.com/apache/datafusion-ballista/pull/1342) (andygrove)
+- feat: Support distributed plan in `EXPLAIN` command [#1309](https://github.com/apache/datafusion-ballista/pull/1309) (danielhumanmod)
+- chore: update datafusion to 51.0 [#1345](https://github.com/apache/datafusion-ballista/pull/1345) (danielhumanmod)
+- doc: Add a note that datafusion may need to be downgraded after installing it [#1348](https://github.com/apache/datafusion-ballista/pull/1348) (martin-g)
+- minor: Make `DisplayAs` consistent and more readable for ShuffleExec [#1347](https://github.com/apache/datafusion-ballista/pull/1347) (milenkovicm)
+- minor: remove unnecessary clone functions [#1352](https://github.com/apache/datafusion-ballista/pull/1352) (mmooyyii)
+- feat: update rust edition to 2024 [#1355](https://github.com/apache/datafusion-ballista/pull/1355) (killzoner)
+- chore(doc): Clean up deployment docs. [#1354](https://github.com/apache/datafusion-ballista/pull/1354) (LouisBurke)
+- feat: capture more metrics in distributed_query [#1353](https://github.com/apache/datafusion-ballista/pull/1353) (PhVHoang)
+- doc: Fix plan translation example to use correct aggregation and column [#1362](https://github.com/apache/datafusion-ballista/pull/1362) (mattcuento)
+- chore: update ballista version to 51.0.0 (from 50.0.0) [#1363](https://github.com/apache/datafusion-ballista/pull/1363) (milenkovicm)
+- feat: Bump docker rust to `rust:1.92-trixie` [#1365](https://github.com/apache/datafusion-ballista/pull/1365) (mattcuento)
+- chore: Add missing public API documentation/comments [#1364](https://github.com/apache/datafusion-ballista/pull/1364) (killzoner)
+- feat: Scheduler supports `substrait` logical plan and remove deprecated `sql` support [#1360](https://github.com/apache/datafusion-ballista/pull/1360) (mattcuento)
diff --git a/docs/source/changelog/52.0.0.md b/docs/source/changelog/52.0.0.md
new file mode 100644
index 0000000000..eed5a10076
--- /dev/null
+++ b/docs/source/changelog/52.0.0.md
@@ -0,0 +1,92 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 52.0.0 Changelog
+
+**Performance related:**
+
+- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove)
+
+**Implemented enhancements:**
+
+- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner)
+- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove)
+- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove)
+- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm)
+- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod)
+- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg)
+- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento)
+- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc)
+- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove)
+- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm)
+- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento)
+- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner)
+- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm)
+- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm)
+- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm)
+
+**Fixed bugs:**
+
+- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm)
+- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner)
+- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm)
+- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g)
+
+**Documentation updates:**
+
+- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove)
+- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove)
+- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove)
+- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento)
+
+**Merged pull requests:**
+
+- feat: add config option for skipping arrow ipc read validation [#1374](https://github.com/apache/datafusion-ballista/pull/1374) (killzoner)
+- docs: fix outdated content in documentation [#1385](https://github.com/apache/datafusion-ballista/pull/1385) (andygrove)
+- restrict python CI to python directory [#1383](https://github.com/apache/datafusion-ballista/pull/1383) (Huy1Ng)
+- perf: optimize shuffle writer with buffered I/O and fix file size bug [#1386](https://github.com/apache/datafusion-ballista/pull/1386) (andygrove)
+- docs: use tpchgen-rs for TPC-H data generation [#1390](https://github.com/apache/datafusion-ballista/pull/1390) (andygrove)
+- feat: improve tpch benchmark CLI [#1391](https://github.com/apache/datafusion-ballista/pull/1391) (andygrove)
+- doc: Add Ballista extensions example to the docs. [#1382](https://github.com/apache/datafusion-ballista/pull/1382) (LouisBurke)
+- feat: Add sort-based shuffle implementation [#1389](https://github.com/apache/datafusion-ballista/pull/1389) (andygrove)
+- feat: New ballista python interface [#1338](https://github.com/apache/datafusion-ballista/pull/1338) (milenkovicm)
+- doc: add more details for protobuf extension [#1393](https://github.com/apache/datafusion-ballista/pull/1393) (LouisBurke)
+- feat: Add batch coalescing ability to shuffle reader exec [#1380](https://github.com/apache/datafusion-ballista/pull/1380) (danielhumanmod)
+- docs: add Jupyter notebook support documentation [#1399](https://github.com/apache/datafusion-ballista/pull/1399) (andygrove)
+- feat: Add arrow flight proxy to scheduler [#1351](https://github.com/apache/datafusion-ballista/pull/1351) (sebbegg)
+- chore: update datafusion to 52 [#1394](https://github.com/apache/datafusion-ballista/pull/1394) (killzoner)
+- feat: Creating SubstraitSchedulerClient and standalone Substrait examples [#1376](https://github.com/apache/datafusion-ballista/pull/1376) (mattcuento)
+- fix: compile issue after unsuccessful merge [#1402](https://github.com/apache/datafusion-ballista/pull/1402) (milenkovicm)
+- feat: Cluster RPC customisations to support TLS and custom headers [#1400](https://github.com/apache/datafusion-ballista/pull/1400) (phillipleblanc)
+- chore: Document ballista features in README.md [#1418](https://github.com/apache/datafusion-ballista/pull/1418) (mattcuento)
+- fix: prost build keda and TLS RPC example [#1429](https://github.com/apache/datafusion-ballista/pull/1429) (killzoner)
+- Improve sort-based shuffle: single spill file per partition and batch coalescing [#1431](https://github.com/apache/datafusion-ballista/pull/1431) (andygrove)
+- feat: add -c config override flag to tpch benchmark [#1435](https://github.com/apache/datafusion-ballista/pull/1435) (andygrove)
+- feat: Extract `execution_graph` to a trait [#1361](https://github.com/apache/datafusion-ballista/pull/1361) (milenkovicm)
+- chore: add confirmation before tarball is released [#1445](https://github.com/apache/datafusion-ballista/pull/1445) (milenkovicm)
+- minor: add test to cover IPC arrow file read [#1450](https://github.com/apache/datafusion-ballista/pull/1450) (milenkovicm)
+- feat: Add spark-compat mode to integrate datafusion-spark features au… [#1416](https://github.com/apache/datafusion-ballista/pull/1416) (mattcuento)
+- feat: add `Dataframe.cache()` factory (no planner handling) [#1420](https://github.com/apache/datafusion-ballista/pull/1420) (killzoner)
+- fix: remove `scheduler_config_spec.toml` as it is unused [#1462](https://github.com/apache/datafusion-ballista/pull/1462) (milenkovicm)
+- feat: Adaptive query execution (AQE) planner fundamentals [#1372](https://github.com/apache/datafusion-ballista/pull/1372) (milenkovicm)
+- feat: Make push scheduling policy default as it has lower latency [#1461](https://github.com/apache/datafusion-ballista/pull/1461) (milenkovicm)
+- minor: improve log statements [#1482](https://github.com/apache/datafusion-ballista/pull/1482) (milenkovicm)
+- chore: update datafusion to 52.2 and other deps to latest [#1483](https://github.com/apache/datafusion-ballista/pull/1483) (milenkovicm)
+- fix: Don't use `maxrows` as a "fetched rows" but calculate it from the batches [#1480](https://github.com/apache/datafusion-ballista/pull/1480) (martin-g)
+- feat: job scheduling with push based job status updates [#1478](https://github.com/apache/datafusion-ballista/pull/1478) (milenkovicm)
diff --git a/docs/source/changelog/53.0.0.md b/docs/source/changelog/53.0.0.md
new file mode 100644
index 0000000000..94a5bd1c26
--- /dev/null
+++ b/docs/source/changelog/53.0.0.md
@@ -0,0 +1,233 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Ballista 53.0.0 Changelog
+
+This release consists of 174 commits from 16 contributors. See credits at the end of this changelog for more information.
+
+**Fixed bugs:**
+
+- fix: remove unwrap from executor and improve task error handling [#1540](https://github.com/apache/datafusion-ballista/pull/1540) (eachsaj)
+- fix: handle None task slot in update_task_info after executor lost [#1523](https://github.com/apache/datafusion-ballista/pull/1523) (milenkovicm)
+- fix: [Python] serialize optimized logical plan to fix subquery support [#1586](https://github.com/apache/datafusion-ballista/pull/1586) (andygrove)
+- fix: allow S3 access without explicit credentials [#1584](https://github.com/apache/datafusion-ballista/pull/1584) (andygrove)
+- fix: route collect/show/to_pandas through Ballista cluster [#1585](https://github.com/apache/datafusion-ballista/pull/1585) (andygrove)
+- fix: propagate session config from Python client to Ballista cluster [#1592](https://github.com/apache/datafusion-ballista/pull/1592) (andygrove)
+- fix(python): ignore ballista-namespaced cluster_config keys locally [#1613](https://github.com/apache/datafusion-ballista/pull/1613) (andygrove)
+- fix: `df.write_` fix as it was broken after update [#1625](https://github.com/apache/datafusion-ballista/pull/1625) (milenkovicm)
+- fix(rest): remove unwrap and return 404 if executor does not exist [#1628](https://github.com/apache/datafusion-ballista/pull/1628) (milenkovicm)
+- fix(metrics): avoid stage metrics inflation by tracking partition snapshots [#1652](https://github.com/apache/datafusion-ballista/pull/1652) (danielhumanmod)
+- fix: compilation issue after merge [#1658](https://github.com/apache/datafusion-ballista/pull/1658) (milenkovicm)
+- fix: rest api calculates stage running time correctly [#1675](https://github.com/apache/datafusion-ballista/pull/1675) (milenkovicm)
+- fix: REST API does not show running jobs [#1703](https://github.com/apache/datafusion-ballista/pull/1703) (gittihub-jpg)
+- fix: no executor warning, correct prometheus feature name in TUI [#1698](https://github.com/apache/datafusion-ballista/pull/1698) (killzoner)
+
+**Implemented enhancements:**
+
+- [REST]: Cancelling a completed/failed job should return "false" [#1494](https://github.com/apache/datafusion-ballista/pull/1494) (martin-g)
+- feat: Add plain "status" field to the JobResponse [#1497](https://github.com/apache/datafusion-ballista/pull/1497) (martin-g)
+- feat: Expose Logical and Physical plan details in the REST API [#1498](https://github.com/apache/datafusion-ballista/pull/1498) (milenkovicm)
+- feat: (remote) shuffle reader cleanup [#1503](https://github.com/apache/datafusion-ballista/pull/1503) (milenkovicm)
+- feat: enable scheduler rest api by default [#1506](https://github.com/apache/datafusion-ballista/pull/1506) (milenkovicm)
+- feat: [REST] Return the job's start and end times [#1511](https://github.com/apache/datafusion-ballista/pull/1511) (martin-g)
+- feat: remove hash policy task distribution as not used [#1526](https://github.com/apache/datafusion-ballista/pull/1526) (eachsaj)
+- feat: remove full path from partition locations [#1527](https://github.com/apache/datafusion-ballista/pull/1527) (milenkovicm)
+- feat: add config to ExecutorEngine::create_query_stage_exec [#1542](https://github.com/apache/datafusion-ballista/pull/1542) (milenkovicm)
+- feat: Improve REST API adding task related info to job stages [#1543](https://github.com/apache/datafusion-ballista/pull/1543) (milenkovicm)
+- feat: Ballista Text User Interface [#1436](https://github.com/apache/datafusion-ballista/pull/1436) (martin-g)
+- feat: jupyter notebook support [#1513](https://github.com/apache/datafusion-ballista/pull/1513) (sandugood)
+- feat: ExecutionEngine::create_query_stage_exec accepts partition_id [#1556](https://github.com/apache/datafusion-ballista/pull/1556) (milenkovicm)
+- feat: make ballista client retry policy configurable [#1577](https://github.com/apache/datafusion-ballista/pull/1577) (eachsaj)
+- feat: Executor system & process metrics reporting [#1547](https://github.com/apache/datafusion-ballista/pull/1547) (sandugood)
+- feat: Scheduler config update [#1597](https://github.com/apache/datafusion-ballista/pull/1597) (sandugood)
+- feat: Support `EXPLAIN ANALYZE` in Ballista [#1567](https://github.com/apache/datafusion-ballista/pull/1567) (danielhumanmod)
+- feat: Add standalone shuffle writer benchmark that shuffles real Parquet input [#1600](https://github.com/apache/datafusion-ballista/pull/1600) (andygrove)
+- feat: defer sort-shuffle materialization with interleave_record_batch [#1598](https://github.com/apache/datafusion-ballista/pull/1598) (andygrove)
+- feat(executor): bound executor memory via --memory-pool-size [#1624](https://github.com/apache/datafusion-ballista/pull/1624) (andygrove)
+- feat(bench): show TPC-H query timings in seconds and add total time [#1641](https://github.com/apache/datafusion-ballista/pull/1641) (andygrove)
+- feat(aqe): Support sort-based shuffle writer in AQE [#1640](https://github.com/apache/datafusion-ballista/pull/1640) (danielhumanmod)
+- feat: rest api supports plan tree rendering [#1650](https://github.com/apache/datafusion-ballista/pull/1650) (sandugood)
+- feat: Cache ballista clients on executor [#1578](https://github.com/apache/datafusion-ballista/pull/1578) (milenkovicm)
+- feat(aqe): Lazy stage evaluation in AQE [#1649](https://github.com/apache/datafusion-ballista/pull/1649) (milenkovicm)
+- feat: move shuffle writer disk I/O off tokio worker threads [#1537](https://github.com/apache/datafusion-ballista/pull/1537) (hcrosse)
+- feat(scheduler): broadcast-style hash join for small-side joins [#1647](https://github.com/apache/datafusion-ballista/pull/1647) (andygrove)
+- feat: default to sort-merge join [#1651](https://github.com/apache/datafusion-ballista/pull/1651) (andygrove)
+- feat: TUI shows running job information [#1717](https://github.com/apache/datafusion-ballista/pull/1717) (milenkovicm)
+- feat(aqe): CoalescePartitionsRule — shuffle-partition coalescing on resolved stats [#1684](https://github.com/apache/datafusion-ballista/pull/1684) (metegenez)
+- feat: TUI make task popup scrollable [#1725](https://github.com/apache/datafusion-ballista/pull/1725) (milenkovicm)
+- feat(tui): Use separate areas for the table and its associated scrollbar [#1729](https://github.com/apache/datafusion-ballista/pull/1729) (martin-g)
+
+**Documentation updates:**
+
+- docs: document hash-based and sort-based shuffle implementations [#1595](https://github.com/apache/datafusion-ballista/pull/1595) (andygrove)
+- docs: improve Python documentation structure [#1579](https://github.com/apache/datafusion-ballista/pull/1579) (andygrove)
+- docs: add Python client build-from-source section to contributor guide [#1620](https://github.com/apache/datafusion-ballista/pull/1620) (andygrove)
+- chore: remove NYC Taxi benchmark [#1644](https://github.com/apache/datafusion-ballista/pull/1644) (andygrove)
+- docs: document experimental Adaptive Query Execution in tuning guide [#1645](https://github.com/apache/datafusion-ballista/pull/1645) (andygrove)
+- build(bench): rework docker-compose TPC-H stack [#1646](https://github.com/apache/datafusion-ballista/pull/1646) (andygrove)
+- docs: add Ballista TUI documentation [#1593](https://github.com/apache/datafusion-ballista/pull/1593) (goingforstudying-ctrl)
+- [TUI] Show executor's details in a popup [#1670](https://github.com/apache/datafusion-ballista/pull/1670) (martin-g)
+- [TUI] Add a config setting for rendering job stage's plan as a tree [#1704](https://github.com/apache/datafusion-ballista/pull/1704) (martin-g)
+- [TUI] Add support for horizontal scrolling to the job/stage plan popups [#1711](https://github.com/apache/datafusion-ballista/pull/1711) (martin-g)
+- [TUI] Add screenshots of the TUI application in README/cli.md [#1714](https://github.com/apache/datafusion-ballista/pull/1714) (martin-g)
+
+**Other:**
+
+- CI: Add CodeQL workflow for GitHub Actions security scanning [#1484](https://github.com/apache/datafusion-ballista/pull/1484) (kevinjqliu)
+- ci: Harden labeler workflow, remove unnecessary checkout from pull_request_target job [#1487](https://github.com/apache/datafusion-ballista/pull/1487) (kevinjqliu)
+- chore(deps): bump tokio from 1.49.0 to 1.50.0 [#1490](https://github.com/apache/datafusion-ballista/pull/1490) (dependabot[bot])
+- chore(deps): bump actions/setup-node from 6.2.0 to 6.3.0 [#1489](https://github.com/apache/datafusion-ballista/pull/1489) (dependabot[bot])
+- ci: add take and stale [#1488](https://github.com/apache/datafusion-ballista/pull/1488) (kevinjqliu)
+- chore(deps): bump uuid from 1.21.0 to 1.22.0 [#1492](https://github.com/apache/datafusion-ballista/pull/1492) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.32.5 to 4.32.6 [#1491](https://github.com/apache/datafusion-ballista/pull/1491) (dependabot[bot])
+- chore(deps): bump libc from 0.2.182 to 0.2.183 [#1495](https://github.com/apache/datafusion-ballista/pull/1495) (dependabot[bot])
+- chore(deps): bump graphviz-rust from 0.9.6 to 0.9.7 [#1496](https://github.com/apache/datafusion-ballista/pull/1496) (dependabot[bot])
+- chore(deps): bump quinn-proto from 0.11.13 to 0.11.14 [#1499](https://github.com/apache/datafusion-ballista/pull/1499) (dependabot[bot])
+- chore(deps): bump quinn-proto from 0.11.13 to 0.11.14 in /python [#1500](https://github.com/apache/datafusion-ballista/pull/1500) (dependabot[bot])
+- chore(deps): bump tempfile from 3.26.0 to 3.27.0 [#1501](https://github.com/apache/datafusion-ballista/pull/1501) (dependabot[bot])
+- chore(deps): cargo update deps [#1502](https://github.com/apache/datafusion-ballista/pull/1502) (milenkovicm)
+- chore(deps): bump once_cell from 1.21.3 to 1.21.4 [#1505](https://github.com/apache/datafusion-ballista/pull/1505) (dependabot[bot])
+- chore(deps): bump clap from 4.5.60 to 4.6.0 [#1504](https://github.com/apache/datafusion-ballista/pull/1504) (dependabot[bot])
+- minor: task scheduling policy config cleanup [#1507](https://github.com/apache/datafusion-ballista/pull/1507) (milenkovicm)
+- minor: address task policy config comments [#1508](https://github.com/apache/datafusion-ballista/pull/1508) (milenkovicm)
+- chore(deps): bump tracing-subscriber from 0.3.22 to 0.3.23 [#1510](https://github.com/apache/datafusion-ballista/pull/1510) (dependabot[bot])
+- minor: [REST] add datafusion version info [#1512](https://github.com/apache/datafusion-ballista/pull/1512) (milenkovicm)
+- chore(deps): bump lz4_flex from 0.12.0 to 0.12.1 in /python [#1514](https://github.com/apache/datafusion-ballista/pull/1514) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.32.6 to 4.33.0 [#1515](https://github.com/apache/datafusion-ballista/pull/1515) (dependabot[bot])
+- chore(deps): bump rustls-webpki from 0.103.9 to 0.103.10 in /python [#1517](https://github.com/apache/datafusion-ballista/pull/1517) (dependabot[bot])
+- chore(deps): bump lz4_flex from 0.12.0 to 0.12.1 [#1518](https://github.com/apache/datafusion-ballista/pull/1518) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.33.0 to 4.34.1 [#1519](https://github.com/apache/datafusion-ballista/pull/1519) (dependabot[bot])
+- chore(deps): bump rustls-webpki from 0.103.9 to 0.103.10 [#1521](https://github.com/apache/datafusion-ballista/pull/1521) (dependabot[bot])
+- ci: pin third-party actions to Apache-approved SHAs [#1516](https://github.com/apache/datafusion-ballista/pull/1516) (kevinjqliu)
+- chore(deps): bump env_logger from 0.11.9 to 0.11.10 [#1522](https://github.com/apache/datafusion-ballista/pull/1522) (dependabot[bot])
+- chore(deps): update to datafusion v.53 [#1486](https://github.com/apache/datafusion-ballista/pull/1486) (milenkovicm)
+- chore(deps): bump insta from 1.46.3 to 1.47.0 [#1524](https://github.com/apache/datafusion-ballista/pull/1524) (dependabot[bot])
+- chore(deps): bump uuid from 1.22.0 to 1.23.0 [#1525](https://github.com/apache/datafusion-ballista/pull/1525) (dependabot[bot])
+- chore: update datafusion proto [#1528](https://github.com/apache/datafusion-ballista/pull/1528) (milenkovicm)
+- minor: cleanup scheduler clap configuration [#1529](https://github.com/apache/datafusion-ballista/pull/1529) (milenkovicm)
+- chore(deps): bump insta from 1.47.0 to 1.47.1 [#1532](https://github.com/apache/datafusion-ballista/pull/1532) (dependabot[bot])
+- chore(deps): bump ctor from 0.6.3 to 0.8.0 [#1534](https://github.com/apache/datafusion-ballista/pull/1534) (dependabot[bot])
+- chore(deps): bump md-5 from 0.10.6 to 0.11.0 [#1533](https://github.com/apache/datafusion-ballista/pull/1533) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.34.1 to 4.35.1 [#1530](https://github.com/apache/datafusion-ballista/pull/1530) (dependabot[bot])
+- chore(deps): bump insta from 1.47.1 to 1.47.2 [#1535](https://github.com/apache/datafusion-ballista/pull/1535) (dependabot[bot])
+- chore(deps): bump libc from 0.2.183 to 0.2.184 [#1538](https://github.com/apache/datafusion-ballista/pull/1538) (dependabot[bot])
+- chore(deps): bump tokio from 1.50.0 to 1.51.0 [#1544](https://github.com/apache/datafusion-ballista/pull/1544) (dependabot[bot])
+- chore(deps): bump tui-big-text from 0.8.3 to 0.8.4 [#1546](https://github.com/apache/datafusion-ballista/pull/1546) (dependabot[bot])
+- chore(deps): bump tokio from 1.51.0 to 1.51.1 [#1545](https://github.com/apache/datafusion-ballista/pull/1545) (dependabot[bot])
+- chore(deps): bump ctor from 0.8.0 to 0.9.1 [#1548](https://github.com/apache/datafusion-ballista/pull/1548) (dependabot[bot])
+- chore(deps): bump ctor from 0.9.1 to 0.10.0 [#1549](https://github.com/apache/datafusion-ballista/pull/1549) (dependabot[bot])
+- chore(deps): bump rustls from 0.23.37 to 0.23.38 [#1550](https://github.com/apache/datafusion-ballista/pull/1550) (dependabot[bot])
+- chore(deps): bump libc from 0.2.184 to 0.2.185 [#1553](https://github.com/apache/datafusion-ballista/pull/1553) (dependabot[bot])
+- chore(deps): bump rand from 0.9.2 to 0.9.4 in /python [#1552](https://github.com/apache/datafusion-ballista/pull/1552) (dependabot[bot])
+- chore(deps): bump tokio from 1.51.1 to 1.52.0 [#1555](https://github.com/apache/datafusion-ballista/pull/1555) (dependabot[bot])
+- chore(deps): bump axum from 0.8.8 to 0.8.9 [#1554](https://github.com/apache/datafusion-ballista/pull/1554) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.35.1 to 4.35.2 [#1557](https://github.com/apache/datafusion-ballista/pull/1557) (dependabot[bot])
+- chore: Fix Clippy issues with Rust 1.95.0 [#1558](https://github.com/apache/datafusion-ballista/pull/1558) (martin-g)
+- chore: update deps [#1561](https://github.com/apache/datafusion-ballista/pull/1561) (milenkovicm)
+- chore(deps): bump actions/setup-node from 6.3.0 to 6.4.0 [#1562](https://github.com/apache/datafusion-ballista/pull/1562) (dependabot[bot])
+- chore(deps): bump mimalloc from 0.1.48 to 0.1.49 [#1566](https://github.com/apache/datafusion-ballista/pull/1566) (dependabot[bot])
+- chore(deps): bump aws-config from 1.8.15 to 1.8.16 [#1564](https://github.com/apache/datafusion-ballista/pull/1564) (dependabot[bot])
+- chore(deps): bump rand from 0.9.4 to 0.10.1 [#1565](https://github.com/apache/datafusion-ballista/pull/1565) (dependabot[bot])
+- chore(deps): bump ctor from 0.10.0 to 0.10.1 [#1571](https://github.com/apache/datafusion-ballista/pull/1571) (dependabot[bot])
+- chore(deps): bump mimalloc from 0.1.49 to 0.1.50 [#1570](https://github.com/apache/datafusion-ballista/pull/1570) (dependabot[bot])
+- chore(deps): bump rustls from 0.23.38 to 0.23.39 [#1569](https://github.com/apache/datafusion-ballista/pull/1569) (dependabot[bot])
+- chore(deps): bump libc from 0.2.185 to 0.2.186 [#1572](https://github.com/apache/datafusion-ballista/pull/1572) (dependabot[bot])
+- [TUI] Change the key binding for job's plans [#1573](https://github.com/apache/datafusion-ballista/pull/1573) (martin-g)
+- chore(deps): bump rustls-webpki from 0.103.12 to 0.103.13 [#1576](https://github.com/apache/datafusion-ballista/pull/1576) (dependabot[bot])
+- chore(deps): bump rustls-webpki from 0.103.10 to 0.103.13 in /python [#1575](https://github.com/apache/datafusion-ballista/pull/1575) (dependabot[bot])
+- minor: make config naming convention consistent [#1580](https://github.com/apache/datafusion-ballista/pull/1580) (milenkovicm)
+- chore: Remove unnecessary optimizer rules due to datafusion upgrade to v53 [#1594](https://github.com/apache/datafusion-ballista/pull/1594) (sandugood)
+- chore(deps): bump reqwest from 0.13.2 to 0.13.3 [#1605](https://github.com/apache/datafusion-ballista/pull/1605) (dependabot[bot])
+- ci: drop Intel macOS Python wheel build [#1612](https://github.com/apache/datafusion-ballista/pull/1612) (andygrove)
+- chore(deps): bump ctor from 0.10.1 to 0.11.1 [#1618](https://github.com/apache/datafusion-ballista/pull/1618) (dependabot[bot])
+- chore(deps): bump rustls from 0.23.39 to 0.23.40 [#1619](https://github.com/apache/datafusion-ballista/pull/1619) (dependabot[bot])
+- chore: update python deps to ballista and datafusion 52 [#1590](https://github.com/apache/datafusion-ballista/pull/1590) (andygrove)
+- feat(sort-shuffle): byte-copy spill files and enable block-IO transport [#1615](https://github.com/apache/datafusion-ballista/pull/1615) (andygrove)
+- Use BallistaSessionContext instead of BallistaBuilder in tpch.py [#1621](https://github.com/apache/datafusion-ballista/pull/1621) (martin-g)
+- feat(sort-shuffle): enable sort-based shuffle by default [#1623](https://github.com/apache/datafusion-ballista/pull/1623) (andygrove)
+- perf(sort-shuffle): fix performance regression caused by datafusion upgrade [#1626](https://github.com/apache/datafusion-ballista/pull/1626) (andygrove)
+- chore(deps): bump ctor from 0.11.1 to 0.12.0 [#1639](https://github.com/apache/datafusion-ballista/pull/1639) (dependabot[bot])
+- fix(sort-shuffle): bound writer memory with per-task spill threshold [#1636](https://github.com/apache/datafusion-ballista/pull/1636) (andygrove)
+- chore(deps): bump graphviz-rust from 0.9.7 to 0.9.8 [#1655](https://github.com/apache/datafusion-ballista/pull/1655) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.35.2 to 4.35.3 [#1653](https://github.com/apache/datafusion-ballista/pull/1653) (dependabot[bot])
+- chore(deps): bump tokio from 1.52.1 to 1.52.2 [#1657](https://github.com/apache/datafusion-ballista/pull/1657) (dependabot[bot])
+- chore(deps): bump actions/labeler from 6.0.1 to 6.1.0 [#1659](https://github.com/apache/datafusion-ballista/pull/1659) (dependabot[bot])
+- [TUI] Show job's stages and their tasks [#1574](https://github.com/apache/datafusion-ballista/pull/1574) (martin-g)
+- chore(deps): bump tonic-build from 0.14.5 to 0.14.6 [#1662](https://github.com/apache/datafusion-ballista/pull/1662) (dependabot[bot])
+- chore(deps): bump tonic from 0.14.5 to 0.14.6 [#1661](https://github.com/apache/datafusion-ballista/pull/1661) (dependabot[bot])
+- chore(deps): bump astral-tokio-tar from 0.6.0 to 0.6.1 [#1664](https://github.com/apache/datafusion-ballista/pull/1664) (dependabot[bot])
+- chore(deps): bump ctor from 0.12.0 to 1.0.2 [#1663](https://github.com/apache/datafusion-ballista/pull/1663) (dependabot[bot])
+- chore(deps): bump github/codeql-action from 4.35.3 to 4.35.4 [#1665](https://github.com/apache/datafusion-ballista/pull/1665) (dependabot[bot])
+- chore(deps): bump ctor from 1.0.2 to 1.0.3 [#1668](https://github.com/apache/datafusion-ballista/pull/1668) (dependabot[bot])
+- chore(deps): bump tonic-prost from 0.14.5 to 0.14.6 [#1667](https://github.com/apache/datafusion-ballista/pull/1667) (dependabot[bot])
+- chore(deps): bump tonic-prost-build from 0.14.5 to 0.14.6 [#1666](https://github.com/apache/datafusion-ballista/pull/1666) (dependabot[bot])
+- [TUI] Configurable tick interval [#1669](https://github.com/apache/datafusion-ballista/pull/1669) (martin-g)
+- minor: add pending stage indicator for `AdaptiveDatafusionExec` [#1672](https://github.com/apache/datafusion-ballista/pull/1672) (milenkovicm)
+- chore(deps): bump ctor from 1.0.3 to 1.0.4 [#1680](https://github.com/apache/datafusion-ballista/pull/1680) (dependabot[bot])
+- chore(deps): bump tokio from 1.52.2 to 1.52.3 [#1678](https://github.com/apache/datafusion-ballista/pull/1678) (dependabot[bot])
+- minor: change parameter ordering in `AdaptivePlanner::try_new_with_optimizers` [#1687](https://github.com/apache/datafusion-ballista/pull/1687) (milenkovicm)
+- chore(deps): bump ctor from 1.0.4 to 1.0.5 [#1694](https://github.com/apache/datafusion-ballista/pull/1694) (dependabot[bot])
+- Fix REST API panic on job list/detail when end_time < start_time [#1693](https://github.com/apache/datafusion-ballista/pull/1693) (abhinavgautam01)
+- Right align all numeric columns in the TUI tables [#1695](https://github.com/apache/datafusion-ballista/pull/1695) (martin-g)
+- fix(join-selection): guard CollectLeft swap when right has multiple partitions [#1691](https://github.com/apache/datafusion-ballista/pull/1691) (andygrove)
+- Minor improvements for #1675 [#1686](https://github.com/apache/datafusion-ballista/pull/1686) (martin-g)
+- [CLI/TUI] Use only tracing crate for logging in CLI and TUI [#1697](https://github.com/apache/datafusion-ballista/pull/1697) (martin-g)
+- chore(deps): bump urllib3 from 2.6.3 to 2.7.0 in /python [#1699](https://github.com/apache/datafusion-ballista/pull/1699) (dependabot[bot])
+- chore(deps): bump pyjwt from 2.10.1 to 2.12.0 in /python [#1700](https://github.com/apache/datafusion-ballista/pull/1700) (dependabot[bot])
+- chore(deps): bump pytest from 9.0.2 to 9.0.3 in /python [#1702](https://github.com/apache/datafusion-ballista/pull/1702) (dependabot[bot])
+- minor: change log level for few statements [#1706](https://github.com/apache/datafusion-ballista/pull/1706) (milenkovicm)
+- chore(deps): bump config from 0.15.22 to 0.15.23 [#1709](https://github.com/apache/datafusion-ballista/pull/1709) (dependabot[bot])
+- Saturate scheduler job elapsed time [#1708](https://github.com/apache/datafusion-ballista/pull/1708) (MukundaKatta)
+- [TUI] Executor's id is not a numeric column. It should be center aligned [#1713](https://github.com/apache/datafusion-ballista/pull/1713) (martin-g)
+- Make use of Swatinem/rust-cache to make the CI workflows faster [#1705](https://github.com/apache/datafusion-ballista/pull/1705) (martin-g)
+- Merge Executor's brief and extended properties [#1716](https://github.com/apache/datafusion-ballista/pull/1716) (martin-g)
+- [INFRA] Set up default rulesets for default and release branches [#1715](https://github.com/apache/datafusion-ballista/pull/1715) (asf-gitbox-commits)
+- chore(deps): bump github/codeql-action from 4.35.4 to 4.35.5 [#1719](https://github.com/apache/datafusion-ballista/pull/1719) (dependabot[bot])
+- chore(deps): bump ctor from 1.0.5 to 1.0.6 [#1720](https://github.com/apache/datafusion-ballista/pull/1720) (dependabot[bot])
+- chore(deps): bump dashmap from 6.1.0 to 6.2.1 [#1721](https://github.com/apache/datafusion-ballista/pull/1721) (dependabot[bot])
+- ci: add TPC-H SF10 workflow [#1688](https://github.com/apache/datafusion-ballista/pull/1688) (andygrove)
+- minor: [TUI] Extract a helper method for the table/scrollbar area splitter [#1730](https://github.com/apache/datafusion-ballista/pull/1730) (martin-g)
+- minor: [TUI] Sort the metrics before rendering them [#1731](https://github.com/apache/datafusion-ballista/pull/1731) (martin-g)
+- chore: Do not run -cli tests twice [#1732](https://github.com/apache/datafusion-ballista/pull/1732) (martin-g)
+- chore: generate changelog for ballista 53 [#1735](https://github.com/apache/datafusion-ballista/pull/1735) (milenkovicm)
+
+## Credits
+
+Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
+
+```
+    74	dependabot[bot]
+    29	Marko Milenković
+    25	Andy Grove
+    23	Martin Grigorov
+     5	alexander domenti
+     4	Kevin Liu
+     3	Daniel Tu
+     3	Saj
+     1	Abhinav Gautam
+     1	Harrison Crosse
+     1	Mukunda Rao Katta
+     1	The Apache Software Foundation
+     1	gittihub-jpg
+     1	goingforstudying-ctrl
+     1	jgrim
+     1	mete
+```
+
+Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
diff --git a/docs/source/changelog/index.md b/docs/source/changelog/index.md
new file mode 100644
index 0000000000..653f006729
--- /dev/null
+++ b/docs/source/changelog/index.md
@@ -0,0 +1,46 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Changelog
+
+Per-release change logs for Apache DataFusion Ballista.
+
+```{toctree}
+:maxdepth: 1
+
+53.0.0
+52.0.0
+51.0.0
+50.0.0
+49.0.0
+48.0.0
+47.0.0
+46.0.0
+45.0.0
+44.0.0
+43.0.0
+0.12.0
+0.11.0
+0.10.0
+0.9.0
+0.8.0
+0.7.0
+0.6.0
+0.5.0
+```
diff --git a/docs/source/contributors-guide/architecture.md b/docs/source/contributors-guide/architecture.md
index b25321d040..8a7ff10636 100644
--- a/docs/source/contributors-guide/architecture.md
+++ b/docs/source/contributors-guide/architecture.md
@@ -199,3 +199,65 @@ and [ShuffleReaderExec] operators.
 
 [shufflewriterexec]: https://github.com/apache/datafusion-ballista/blob/main/ballista/core/src/execution_plans/shuffle_writer.rs
 [shufflereaderexec]: https://github.com/apache/datafusion-ballista/blob/main/ballista/core/src/execution_plans/shuffle_reader.rs
+
+## Adaptive Query Execution (AQE)
+
+The scheduling described above is _static_: the full set of query stages is computed once, at job
+submission time, from the plan-time cost estimates. Adaptive Query Execution (AQE) is an experimental
+alternative in which the stage DAG is built _incrementally_, re-optimizing the remaining plan using the
+exact row counts and byte sizes observed as each shuffle stage completes. It is disabled by default and
+enabled with the `ballista.planner.adaptive.enabled` configuration setting.
+
+### Static vs adaptive execution graphs
+
+Job scheduling revolves around an `ExecutionGraph`, which translates a physical plan into a set of stages.
+There are two implementations, selected by configuration when a job is submitted:
+
+- **`StaticExecutionGraph`** — the default. All stages are planned up front by the `DistributedPlanner`,
+  which returns a static list of stages.
+- **`AdaptiveExecutionGraph`** — the adaptive counterpart. It is driven by the `AdaptivePlanner`, which runs
+  a set of pluggable physical optimizer rules after each stage completes and returns only the stages that are
+  currently runnable. Because decisions are deferred, stages come back already resolved, so the
+  `UnResolved` stage state used by the static path is not needed here.
+
+Keeping the two graphs side by side lets the adaptive implementation mature without destabilizing the static
+path.
+
+### How re-optimization works
+
+As each shuffle stage finishes, its exact output statistics are attached to the remaining plan and the
+optimizer rules are re-run. This is the key difference from DataFusion's normal single-shot physical
+optimization, where each rule runs once per plan. Running the full rule set repeatedly means the rules must be
+**idempotent** — otherwise they would keep inserting redundant exec nodes on each pass.
+
+Two adaptive optimizations are implemented today:
+
+- **Join reordering** — runtime row counts are used so the smaller side drives the join.
+- **Empty stage elimination** — when a completed stage produces zero rows, its downstream exchange is
+  replaced with an empty execution node and emptiness is propagated up the plan so dependent stages are
+  skipped entirely.
+
+### Code layout
+
+The adaptive scheduler code lives under [`ballista/scheduler/src/state/aqe`], with the
+`AdaptiveExecutionGraph` in `mod.rs`, the planner in `planner.rs`, and the optimizer rules in
+`optimizer_rule.rs`.
+
+### Current limitations
+
+The adaptive path currently covers the happy path only. Known gaps, each with a tracking issue:
+
+- Executor failure handling on the AQE path ([#1986])
+- Dynamic coalescing of shuffle partitions ([#1987])
+- Switching from hash join to sort-merge join based on runtime statistics ([#1988])
+- Switching from streaming aggregation to hash aggregation based on runtime statistics ([#1989])
+
+Design and progress are tracked in the AQE epic ([#1359]). See also the
+[AQE section of the tuning guide](../user-guide/tuning-guide.md) for the user-facing description.
+
+[`ballista/scheduler/src/state/aqe`]: https://github.com/apache/datafusion-ballista/tree/main/ballista/scheduler/src/state/aqe
+[#1359]: https://github.com/apache/datafusion-ballista/issues/1359
+[#1986]: https://github.com/apache/datafusion-ballista/issues/1986
+[#1987]: https://github.com/apache/datafusion-ballista/issues/1987
+[#1988]: https://github.com/apache/datafusion-ballista/issues/1988
+[#1989]: https://github.com/apache/datafusion-ballista/issues/1989
diff --git a/docs/source/contributors-guide/benchmarking.md b/docs/source/contributors-guide/benchmarking.md
new file mode 100644
index 0000000000..a066f0aa05
--- /dev/null
+++ b/docs/source/contributors-guide/benchmarking.md
@@ -0,0 +1,351 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Benchmarking
+
+This page describes how Ballista is benchmarked at scale, and records the current
+results. It is aimed at contributors who want to reproduce a number, understand why
+a comparison is set up the way it is, or add a result of their own.
+
+The benchmark used here is derived from TPC-H. For running TPC-H locally at small
+scale factors, see [`benchmarks/README.md`][bench-readme] in the repository; this
+page covers the multi-node, large-scale-factor setup and the cross-engine
+comparisons.
+
+[bench-readme]: https://github.com/apache/datafusion-ballista/blob/main/benchmarks/README.md
+
+## What we measure, and why
+
+Ballista's central performance question is **how evenly work is spread across
+executor tasks**. A distributed query is only as fast as its slowest stage, and a
+stage is only as fast as its slowest task, so a query whose work concentrates onto
+a few partitions will underperform no matter how fast the underlying operators are.
+Benchmarks are therefore run at a scale where that imbalance actually shows up.
+
+Two configurations are always measured, because they select **different planners**
+with materially different join behaviour:
+
+- **AQE off** (`ballista.planner.adaptive.enabled=false`, the default) — the static
+  `DefaultDistributedPlanner`.
+- **AQE on** (`ballista.planner.adaptive.enabled=true`, experimental) — the adaptive
+  planner, which can re-plan stages using runtime statistics.
+
+A change that helps one planner can be a no-op or a regression under the other, so
+results are reported for both rather than for whichever looks better.
+
+## Environment
+
+The results on this page currently come from a **small homelab cluster**: two
+bare-metal nodes running Kubernetes, with the TPC-H data staged on node-local disk.
+This is a deliberate starting point rather than a final destination. A two-node
+cluster with local disk removes as many confounds as possible — no object-store
+latency, no cloud network variance, no noisy neighbours — so that when a query is
+slow, the cause is Ballista and not the environment.
+
+The intent is to **move these benchmarks to AWS with data in S3** once the results
+here are good. That environment is the one users actually run, and it exercises
+things a homelab cannot: object-store reads instead of local disk, higher and more
+variable network latency between executors, and larger executor counts. Some of
+Ballista's behaviour is expected to change there — a shuffle that is cheap over a
+local link is not cheap over a cloud network, and object-store reads make scan
+parallelism matter differently.
+
+So results on this page should be read as **relative comparisons on controlled
+hardware**, useful for "did this change help", not as absolute throughput numbers
+for a cloud deployment.
+
+## Reference cluster
+
+All results on this page use the following shape. It is a reference point, not a
+requirement — the commands below work on any cluster, but numbers are only
+comparable when the shape matches.
+
+|                     |                                            |
+| ------------------- | ------------------------------------------ |
+| Executors           | 2, one per physical node                   |
+| Per executor        | 8 cores, 56 GiB, `--memory-pool-size=48GB` |
+| Per task slot       | 8 concurrent tasks → 6 GB pool each        |
+| Scheduler           | 1                                          |
+| Data                | TPC-H SF1000 Parquet, node-local disk      |
+| `target_partitions` | 32                                         |
+
+Two details matter more than they look:
+
+- **Executors are spread one per node.** Packing two executors onto one node makes
+  them contend for the same disk and memory bandwidth, which is measuring the host,
+  not the engine.
+- **Data is on node-local disk, not object storage.** With a modest network between
+  nodes, reading from object storage makes the interconnect the bottleneck and the
+  engine comparison becomes an I/O comparison.
+
+The memory pool is deliberately set below the container limit. Ballista splits
+`--memory-pool-size` into one `FairSpillPool` per task slot, and that accounting
+does not cover every allocation, so leaving headroom between the pool and the
+container limit avoids the container being killed outright instead of reporting a
+graceful resource error.
+
+## Queries
+
+All engines run the **same SQL**, which is what makes a cross-engine comparison
+apples-to-apples.
+
+The shared set is the TPC-H queries from [apache/datafusion-benchmarks][dfb]
+(SQLBench-H). Ballista's bundled queries in `benchmarks/queries/` are the classic
+TPC-H phrasing and differ in places, so the shared set is overlaid over them for a
+comparison run.
+
+Spark and Comet are driven through Comet's benchmark harness, which reads its own
+bundled copy of the queries (labelled CometBench-H). That copy is textually
+identical to the SQLBench-H set — all 22 queries match once the licence header
+comment is ignored — so the engines are executing the same statements even though
+they load them from different paths. Worth re-checking if either set is ever
+regenerated.
+
+[dfb]: https://github.com/apache/datafusion-benchmarks
+
+## Running the benchmark
+
+### Ballista
+
+Start a scheduler and one executor per node:
+
+```sh
+# scheduler
+ballista-scheduler --bind-host 0.0.0.0 --bind-port 50050
+
+# executor (one per node; --concurrent-tasks defaults to the detected core count)
+ballista-executor \
+  --bind-host 0.0.0.0 \
+  --scheduler-host <scheduler> --scheduler-port 50050 \
+  --memory-pool-size=48GB \
+  --work-dir /work \
+  --client-ttl=60
+```
+
+`--client-ttl=60` enables shuffle-client connection caching. Without it every
+shuffle fetch opens a new connection, and a high `target_partitions` can exhaust
+ephemeral ports.
+
+Run a query:
+
+```sh
+tpch benchmark ballista \
+  --host <scheduler> --port 50050 \
+  --query 18 \
+  --path /mnt/bigdata/tpch/sf1000 --format parquet \
+  --partitions 32 --iterations 1 \
+  -c datafusion.optimizer.prefer_hash_join=false \
+  -c datafusion.optimizer.enable_dynamic_filter_pushdown=false \
+  -c ballista.planner.adaptive.enabled=true
+```
+
+Omit `--query` to run all 22. Flip `ballista.planner.adaptive.enabled` to `false`
+for the AQE-off number.
+
+Note `datafusion.optimizer.enable_dynamic_filter_pushdown=false`: DataFusion's
+dynamic filter pushdown assumes single-process execution and can deadlock
+distributed execution, so it is pinned off for benchmark runs.
+
+### Spark
+
+Spark runs the same queries via `tpcbench.py` from
+[apache/datafusion-benchmarks][dfb]:
+
+```sh
+spark-submit \
+  --master <master> \
+  --conf spark.executor.instances=2 \
+  --conf spark.executor.cores=8 \
+  --conf spark.executor.memory=32G \
+  --conf spark.executor.memoryOverhead=8G \
+  --conf spark.memory.offHeap.enabled=true \
+  --conf spark.memory.offHeap.size=16g \
+  --conf spark.comet.enabled=false \
+  --conf spark.shuffle.manager=org.apache.spark.shuffle.sort.SortShuffleManager \
+  tpcbench.py \
+    --benchmark tpch \
+    --data /mnt/bigdata/tpch/sf1000 \
+    --format parquet \
+    --iterations 1 \
+    --query 18
+```
+
+`spark.comet.enabled=false` and the stock `SortShuffleManager` are what make this a
+**vanilla** Spark baseline, in case the image being used ships Comet.
+
+### Comet
+
+[Apache DataFusion Comet][comet] accelerates Spark by translating supported
+operators to DataFusion. Same queries, same sizing; the difference is the plugin,
+the Comet shuffle manager, and the jar on the classpath:
+
+```sh
+spark-submit \
+  --master <master> \
+  --jars $COMET_JAR --driver-class-path $COMET_JAR \
+  --conf spark.executor.extraClassPath=$COMET_JAR \
+  --conf spark.plugins=org.apache.spark.CometPlugin \
+  --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
+  --conf spark.comet.exec.replaceSortMergeJoin=true \
+  --conf spark.comet.exec.memoryPool=fair_unified \
+  --conf spark.comet.exec.memoryPool.fraction=0.8 \
+  --conf spark.executor.instances=2 \
+  --conf spark.executor.cores=8 \
+  --conf spark.executor.memory=32G \
+  --conf spark.executor.memoryOverhead=8G \
+  --conf spark.memory.offHeap.enabled=true \
+  --conf spark.memory.offHeap.size=16g \
+  tpcbench.py \
+    --benchmark tpch \
+    --data /mnt/bigdata/tpch/sf1000 \
+    --format parquet \
+    --iterations 1 \
+    --query 18
+```
+
+[comet]: https://datafusion.apache.org/comet/
+
+## Why compare against Comet
+
+Comet is the most informative comparison available to Ballista, because **Comet and
+Ballista execute DataFusion physical plans using the same DataFusion operators**.
+The scan, filter, join, and aggregate implementations doing the work are largely
+shared code. When Ballista and Comet diverge on a query, the difference therefore
+points at what is _not_ shared — how work is distributed, scheduled, shuffled, and
+bounded by memory — rather than at the speed of the operators themselves. That is
+precisely the surface Ballista is trying to improve, which makes the comparison
+diagnostic rather than merely competitive.
+
+That said, **the two do not necessarily run the same plan shape**, and the numbers
+should not be read as an operator-level A/B:
+
+- **Different planners produce the plan.** Comet accelerates a plan that _Spark's_
+  optimizer produced: Spark chooses the join order and join strategies, and
+  **Spark's AQE** coalesces shuffle partitions, converts joins, and splits skewed
+  partitions at runtime. Ballista plans with DataFusion's optimizer and, when
+  enabled, its own experimental AQE. Spark's AQE is a mature implementation and
+  Ballista's is not, so the same SQL can arrive at execution with materially
+  different plans.
+- **Comet falls back to Spark.** Operators and expressions Comet does not support
+  stay on the JVM, so a Comet run is generally a mix of DataFusion and Spark
+  execution rather than an all-DataFusion one.
+- **The distribution models differ.** Comet executes within Spark's task model and
+  shuffle service; Ballista has its own scheduler, stage/task model, and shuffle.
+
+So a Comet-vs-Ballista gap is best read as a question — _what is Spark's planner or
+execution model doing here that Ballista's is not?_ — rather than as a verdict on
+DataFusion. Vanilla Spark is included as the third data point, since it isolates
+how much of any Comet result comes from DataFusion acceleration versus from Spark's
+planner.
+
+## Results
+
+TPC-H **SF1000**, reference cluster above, 1 iteration, `target_partitions=32`,
+`prefer_hash_join=false`, `enable_dynamic_filter_pushdown=false`. Times in seconds;
+lower is better.
+
+Versions under test:
+
+| Engine   | Version                         |
+| -------- | ------------------------------- |
+| Ballista | `main` @ `26b29391`             |
+| Spark    | 3.5.3 (vanilla, Comet disabled) |
+| Comet    | 0.17.0                          |
+
+Pinning the Ballista commit rather than "main" matters: `26b29391` removed the
+static planner's sort-merge-join broadcast conversion, which changes the AQE-off
+path specifically. A number from before that commit is not comparable to one after
+it, and only the exact commit makes that visible.
+
+Every figure is from a **full 22-query suite run** — one query after another in a
+single session, on freshly started executors.
+
+|     Query |      Spark |      Comet | Ballista (AQE off) | Ballista (AQE on) |   Rows |
+| --------: | ---------: | ---------: | -----------------: | ----------------: | -----: |
+|         1 |      444.8 |       49.3 |               70.8 |              36.2 |      4 |
+|         2 |       74.3 |       37.3 |              155.5 |              63.6 |    100 |
+|         3 |      158.4 |       99.1 |              206.2 |             255.0 |     10 |
+|         4 |      104.1 |       42.3 |               76.8 |              53.9 |      5 |
+|         5 |      364.9 |      234.6 |              542.7 |             700.5 |      5 |
+|         6 |       22.0 |       15.3 |               18.3 |              12.7 |      1 |
+|         7 |      196.1 |      141.8 |              575.5 |             531.4 |      4 |
+|         8 |      412.7 |      291.6 |              657.1 |             801.2 |      2 |
+|         9 |      570.2 |      392.1 |              891.8 |            1025.2 |    175 |
+|        10 |      147.3 |      112.5 |              240.3 |             166.7 |     20 |
+|        11 |       58.3 |       48.7 |              105.1 |              76.2 |  0 [1] |
+|        12 |       75.9 |       52.9 |               79.4 |              85.2 |      2 |
+|        13 |      114.1 |       71.9 |               96.7 |              95.6 |     30 |
+|        14 |       44.6 |       29.0 |               38.0 |              34.3 |      1 |
+|        15 |      108.9 |       63.8 |               45.9 |              52.7 | \* [2] |
+|        16 |       33.9 |       18.7 |               21.9 |              23.0 |  27840 |
+|        17 |      519.5 |      308.2 |              697.8 |             400.4 |      1 |
+|        18 |      492.8 |      234.2 |              620.7 |             820.9 |    100 |
+|        19 |       53.7 |       39.4 |               39.6 |              42.1 |      1 |
+|        20 |      108.1 |       74.6 |               80.5 |              82.9 | 110759 |
+|        21 |      536.4 |      351.4 |             1083.0 |             926.6 |    100 |
+|        22 |       47.0 |       29.8 |               34.9 |              34.6 |      7 |
+| **Total** | **4687.9** | **2738.5** |         **6378.2** |        **6320.7** |        |
+
+**All 22 queries completed on all four configurations**; there are no failures to
+report. Row counts agreed across every engine except Q15.
+
+[1] Q11 returns 0 rows for every engine at this scale factor: the query's threshold
+constant is tuned for SF1.
+
+[2] **Row counts disagree on Q15: Spark returns 0, Comet and Ballista both return 1.** Q15 is a multi-statement query (`CREATE VIEW` / `SELECT` / `DROP VIEW`), and
+how many rows a harness reports depends on which statement it takes as the result.
+Recorded rather than resolved: it is not established which count is correct, or
+whether the engines computed different answers at all.
+
+The `Rows` column is the row count the query returned, recorded so a time is never
+read without the answer it produced.
+
+This table records **one current result set**. When results are refreshed, the
+table and the pinned versions above are replaced together — a row must never mix
+numbers from different commits, because a stale row silently misattributes a
+regression.
+
+`TBD` means not yet measured on this cluster at this commit; a query that ran but
+did not produce an answer is recorded as `FAIL`, or `OOM` where the failure is a
+known memory exhaustion.
+
+### Recording a result
+
+- Pin the **exact commit** the numbers came from, not a branch name.
+- Report **AQE on and AQE off** from the **same** commit and cluster. Comparing an
+  AQE-on number against an AQE-off number from an older build attributes the build
+  difference to the planner.
+- Take the figure from a **full suite run**, not a standalone single-query run. The
+  two differ measurably: a long-lived executor deep into a suite is not in the same
+  state as a freshly started one.
+- Note the **row count** each query returned. A fast wrong answer is not a result,
+  and distributed execution has produced silently wrong row counts before.
+- Flag any stage whose runtime is dominated by a few partitions — that is the
+  imbalance this page exists to surface.
+
+## Known issues found by benchmarking
+
+Benchmarking at SF1000 is how most of the following were found. They are worth
+knowing about before interpreting a number:
+
+| Issue                                                              | Summary                                                                                                                                                                                                  |
+| ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [#1359](https://github.com/apache/datafusion-ballista/issues/1359) | Umbrella issue for adaptive (AQE) query execution.                                                                                                                                                       |
+| [#2025](https://github.com/apache/datafusion-ballista/issues/2025) | Q18's hash-join build side exhausts the memory pool at SF1000. DataFusion's hash-join build side does not spill, so a per-partition build side larger than one task slot's pool fails the task outright. |
+| [#2063](https://github.com/apache/datafusion-ballista/issues/2063) | AQE can hang when a re-plan cancels an in-flight stage; the job never reports completion.                                                                                                                |
diff --git a/docs/source/contributors-guide/development.md b/docs/source/contributors-guide/development.md
index a54ae08f2e..bf404d3186 100644
--- a/docs/source/contributors-guide/development.md
+++ b/docs/source/contributors-guide/development.md
@@ -59,6 +59,31 @@ cargo build --release
 cargo test
 ```
 
+### TPC-H plan-stability golden files
+
+The `tpch_plan_stability` test suite in the `ballista-scheduler` crate guards the shape of the
+distributed physical plans that Ballista produces for the 22 TPC-H queries. For each query it plans
+the SQL and compares the staged plan text — the stage boundaries (`=== Stage N ===` banners),
+`ShuffleWriterExec` / `UnresolvedShuffleExec` nodes, and any exchange reuse — against an approved
+"golden" file. The queries are planned against a synthetic statistics-only table, so the suite is
+deterministic and runs without TPC-H data or a running cluster.
+
+The golden files live in
+[`ballista/scheduler/tests/tpch_plan_stability/approved/`](https://github.com/apache/datafusion-ballista/tree/main/ballista/scheduler/tests/tpch_plan_stability/approved)
+(`q1.txt` through `q22.txt`). When a change alters distributed planning, the suite fails with a
+plan-drift diff. Review the diff: if the change is unintended, fix the regression; if the new plans
+are correct, regenerate the golden files:
+
+```shell
+./dev/update-tpch-plan-stability.sh
+```
+
+This runs the suite in generate mode
+(`BALLISTA_GENERATE_GOLDEN=1 cargo test -p ballista-scheduler --test tpch_plan_stability`) and
+rewrites the approved files. Review the resulting changes under
+`ballista/scheduler/tests/tpch_plan_stability/approved/` and commit them together with the code
+change that caused them.
+
 ## Running the examples
 
 ```shell
@@ -156,3 +181,12 @@ This includes instructions for:
 - Running benchmarks against DataFusion and Ballista
 - Comparing performance with Apache Spark
 - Running load tests
+
+## Upgrade notes
+
+When a change breaks a public API, renames or removes a configuration key or CLI
+flag, changes a default that alters query plans or results, or changes the
+wire/serialization format, add an entry describing the change and the required
+action to the current unreleased page under
+[`docs/source/upgrading/`](../upgrading/index.rst). This keeps the upgrade guide
+complete for the next release.
diff --git a/docs/source/contributors-guide/user-personas.md b/docs/source/contributors-guide/user-personas.md
new file mode 100644
index 0000000000..a54d038548
--- /dev/null
+++ b/docs/source/contributors-guide/user-personas.md
@@ -0,0 +1,148 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# User Personas
+
+Ballista serves several distinct audiences, each of which comes to the project
+for a different reason and depends on a different set of capabilities. This page
+names those audiences as **personas** and, for each one, records the guarantees
+that audience relies on.
+
+## Why this page exists
+
+This is a review contract, not marketing copy. It exists so that changes to
+Ballista stay **additive**:
+
+- A pull request may **add** a new persona, or **extend** an existing one with
+  new capabilities.
+- A pull request must **not remove or silently regress** a capability that an
+  existing persona depends on.
+
+Reviewers evaluate every non-trivial pull request against this list. Each
+persona lists concrete "red flags" — the kinds of change that would quietly take
+functionality away from that audience. A red flag does not mean a change is
+forbidden, but it does mean the change needs explicit discussion and maintainer
+sign-off, and usually a migration path (for example an entry in the
+[upgrade guide](../upgrading/index)), rather than landing silently.
+
+These personas are deliberately in tension with one another. One audience wants
+distributed execution to be a transparent implementation detail; another wants
+the execution model exposed and tunable; another wants Ballista's internals to
+be a stable, composable library. The point of keeping them all written down is
+to make sure a change that delights one audience does not quietly break another.
+
+## Persona 1: The DataFusion user going multi-node
+
+- **Who**: A data engineer or analyst already using
+  [DataFusion](https://datafusion.apache.org/) single-process (often through the
+  Python bindings) who has outgrown a single machine.
+- **Coming from**: DataFusion, single process.
+- **Why Ballista**: Run the same SQL and DataFrame workloads across a cluster
+  with minimal code change and _identical results_.
+- **Depends on** (must not regress):
+  - Query results and semantics match single-process DataFusion for the same
+    input.
+  - A client surface that mirrors DataFusion's `SessionContext` / DataFrame / SQL
+    API.
+  - `datafusion.*` session configuration overrides are honored end-to-end, so
+    users tune DataFusion the way they already do.
+  - Standard object stores and file formats work and produce correct results
+    across partitions.
+  - Distributed execution is _transparent_: getting a correct answer never
+    requires the user to reason about stages, tasks, or partitioning.
+- **Red flags in a pull request**:
+  - Results diverge from single-process DataFusion for the same input.
+  - `datafusion.*` overrides are dropped or ignored during scheduling/planning.
+  - A DataFusion feature that is correct single-node becomes silently wrong when
+    the plan is split across stages (for example dynamic filter pushdown,
+    uncorrelated scalar subqueries, or a scan reading more than its assigned
+    partition).
+  - Client API changes that break the DataFusion-mirroring surface.
+
+## Persona 2: The Spark user wanting the same execution model
+
+- **Who**: A data engineer or platform owner running Spark SQL / batch jobs who
+  wants a lighter, Rust-native alternative without relearning a new execution
+  paradigm.
+- **Coming from**: Apache Spark.
+- **Why Ballista**: Keep the Spark mental model — plans split into **stages** at
+  shuffle boundaries, one **task** per partition, shuffle files, executors with
+  task slots, and adaptive query execution (**AQE**) for runtime adaptivity — on
+  top of DataFusion and Arrow.
+- **Depends on** (must not regress):
+  - The stage/task execution model at shuffle boundaries.
+  - The AQE / adaptive planner path (partition coalescing, dynamic join
+    selection, runtime broadcast), available and gated behind configuration.
+  - Broadcast joins for small build sides, to avoid shuffles and skew (the
+    equivalent of Spark's broadcast hash join).
+  - The operational surface Spark users expect: a scheduler plus executors,
+    task-slot concurrency, a tunable `target_partitions`, and stage/task
+    observability (task timings, a history server / UI) for debugging skew.
+  - Behavior driven by configuration and `SET`, not hard-coded planner choices.
+- **Red flags in a pull request**:
+  - Removing or short-circuiting the stage/task boundary model, or collapsing
+    toward single-process execution.
+  - Regressing or removing the AQE / adaptive planner path — or making it the
+    only path in a way that breaks Persona 1's transparent static path.
+  - Hard-coding a join or planner strategy instead of gating it behind
+    configuration.
+  - Removing the stage/task observability that Spark users rely on to diagnose
+    skew.
+
+## Persona 3: The library user building a specialized engine
+
+- **Who**: An engineer using Ballista as a _framework_ — the foundation for a
+  specialized or bespoke distributed query engine — rather than as a turnkey
+  cluster.
+- **Coming from**: Building on DataFusion as a library, now needing distribution
+  (or replacing a hand-rolled distributed layer).
+- **Why Ballista**: Reusable scheduler, executor, and plan-serialization
+  building blocks with extension points, instead of writing distributed
+  execution from scratch.
+- **Depends on** (must not regress):
+  - Stable, composable public APIs across the `scheduler`, `executor`, `client`,
+    and `core` crates (the execution graph and plan serialization included).
+  - Extension points that do not require forking: custom object stores,
+    physical/logical extension codecs, custom operators and functions, a
+    pluggable runtime producer, custom session configuration, and a custom query
+    planner.
+  - Behavior that stays _configurable rather than hard-coded_, so an embedder can
+    swap the planner, join strategy, partitioning, or runtime environment.
+  - Backward-compatible public APIs, with breaking changes signaled in the
+    [upgrade guide](../upgrading/index).
+  - No assumptions baked into the core that only fit the built-in clients or the
+    TPC-H benchmark.
+- **Red flags in a pull request**:
+  - Breaking a public API signature without an upgrade-guide entry.
+  - Hard-coding behavior that used to be pluggable, or removing an extension
+    point (an extension-codec argument, the runtime-producer override,
+    session-config injection).
+  - Coupling core logic to a specific client, planner, or benchmark instead of
+    keeping it behind a trait or configuration.
+  - Wire-format or serialization changes that break externally-produced plans
+    with no compatibility path.
+
+## Adding a persona
+
+This registry is append-only. As Ballista attracts new kinds of users, add a
+persona for each one via a pull request, using the same schema (who they are,
+what they are coming from, why Ballista, what they depend on, and the red flags
+that would regress them). Do not remove existing personas: an audience that
+relied on Ballista does not stop existing because our priorities shifted, and
+the whole value of this page is that the guarantees only accumulate.
diff --git a/docs/source/index.rst b/docs/source/index.rst
index f1b3e3d59e..ca71559d00 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -58,6 +58,18 @@ Table of content
    user-guide/spark-compatible-functions
    user-guide/extensions-example
 
+.. toctree::
+   :maxdepth: 1
+   :caption: Changelog
+
+   changelog/index
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Upgrade Guides
+
+   upgrading/index
+
 .. _toc.contributors:
 
 .. toctree::
@@ -66,7 +78,9 @@ Table of content
 
    contributors-guide/architecture
    contributors-guide/code-organization
+   contributors-guide/user-personas
    contributors-guide/development
+   contributors-guide/benchmarking
    Source code <https://github.com/apache/datafusion-ballista/>
 
 .. _toc.community:
diff --git a/docs/source/upgrading/54.0.0.md b/docs/source/upgrading/54.0.0.md
new file mode 100644
index 0000000000..ad7abca2ac
--- /dev/null
+++ b/docs/source/upgrading/54.0.0.md
@@ -0,0 +1,121 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Upgrade Guides
+
+## Ballista 54.0.0
+
+### For library and embedding users
+
+#### Upgrade to DataFusion 54
+
+Ballista `54.0.0` upgrades its DataFusion dependency to `54.0.0`. Code built on
+`ballista-core` / DataFusion APIs must move with it. See DataFusion's own
+[Upgrade Guide for 54.0.0](https://datafusion.apache.org/library-user-guide/upgrading/54.0.0.html)
+for the DataFusion-side API changes.
+
+#### `job_id` is now a newtype
+
+Scheduler job identifiers are no longer bare `String`s. `ballista-core` now
+exports a transparent `JobId` newtype (`ballista_core::JobId`) so a job id can
+no longer be confused with another string identifier at the type level. Public
+signatures that previously carried a `String`/`&str` job id now carry a `JobId`
+/ `&JobId` — for example `DistributedQueryExec::job_id()` now returns
+`Option<JobId>` instead of `Option<String>`. Construct one with
+`JobId::new(...)` or `.into()`, and recover the inner string with `.as_str()`
+or `.into_inner()`.
+
+```rust
+// before
+let id: Option<String> = query_exec.job_id();
+
+// after
+use ballista_core::JobId;
+let id: Option<JobId> = query_exec.job_id();
+let id_str: Option<String> = id.map(|id| id.into_inner());
+```
+
+### For cluster operators
+
+#### CORS origins and methods are configured via CLI flags
+
+The scheduler REST API no longer reads CORS configuration from environment
+variables. The `BALLISTA_CORS_ALLOWED_ORIGINS` and
+`BALLISTA_CORS_ALLOWED_METHODS` environment variables are removed and replaced
+by the `--cors-allowed-origins` and `--cors-allowed-methods` scheduler CLI
+flags (each a comma-separated list; `*` allows any). If you set these
+environment variables, move the values to the flags — otherwise the scheduler
+falls back to its defaults (origins `http://localhost:8080` and
+`https://nightlies.apache.org`; methods `GET`, `PATCH`, `OPTIONS`).
+
+```
+# before (environment variables)
+BALLISTA_CORS_ALLOWED_ORIGINS=https://example.com \
+BALLISTA_CORS_ALLOWED_METHODS=GET,PATCH \
+  ballista-scheduler
+
+# after (scheduler CLI flags)
+ballista-scheduler \
+  --cors-allowed-origins https://example.com \
+  --cors-allowed-methods GET,PATCH
+```
+
+### Planner and execution behavior changes
+
+#### The static planner now broadcasts small join build sides
+
+Ballista previously disabled `CollectLeft` broadcast joins entirely (both
+`datafusion.optimizer.hash_join_single_partition_threshold` and
+`hash_join_single_partition_threshold_rows` defaulted to `0`). In `54.0.0`
+those defaults are raised to `10 MB` (`10485760`) and `1000000` rows, and the
+static `DefaultDistributedPlanner` now converts a `SortMergeJoinExec` whose
+smaller side fits under `ballista.optimizer.broadcast_join_threshold_bytes`
+(default `10 MB`) into a broadcast `CollectLeft` hash join. This conversion is
+governed by the new `ballista.optimizer.broadcast_sort_merge_join_enabled`
+config key, which defaults to `true`.
+
+The consequence is that join plans — and therefore performance and shuffle
+behavior — change on upgrade **even with AQE off**. Broadcast is applied only
+to join types that are safe to broadcast (inner and right-side variants); other
+join types remain repartitioned. To restore the previous behavior, set
+`ballista.optimizer.broadcast_sort_merge_join_enabled = false`,
+`ballista.optimizer.broadcast_join_threshold_bytes = 0`,
+`datafusion.optimizer.hash_join_single_partition_threshold = 0`, and
+`hash_join_single_partition_threshold_rows = 0`.
+
+#### Serialized empty projections
+
+The plan serialization for a filter that projects to zero columns changed. In
+`53.x` an empty projection (`Some([])`) and "all columns" (`None`) both encoded
+to an empty list and decoded back to `None`, shifting downstream column indices;
+the scheduler now rewrites such filters into a serde-safe form before
+serialization. Together with the DataFusion 54 protobuf bump, this means a
+`54.0.0` scheduler and a `53.x` executor (or vice versa) cannot reliably
+exchange serialized plans. Upgrade schedulers and executors together — do not
+run a mixed-version cluster across this upgrade.
+
+#### AQE join behavior (opt-in)
+
+If you have enabled `ballista.planner.adaptive.enabled = true`, adaptive join
+selection now delays the join decision and can promote a small build side to a
+broadcast (`CollectLeft`) hash join at runtime, and empty-join inputs are
+short-circuited by the propagate-empty rule. This changes the runtime join
+plans (and their results' timing/shape) for adaptive clusters; it is gated by
+`ballista.planner.adaptive_join.enabled` (default `true`). Clusters using the
+default (AQE off) are unaffected.
diff --git a/docs/source/upgrading/55.0.0.md b/docs/source/upgrading/55.0.0.md
new file mode 100644
index 0000000000..ccbabfe89e
--- /dev/null
+++ b/docs/source/upgrading/55.0.0.md
@@ -0,0 +1,73 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Upgrade Guides
+
+## Ballista 55.0.0
+
+**Note:** Ballista `55.0.0` has not been released yet. The information provided
+in this section pertains to features and changes that have already been merged
+to the main branch and are awaiting release in this version.
+
+### Planner and execution behavior changes
+
+#### `SortMergeJoinExec` is no longer converted to a broadcast join
+
+`54.0.0` added a static-planner rewrite that converted a `SortMergeJoinExec`
+with a small build side into a broadcast `CollectLeft` hash join, governed by
+`ballista.optimizer.broadcast_sort_merge_join_enabled` (default `true`). That
+rewrite produced incorrect results for some plans, so both the rewrite and its
+config key are removed in `55.0.0`.
+
+A `SortMergeJoinExec` now always executes as a sort-merge join over
+repartitioned inputs. Broadcast promotion still applies to a `HashJoinExec`
+whose smaller side fits under `ballista.optimizer.broadcast_join_threshold_bytes`
+(default `10 MB`), which is unchanged.
+
+**This affects the default configuration.** Ballista sets
+`datafusion.optimizer.prefer_hash_join = false` by default (see
+[#1648](https://github.com/apache/datafusion-ballista/issues/1648)), so joins are
+planned as `SortMergeJoinExec` unless you opt out. Under `54.0.0` those joins were
+then silently converted to broadcast hash joins whenever one side fit under the
+threshold, so expect join plan shape — and therefore performance and shuffle
+behavior — to change on upgrade for any query with a join whose smaller side is
+under `10 MB`.
+
+Alongside the incorrect results, the conversion also worked against the reason
+sort-merge join is the default. DataFusion's hash join has no spill support, so a
+`CollectLeft` build side must fit in memory in every parallel task on an executor;
+sort-merge join spills. Converting a sort-merge join into a broadcast hash join
+reintroduced exactly the memory behavior that the `prefer_hash_join = false`
+default exists to avoid.
+
+**Action required:** the key `ballista.optimizer.broadcast_sort_merge_join_enabled`
+no longer exists, and Ballista rejects unknown configuration keys. If you set it
+(including to `false`, which was the workaround for the incorrect results), remove
+it — otherwise the session fails with:
+
+```text
+configuration key `optimizer.broadcast_sort_merge_join_enabled` does not exist
+```
+
+If you relied on it being `false`, no replacement is needed: that is now the only
+behavior. If you relied on it being `true` and want small joins to broadcast, set
+`datafusion.optimizer.prefer_hash_join = true` so joins are planned as a
+`HashJoinExec`, which remains eligible for broadcast promotion. Note this opts you
+into the non-spilling hash join for _all_ joins in the session, not only the small
+ones.
diff --git a/docs/source/upgrading/index.rst b/docs/source/upgrading/index.rst
new file mode 100644
index 0000000000..1d490ea4a6
--- /dev/null
+++ b/docs/source/upgrading/index.rst
@@ -0,0 +1,25 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Upgrade Guides
+==============
+
+.. toctree::
+   :maxdepth: 1
+
+   Ballista 55.0.0 <55.0.0>
+   Ballista 54.0.0 <54.0.0>
diff --git a/docs/source/user-guide/cli.md b/docs/source/user-guide/cli.md
index 777e68d92e..b4fec1faa3 100644
--- a/docs/source/user-guide/cli.md
+++ b/docs/source/user-guide/cli.md
@@ -158,7 +158,7 @@ ballista-cli
 
 The TUI provides the following views:
 
-- **Executors**: Lists all registered executors with their host, port, task slots, memory usage, and last seen time.
+- **Executors**: Lists all registered executors with their host, port, vcores, memory usage, and last seen time.
   Supports sorting by any column.
 - **Executor details**: Select an executor and press `Enter` to show extra details about it.
 - **Jobs**: Displays active and completed jobs with their status, start time, and duration. Supports sorting, job
diff --git a/docs/source/user-guide/deployment/docker.md b/docs/source/user-guide/deployment/docker.md
index ad3571c50b..7b0e1f1e14 100644
--- a/docs/source/user-guide/deployment/docker.md
+++ b/docs/source/user-guide/deployment/docker.md
@@ -98,7 +98,7 @@ Use `docker logs CONTAINER_ID` to check the output from the executor(s):
 $ docker logs fb8b530cee6d
 INFO ballista_executor::executor_process: Running with config:
 INFO ballista_executor::executor_process: work_dir: /tmp/.tmpAkP3pZ
-INFO ballista_executor::executor_process: concurrent_tasks: 48
+INFO ballista_executor::executor_process: Executor vcores (default: available CPU cores): 48
 INFO ballista_executor::executor_process: Ballista v52.0.0 Rust Executor Flight Server listening on 0.0.0.0:50051
 INFO ballista_executor::execution_loop: Starting poll work loop with scheduler
 ```
diff --git a/docs/source/user-guide/deployment/kubernetes.md b/docs/source/user-guide/deployment/kubernetes.md
index a979a6c8c5..9e936abba7 100644
--- a/docs/source/user-guide/deployment/kubernetes.md
+++ b/docs/source/user-guide/deployment/kubernetes.md
@@ -144,6 +144,18 @@ spec:
           ports:
             - containerPort: 50050
               name: flight
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 50050
+            failureThreshold: 3
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 50050
+            failureThreshold: 3
+            periodSeconds: 5
           volumeMounts:
             - mountPath: /mnt
               name: data
@@ -177,6 +189,20 @@ spec:
           ports:
             - containerPort: 50051
               name: flight
+            - containerPort: 50053
+              name: health
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 50053
+            failureThreshold: 3
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 50053
+            failureThreshold: 3
+            periodSeconds: 5
           volumeMounts:
             - mountPath: /mnt
               name: data
@@ -217,6 +243,78 @@ INFO ballista_scheduler::scheduler_server::grpc: Received register_executor requ
 INFO ballista_scheduler::scheduler_server::grpc: Received register_executor request for ExecutorMetadata { id: "816e4502-a876-4ed8-b33f-86d243dcf63f", host: "10.1.23.150", port: 50051 }
 ```
 
+## Health Probes and Rolling Upgrades
+
+Both the scheduler and executor expose two Kubernetes-style HTTP probe endpoints:
+
+- **`/healthz`** — liveness. Always returns `200 OK` while the process is
+  running. Failing this signals only "the process is dead," which is the sole
+  case where a restart can actually recover. Do **not** make liveness reflect
+  scheduler connectivity — a flapping scheduler would then take the whole
+  executor fleet down with it in a restart cascade.
+- **`/readyz`** — readiness. Returns `200 OK` when the pod is prepared to
+  accept traffic (scheduler: at least `--min-ready-executors` executors
+  registered; executor: last heartbeat to the scheduler succeeded).
+  A failing `/readyz` removes the pod from `Service` endpoints but does
+  **not** trigger a restart, so it is safe to gate on cluster state.
+
+The scheduler serves both probes on the same port as its gRPC/REST API
+(default `50050`). The executor serves them on a dedicated HTTP port
+(`--bind-health-port`, default `50053`) since the Arrow Flight port is not
+HTTP.
+
+### Protocol version and forced upgrades
+
+Every executor sends a compile-time `BALLISTA_PROTOCOL_VERSION` on
+registration and heartbeat. The scheduler compares against its own compiled
+value and rejects mismatches with `FailedPrecondition`; the executor then
+flips `/readyz` off and, after several consecutive failures, self-terminates
+so the operator (or Deployment controller) can bring a matching version up.
+
+Bump `BALLISTA_PROTOCOL_VERSION` only on releases that change the
+executor↔scheduler wire format. Most releases will not bump it.
+
+### Rolling both Deployments together
+
+The scheduler and executor live in separate Deployments; there is no single
+"app roll." When you cut a release that bumps `BALLISTA_PROTOCOL_VERSION`,
+update **both** Deployments' image tags in the same change and apply them
+together. Kubernetes will roll each Deployment independently under its own
+rollout policy, but the fleet stays serving throughout:
+
+| scheduler ↔ executor | outcome                                                                      |
+| -------------------- | ---------------------------------------------------------------------------- |
+| old ↔ old            | works (old scheduler pre-dates the check)                                    |
+| old ↔ new            | works (old scheduler ignores the unknown field)                              |
+| new ↔ new            | works                                                                        |
+| new ↔ old            | rejected — new scheduler fails `/readyz` until enough new executors register |
+
+Because the _old_ scheduler is permissive (it never learned to check the
+field), executors that reach it during the transition are always accepted.
+The rejection happens only when a _new_ scheduler sees an _old_ executor,
+which is the case the version check is designed to protect against —
+mismatched pairs never exchange work, so they cannot corrupt state.
+
+Give each Deployment a `maxSurge` so Kubernetes brings new pods up before
+tearing old ones down; this keeps executor capacity at roughly 100% during
+the roll:
+
+```yaml
+spec:
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 0
+```
+
+The scheduler's `/readyz` gate (`--min-ready-executors`, default `1`) means
+a fresh scheduler pod does not receive `Service` traffic until at least one
+matching-version executor has registered with it, so clients keep hitting
+the old scheduler until the new one has real capacity behind it. Set
+`--min-ready-executors=0` if you want the scheduler to advertise readiness
+immediately (e.g. single-node development clusters).
+
 ## Port Forwarding
 
 If you want to run applications outside of the cluster and have them connect to the scheduler then it is necessary to
diff --git a/docs/source/user-guide/python/quickstart.md b/docs/source/user-guide/python/quickstart.md
index 012e4006d2..6dfa6e53d1 100644
--- a/docs/source/user-guide/python/quickstart.md
+++ b/docs/source/user-guide/python/quickstart.md
@@ -51,7 +51,7 @@ ctx = BallistaSessionContext(f"df://{host}:{port}")
 ### Target Partitions
 
 Set `datafusion.execution.target_partitions` to match your cluster capacity
-(`executors × concurrent_tasks_per_executor`) by passing `cluster_config` to the
+(`executors × vcores_per_executor`) by passing `cluster_config` to the
 constructor. The default inherits from DataFusion and is based on the client's CPU
 count, which is far too low for distributed execution:
 
@@ -59,8 +59,8 @@ count, which is far too low for distributed execution:
 from ballista import BallistaSessionContext
 
 executors = 4
-concurrent_tasks = 8
-target_partitions = executors * concurrent_tasks
+vcores = 8
+target_partitions = executors * vcores
 
 ctx = BallistaSessionContext(
     "df://localhost:50050",
diff --git a/docs/source/user-guide/tuning-guide.md b/docs/source/user-guide/tuning-guide.md
index 7b6f38130f..c2daa312b2 100644
--- a/docs/source/user-guide/tuning-guide.md
+++ b/docs/source/user-guide/tuning-guide.md
@@ -56,12 +56,12 @@ let ctx: SessionContext = SessionContext::remote_with_state(&url,state).await?;
 
 ## Configuring Executor Concurrency Levels
 
-Each executor instance has a fixed number of tasks that it can process concurrently. This is specified by passing a
-`concurrent_tasks` command-line parameter. The default setting is to use all available CPU cores.
+Each executor instance advertises a fixed number of virtual cores (vcores) to the scheduler. This is specified by
+passing a `--vcores` command-line parameter. The default setting is to use all available CPU cores.
 
 Increasing this configuration setting will increase the number of tasks that each executor can run concurrently but
 this will also mean that the executor will use more memory. If executors are failing due to out-of-memory errors then
-decreasing the number of concurrent tasks may help.
+decreasing the vcore count may help.
 
 ## Configuring Executor Memory Pool
 
@@ -71,22 +71,22 @@ memory. To bound executor memory and let those operators spill to disk under
 pressure, pass `--memory-pool-size` when starting the executor:
 
 ```sh
-ballista-executor --memory-pool-size 8GB --concurrent-tasks 8
+ballista-executor --memory-pool-size 8GB --vcores 8
 ```
 
 The argument accepts human-readable sizes (`8GB`, `512MiB`) or a plain byte
 count. SI suffixes (`KB`/`MB`/`GB`) are powers of 10; IEC suffixes
 (`KiB`/`MiB`/`GiB`) are powers of 2.
 
-The total budget is divided equally across concurrent task slots: each task
-receives its own `FairSpillPool` of size `memory_pool_size / concurrent_tasks`.
-With `--memory-pool-size 8GB --concurrent-tasks 8`, every task sees a 1 GB
+The total budget is divided equally across the executor's vcores: each task
+receives its own `FairSpillPool` of size `memory_pool_size / vcores`.
+With `--memory-pool-size 8GB --vcores 8`, every task sees a 1 GB
 pool, fully isolated from other tasks. Idle slots do not lend their share to
 busy ones, which keeps task memory predictable at the cost of some unused
 capacity when the executor is under-utilized.
 
 The executor refuses to start if the per-task share would round to zero (i.e.
-`memory_pool_size < concurrent_tasks`).
+`memory_pool_size < vcores`).
 
 When `--memory-pool-size` is not set, the executor behaves as before with no
 memory pool installed.
@@ -196,9 +196,11 @@ Adaptive Query Planning is EXPERIMENTAL, should be used for testing purposes onl
 
 ### Configuration
 
-| key                               | type    | default | description                                 |
-| --------------------------------- | ------- | ------- | ------------------------------------------- |
-| ballista.planner.adaptive.enabled | Boolean | false   | Enables the adaptive planner. Experimental. |
+| key                                               | type    | default  | description                                                                                                                                                            |
+| ------------------------------------------------- | ------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| ballista.planner.adaptive.enabled                 | Boolean | false    | Enables the adaptive planner. Experimental.                                                                                                                            |
+| ballista.optimizer.broadcast_join_threshold_bytes | UInt64  | 10485760 | Byte-size threshold below which a hash join's smaller side is broadcast (`CollectLeft`). Governs both the static planner and AQE. Set to 0 to disable broadcast joins. |
+| ballista.optimizer.broadcast_join_threshold_rows  | UInt64  | 1000000  | Row-count fallback threshold used when byte-size statistics are unavailable. Applies to AQE. Set to 0 to disable promotion via the row-count path.                     |
 
 ### What AQE does today
 
@@ -209,6 +211,9 @@ implemented:
 
 - **Join reordering.** Uses runtime row counts from completed stages so the
   smaller side drives the join.
+- **Broadcast join selection.** When a join input's runtime size falls under
+  `ballista.optimizer.broadcast_join_threshold_bytes` (or the row-count
+  fallback), the smaller side is broadcast (`CollectLeft`) instead of shuffled.
 - **Empty stage elimination.** When a completed stage produces zero rows, its
   downstream exchange is replaced with an empty execution node, and emptiness
   is propagated up the plan so downstream stages are skipped entirely.
@@ -218,14 +223,14 @@ implemented:
 The implementation covers the happy path only. The following are known to be
 missing or incomplete:
 
-- Executor failure handling on the AQE path
-- Dynamic coalescing of shuffle partitions
-- Switching from hash join to sort-merge join based on runtime statistics
-- Switching from streaming aggregation to hash aggregation based on runtime statistics
+- Executor failure handling on the AQE path ([#1986](https://github.com/apache/datafusion-ballista/issues/1986))
+- Dynamic coalescing of shuffle partitions ([#1987](https://github.com/apache/datafusion-ballista/issues/1987))
+- Switching from hash join to sort-merge join based on runtime statistics ([#1988](https://github.com/apache/datafusion-ballista/issues/1988))
+- Switching from streaming aggregation to hash aggregation based on runtime statistics ([#1989](https://github.com/apache/datafusion-ballista/issues/1989))
 
 Until these gaps are closed, AQE should be used for testing and experimentation
-rather than production workloads. See [issue #387](https://github.com/apache/datafusion-ballista/issues/387)
-for the tracking issue and ongoing work.
+rather than production workloads. See [issue #1359](https://github.com/apache/datafusion-ballista/issues/1359)
+for the tracking epic and ongoing work.
 
 ## Push-based vs Pull-based Task Scheduling
 
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 2f6e55451b..531ce36565 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -18,7 +18,7 @@
 [package]
 name = "ballista-examples"
 description = "Ballista usage examples"
-version = "53.0.0"
+version = "54.0.0"
 homepage = "https://datafusion.apache.org/ballista/"
 repository = "https://github.com/apache/datafusion-ballista"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
@@ -57,10 +57,10 @@ rustls = { version = "0.23", features = ["ring"], optional = true }
 
 [dev-dependencies]
 arrow-flight = { workspace = true }
-ballista = { path = "../ballista/client", version = "53.0.0" }
-ballista-core = { path = "../ballista/core", version = "53.0.0", default-features = false, features = ["build-binary"] }
-ballista-executor = { path = "../ballista/executor", version = "53.0.0", default-features = false }
-ballista-scheduler = { path = "../ballista/scheduler", version = "53.0.0", default-features = false }
+ballista = { path = "../ballista/client", version = "54.0.0" }
+ballista-core = { path = "../ballista/core", version = "54.0.0", default-features = false, features = ["build-binary"] }
+ballista-executor = { path = "../ballista/executor", version = "54.0.0", default-features = false }
+ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", default-features = false }
 ctor = { workspace = true }
 datafusion = { workspace = true }
 datafusion-proto = { workspace = true }
diff --git a/examples/examples/mtls-cluster.rs b/examples/examples/mtls-cluster.rs
index 593248183e..609935d6ba 100644
--- a/examples/examples/mtls-cluster.rs
+++ b/examples/examples/mtls-cluster.rs
@@ -54,6 +54,7 @@ use std::process::Command;
 use std::sync::Arc;
 
 use arrow_flight::flight_service_server::FlightServiceServer;
+use ballista_core::BALLISTA_PROTOCOL_VERSION;
 use ballista_core::ConfigProducer;
 use ballista_core::extension::{SessionConfigExt, SessionStateExt};
 use ballista_core::serde::protobuf::executor_resource::Resource;
@@ -363,10 +364,11 @@ async fn run_executor() -> Result<(), Box<dyn std::error::Error>> {
         grpc_port: 0, // Not used in pull-based scheduling
         specification: Some(ExecutorSpecification {
             resources: vec![ExecutorResource {
-                resource: Some(Resource::TaskSlots(4)),
+                resource: Some(Resource::Vcores(4)),
             }],
         }),
         os_info: None,
+        ballista_protocol_version: BALLISTA_PROTOCOL_VERSION,
     };
 
     let config_producer = create_tls_config_producer(tls.client_tls.clone());
@@ -388,7 +390,7 @@ async fn run_executor() -> Result<(), Box<dyn std::error::Error>> {
         config_producer,
         Default::default(), // function_registry
         Arc::new(LoggingMetricsCollector::default()), // metrics_collector
-        4,                  // concurrent_tasks
+        4,                  // vcores
     ));
 
     // Start Flight service with mTLS for serving shuffle data
@@ -429,8 +431,9 @@ async fn run_executor() -> Result<(), Box<dyn std::error::Error>> {
     // Run the pull-based execution loop
     // This registers the executor and starts polling for tasks
     info!("Starting execution poll loop...");
+    let health = ballista_executor::health::ExecutorHealth::new();
     let poll_handle = tokio::spawn(async move {
-        execution_loop::poll_loop(scheduler, executor, codec).await
+        execution_loop::poll_loop(scheduler, executor, codec, None, health).await
     });
 
     tokio::select! {
diff --git a/examples/examples/standalone-substrait.rs b/examples/examples/standalone-substrait.rs
index 762a9c82e4..f5d01dbe8a 100644
--- a/examples/examples/standalone-substrait.rs
+++ b/examples/examples/standalone-substrait.rs
@@ -422,6 +422,8 @@ impl SubstraitSchedulerClient {
             None,
             io_retries_times,
             io_retry_wait_time_ms,
+            0,
+            0,
         )
         .await
         .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?;
@@ -576,13 +578,13 @@ pub async fn setup_standalone(session_state: Option<&SessionState>) -> Result<St
         }
     };
 
-    let concurrent_tasks = config.ballista_standalone_parallelism();
+    let vcores = config.ballista_standalone_parallelism();
 
     match session_state {
         None => {
             ballista_executor::new_standalone_executor(
                 scheduler,
-                concurrent_tasks,
+                vcores,
                 BallistaCodec::default(),
             )
             .await
@@ -591,7 +593,7 @@ pub async fn setup_standalone(session_state: Option<&SessionState>) -> Result<St
         Some(session_state) => {
             ballista_executor::new_standalone_executor_from_state(
                 scheduler,
-                concurrent_tasks,
+                vcores,
                 session_state,
             )
             .await
diff --git a/examples/tests/object_store.rs b/examples/tests/object_store.rs
index b79712ea9b..6202bb8f9a 100644
--- a/examples/tests/object_store.rs
+++ b/examples/tests/object_store.rs
@@ -371,6 +371,149 @@ mod custom_s3_config {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn should_configure_aqe_s3_execute_sql_write_remote()
+    -> datafusion::error::Result<()> {
+        let test_data = examples_test_data();
+
+        //
+        // Minio cluster setup
+        //
+        let container = crate::common::create_minio_container();
+        let node = container.start().await.unwrap();
+
+        node.exec(crate::common::create_bucket_command())
+            .await
+            .unwrap();
+
+        let endpoint_host = node.get_host().await.unwrap();
+        let endpoint_port = node.get_host_port_ipv4(9000).await.unwrap();
+
+        log::info!(
+            "MINIO testcontainers host: {}, port: {}",
+            endpoint_host,
+            endpoint_port
+        );
+
+        //
+        // Session Context and Ballista cluster setup
+        //
+
+        // Setting up configuration producer
+        //
+        // configuration producer registers user defined config extension
+        // S3Option with relevant S3 configuration
+        let config_producer =
+            Arc::new(ballista_core::object_store::session_config_with_s3_support);
+        // Setting up runtime producer
+        //
+        // Runtime producer creates object store registry
+        // which can create object store connecter based on
+        // S3Option configuration.
+        let runtime_producer: RuntimeProducer =
+            Arc::new(ballista_core::object_store::runtime_env_with_s3_support);
+
+        // Session builder creates SessionState
+        //
+        // which is configured using runtime and configuration producer,
+        // producing same runtime environment, and providing same
+        // object store registry.
+
+        let session_builder =
+            Arc::new(ballista_core::object_store::session_state_with_s3_support);
+
+        let state = session_builder(config_producer())?;
+
+        // setting up ballista cluster with new runtime, configuration, and session state producers
+        let (host, port) = crate::common::setup_test_cluster_with_builders(
+            config_producer,
+            runtime_producer,
+            session_builder,
+        )
+        .await;
+        let url = format!("df://{host}:{port}");
+
+        // establishing cluster connection,
+        let ctx: SessionContext = SessionContext::remote_with_state(&url, state).await?;
+
+        ctx.sql("SET ballista.planner.adaptive.enabled = true")
+            .await?
+            .show()
+            .await?;
+        // setting up relevant S3 options
+        ctx.sql("SET s3.allow_http = true").await?.show().await?;
+        ctx.sql(&format!("SET s3.access_key_id = '{}'", ACCESS_KEY_ID))
+            .await?
+            .show()
+            .await?;
+        ctx.sql(&format!("SET s3.secret_access_key = '{}'", SECRET_KEY))
+            .await?
+            .show()
+            .await?;
+        ctx.sql(&format!(
+            "SET s3.endpoint = 'http://{}:{}'",
+            endpoint_host, endpoint_port
+        ))
+        .await?
+        .show()
+        .await?;
+        ctx.sql("SET s3.allow_http = true").await?.show().await?;
+
+        // verifying that we have set S3Options
+        ctx.sql("select name, value from information_schema.df_settings where name like 's3.%'").await?.show().await?;
+
+        ctx.register_parquet(
+            "test",
+            &format!("{test_data}/alltypes_plain.parquet"),
+            Default::default(),
+        )
+        .await?;
+
+        let write_dir_path =
+            &format!("s3://{}/write_test.parquet", crate::common::BUCKET);
+
+        ctx.sql("select * from test")
+            .await?
+            .write_parquet(write_dir_path, Default::default(), Default::default())
+            .await?;
+
+        ctx.register_parquet("written_table", write_dir_path, Default::default())
+            .await?;
+
+        let result = ctx
+            .sql("select id, string_col, timestamp_col from written_table where id > 4")
+            .await?
+            .collect()
+            .await?;
+        let expected = [
+            "+----+------------+---------------------+",
+            "| id | string_col | timestamp_col       |",
+            "+----+------------+---------------------+",
+            "| 5  | 31         | 2009-03-01T00:01:00 |",
+            "| 6  | 30         | 2009-04-01T00:00:00 |",
+            "| 7  | 31         | 2009-04-01T00:01:00 |",
+            "+----+------------+---------------------+",
+        ];
+
+        assert_batches_eq!(expected, &result);
+
+        let result = ctx
+            .sql("select max(id) as max, min(id) as min from written_table")
+            .await?
+            .collect()
+            .await?;
+        let expected = [
+            "+-----+-----+",
+            "| max | min |",
+            "+-----+-----+",
+            "| 7   | 0   |",
+            "+-----+-----+",
+        ];
+
+        assert_batches_eq!(expected, &result);
+        Ok(())
+    }
+
     // this test shows how to register external ObjectStoreRegistry and configure it
     // using infrastructure provided by ballista standalone.
     //
diff --git a/python/Cargo.lock b/python/Cargo.lock
index dc1c986dc8..a0544afc65 100644
--- a/python/Cargo.lock
+++ b/python/Cargo.lock
@@ -2,54 +2,6 @@
 # It is not intended for manual editing.
 version = 4
 
-[[package]]
-name = "abi_stable"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445"
-dependencies = [
- "abi_stable_derive",
- "abi_stable_shared",
- "const_panic",
- "core_extensions",
- "crossbeam-channel",
- "generational-arena",
- "libloading",
- "lock_api",
- "parking_lot",
- "paste",
- "repr_offset",
- "rustc_version",
- "serde",
- "serde_derive",
- "serde_json",
-]
-
-[[package]]
-name = "abi_stable_derive"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898"
-dependencies = [
- "abi_stable_shared",
- "as_derive_utils",
- "core_extensions",
- "proc-macro2",
- "quote",
- "rustc_version",
- "syn 1.0.109",
- "typed-arena",
-]
-
-[[package]]
-name = "abi_stable_shared"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63"
-dependencies = [
- "core_extensions",
-]
-
 [[package]]
 name = "adler2"
 version = "2.0.1"
@@ -87,9 +39,9 @@ checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
 
 [[package]]
 name = "alloc-stdlib"
-version = "0.2.2"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
+checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
 dependencies = [
  "alloc-no-stdlib",
 ]
@@ -111,53 +63,24 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.102"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
-
-[[package]]
-name = "apache-avro"
-version = "0.21.0"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
-dependencies = [
- "bigdecimal",
- "bon",
- "bzip2",
- "crc32fast",
- "digest 0.10.7",
- "liblzma",
- "log",
- "miniz_oxide",
- "num-bigint",
- "quad-rand",
- "rand 0.9.4",
- "regex-lite",
- "serde",
- "serde_bytes",
- "serde_json",
- "snap",
- "strum",
- "strum_macros",
- "thiserror",
- "uuid",
- "zstd",
-]
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
 [[package]]
 name = "ar_archive_writer"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
+checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
 dependencies = [
  "object",
 ]
 
 [[package]]
 name = "arc-swap"
-version = "1.9.1"
+version = "1.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
 dependencies = [
  "rustversion",
 ]
@@ -170,9 +93,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
 
 [[package]]
 name = "arrayvec"
-version = "0.7.6"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
 
 [[package]]
 name = "arrow"
@@ -229,6 +152,30 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-avro"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
+ "bytes",
+ "bzip2",
+ "crc",
+ "flate2",
+ "indexmap",
+ "liblzma",
+ "rand 0.9.5",
+ "serde",
+ "serde_json",
+ "snap",
+ "strum_macros",
+ "uuid",
+ "zstd",
+]
+
 [[package]]
 name = "arrow-buffer"
 version = "58.3.0"
@@ -440,18 +387,6 @@ dependencies = [
  "regex-syntax",
 ]
 
-[[package]]
-name = "as_derive_utils"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4"
-dependencies = [
- "core_extensions",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "async-compression"
 version = "0.4.42"
@@ -469,19 +404,16 @@ name = "async-ffi"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50"
-dependencies = [
- "abi_stable",
-]
 
 [[package]]
 name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 3.0.0",
 ]
 
 [[package]]
@@ -559,7 +491,7 @@ dependencies = [
 
 [[package]]
 name = "ballista"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "async-trait",
  "ballista-core",
@@ -573,7 +505,7 @@ dependencies = [
 
 [[package]]
 name = "ballista-core"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow-flight",
  "async-trait",
@@ -582,13 +514,13 @@ dependencies = [
  "datafusion-proto",
  "datafusion-proto-common",
  "futures",
- "itertools",
+ "itertools 0.15.0",
  "log",
  "md-5 0.11.0",
  "parking_lot",
  "prost",
  "prost-types",
- "rand 0.10.1",
+ "rand 0.10.2",
  "rustc_version",
  "serde",
  "tokio",
@@ -603,7 +535,7 @@ dependencies = [
 
 [[package]]
 name = "ballista-executor"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow",
  "arrow-flight",
@@ -616,6 +548,7 @@ dependencies = [
  "futures",
  "libc",
  "log",
+ "lru",
  "memory-stats",
  "parking_lot",
  "serde",
@@ -630,7 +563,7 @@ dependencies = [
 
 [[package]]
 name = "ballista-scheduler"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "arrow-flight",
  "async-trait",
@@ -642,18 +575,18 @@ dependencies = [
  "futures",
  "http",
  "insta",
- "itertools",
+ "itertools 0.15.0",
  "log",
  "object_store",
  "parking_lot",
  "prost",
  "prost-types",
- "rand 0.10.1",
+ "rand 0.10.2",
  "serde",
  "tokio",
  "tokio-stream",
  "tonic",
- "tower-http",
+ "tower-http 0.7.0",
  "uuid",
 ]
 
@@ -674,14 +607,13 @@ dependencies = [
  "num-bigint",
  "num-integer",
  "num-traits",
- "serde",
 ]
 
 [[package]]
 name = "bitflags"
-version = "2.11.1"
+version = "2.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
 
 [[package]]
 name = "blake2"
@@ -703,7 +635,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "constant_time_eq",
- "cpufeatures 0.3.0",
+ "cpufeatures",
 ]
 
 [[package]]
@@ -717,43 +649,18 @@ dependencies = [
 
 [[package]]
 name = "block-buffer"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
 dependencies = [
  "hybrid-array",
 ]
 
-[[package]]
-name = "bon"
-version = "3.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
-dependencies = [
- "darling",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "brotli"
-version = "8.0.2"
+version = "8.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
+checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -762,9 +669,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "5.0.0"
+version = "5.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
+checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -784,15 +691,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.11.1"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytesize"
-version = "2.3.1"
+version = "2.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3"
+checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
 
 [[package]]
 name = "bzip2"
@@ -805,9 +712,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.62"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -823,26 +730,26 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
 [[package]]
 name = "cfg_aliases"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
 
 [[package]]
 name = "chacha20"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
 dependencies = [
  "cfg-if",
- "cpufeatures 0.3.0",
+ "cpufeatures",
  "rand_core 0.10.1",
 ]
 
 [[package]]
 name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
 dependencies = [
  "iana-time-zone",
  "num-traits",
@@ -893,9 +800,9 @@ checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
 
 [[package]]
 name = "console"
-version = "0.16.3"
+version = "0.16.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"
+checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c"
 dependencies = [
  "encode_unicode",
  "libc",
@@ -928,15 +835,6 @@ dependencies = [
  "tiny-keccak",
 ]
 
-[[package]]
-name = "const_panic"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652"
-dependencies = [
- "typewit",
-]
-
 [[package]]
 name = "constant_time_eq"
 version = "0.4.2"
@@ -960,37 +858,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
-name = "core_extensions"
-version = "1.5.4"
+name = "cpufeatures"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
 dependencies = [
- "core_extensions_proc_macros",
+ "libc",
 ]
 
 [[package]]
-name = "core_extensions_proc_macros"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea"
-
-[[package]]
-name = "cpufeatures"
-version = "0.2.17"
+name = "crc"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
 dependencies = [
- "libc",
+ "crc-catalog",
 ]
 
 [[package]]
-name = "cpufeatures"
-version = "0.3.0"
+name = "crc-catalog"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
-dependencies = [
- "libc",
-]
+checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
 
 [[package]]
 name = "crc32fast"
@@ -1001,20 +890,11 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
-dependencies = [
- "crossbeam-utils",
-]
-
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.21"
+version = "0.8.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
 
 [[package]]
 name = "crunchy"
@@ -1072,40 +952,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "darling"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
-dependencies = [
- "darling_core",
- "darling_macro",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
-dependencies = [
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn 2.0.117",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
-dependencies = [
- "darling_core",
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "dashmap"
 version = "6.2.1"
@@ -1122,14 +968,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "bzip2",
  "chrono",
  "datafusion-catalog",
@@ -1160,14 +1005,13 @@ dependencies = [
  "datafusion-sql",
  "flate2",
  "futures",
- "itertools",
+ "indexmap",
+ "itertools 0.14.0",
  "liblzma",
  "log",
  "object_store",
  "parking_lot",
  "parquet",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -1178,9 +1022,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1194,7 +1038,7 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-session",
  "futures",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
@@ -1203,9 +1047,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1219,42 +1063,42 @@ dependencies = [
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
  "futures",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "object_store",
 ]
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
 dependencies = [
- "ahash",
- "apache-avro",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap",
- "itertools",
+ "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
  "parquet",
- "paste",
  "recursive",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
 dependencies = [
  "futures",
  "log",
@@ -1263,9 +1107,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
 dependencies = [
  "arrow",
  "async-compression",
@@ -1285,11 +1129,12 @@ dependencies = [
  "flate2",
  "futures",
  "glob",
- "itertools",
+ "itertools 0.14.0",
  "liblzma",
  "log",
  "object_store",
- "rand 0.9.4",
+ "parking_lot",
+ "rand 0.9.5",
  "tokio",
  "tokio-util",
  "url",
@@ -1298,9 +1143,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1315,36 +1160,35 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-session",
  "futures",
- "itertools",
+ "itertools 0.14.0",
  "object_store",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-datasource-avro"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4"
+checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195"
 dependencies = [
- "apache-avro",
  "arrow",
+ "arrow-avro",
  "async-trait",
  "bytes",
  "datafusion-common",
  "datafusion-datasource",
- "datafusion-physical-expr-common",
+ "datafusion-physical-expr-adapter",
  "datafusion-physical-plan",
  "datafusion-session",
  "futures",
- "num-traits",
  "object_store",
 ]
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1365,9 +1209,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1382,16 +1226,15 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
+checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1401,6 +1244,7 @@ dependencies = [
  "datafusion-datasource",
  "datafusion-execution",
  "datafusion-expr",
+ "datafusion-functions",
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-adapter",
@@ -1409,7 +1253,7 @@ dependencies = [
  "datafusion-pruning",
  "datafusion-session",
  "futures",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
@@ -1419,20 +1263,19 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -1441,18 +1284,19 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -1462,8 +1306,7 @@ dependencies = [
  "datafusion-functions-window-common",
  "datafusion-physical-expr-common",
  "indexmap",
- "itertools",
- "paste",
+ "itertools 0.14.0",
  "recursive",
  "serde_json",
  "sqlparser",
@@ -1471,28 +1314,27 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap",
- "itertools",
- "paste",
+ "itertools 0.14.0",
 ]
 
 [[package]]
 name = "datafusion-ffi"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c"
+checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152"
 dependencies = [
- "abi_stable",
  "arrow",
  "arrow-schema",
  "async-ffi",
  "async-trait",
+ "chrono",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-datasource",
@@ -1501,22 +1343,25 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
  "datafusion-physical-plan",
  "datafusion-proto",
  "datafusion-proto-common",
  "datafusion-session",
  "futures",
+ "libloading",
  "log",
  "prost",
  "semver",
+ "stabby",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -1531,26 +1376,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
- "itertools",
+ "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "regex",
  "sha2",
- "unicode-segmentation",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -1560,19 +1404,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -1581,9 +1424,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -1597,34 +1440,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
- "itertools",
+ "hashbrown 0.17.1",
+ "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1635,14 +1478,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -1650,20 +1492,20 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
 dependencies = [
  "arrow",
  "chrono",
@@ -1672,7 +1514,7 @@ dependencies = [
  "datafusion-expr-common",
  "datafusion-physical-expr",
  "indexmap",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "recursive",
  "regex",
@@ -1681,11 +1523,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -1693,11 +1534,10 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap",
- "itertools",
+ "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "recursive",
  "tokio",
@@ -1705,9 +1545,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1715,31 +1555,31 @@ dependencies = [
  "datafusion-functions",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
- "itertools",
+ "itertools 0.14.0",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap",
- "itertools",
+ "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1750,18 +1590,19 @@ dependencies = [
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
  "datafusion-pruning",
- "itertools",
+ "itertools 0.14.0",
  "recursive",
 ]
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -1776,9 +1617,9 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "num-traits",
  "parking_lot",
@@ -1788,9 +1629,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-proto"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
+checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae"
 dependencies = [
  "arrow",
  "chrono",
@@ -1811,14 +1652,13 @@ dependencies = [
  "datafusion-proto-common",
  "object_store",
  "prost",
- "rand 0.9.4",
 ]
 
 [[package]]
 name = "datafusion-proto-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
+checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1827,9 +1667,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1838,24 +1678,25 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools",
  "log",
 ]
 
 [[package]]
 name = "datafusion-python"
-version = "53.0.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c25cc11d7d6c3916cb97d7b967110b0679646edadec7f242f720fe64288117b"
+checksum = "4b5da6aab44da83d488e6fb05855f5cd4480c7ab6b44f164166b495cba9fbbb8"
 dependencies = [
  "arrow",
  "arrow-select",
  "async-trait",
+ "chrono",
  "cstr",
  "datafusion",
  "datafusion-ffi",
  "datafusion-proto",
  "datafusion-python-util",
+ "datafusion-spark",
  "futures",
  "log",
  "object_store",
@@ -1864,7 +1705,7 @@ dependencies = [
  "prost-types",
  "pyo3",
  "pyo3-async-runtimes",
- "pyo3-build-config",
+ "pyo3-build-config 0.28.3",
  "pyo3-log",
  "serde_json",
  "tokio",
@@ -1874,13 +1715,14 @@ dependencies = [
 
 [[package]]
 name = "datafusion-python-util"
-version = "53.0.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d766b71c573229f5c9ffd32afd5678d1d2cbff1ff87cb3ef1d722aed7bed7db3"
+checksum = "ba491c15adfebc9ae53aa11f65231814b5199627e9815e2fb62d2f81f4a17fb2"
 dependencies = [
  "arrow",
  "datafusion",
  "datafusion-ffi",
+ "datafusion-proto",
  "prost",
  "pyo3",
  "tokio",
@@ -1888,9 +1730,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -1900,11 +1742,40 @@ dependencies = [
  "parking_lot",
 ]
 
+[[package]]
+name = "datafusion-spark"
+version = "54.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957"
+dependencies = [
+ "arrow",
+ "bigdecimal",
+ "chrono",
+ "crc32fast",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-nested",
+ "log",
+ "num-traits",
+ "percent-encoding",
+ "rand 0.9.5",
+ "serde_json",
+ "sha1",
+ "sha2",
+ "twox-hash",
+ "url",
+]
+
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -1936,20 +1807,20 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
 dependencies = [
- "block-buffer 0.12.0",
+ "block-buffer 0.12.1",
  "const-oid",
  "crypto-common 0.2.2",
 ]
 
 [[package]]
 name = "displaydoc"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2048,9 +1919,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
+checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2063,9 +1934,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
+checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -2073,15 +1944,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
+checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -2090,38 +1961,38 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
+checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
+checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "futures-sink"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
+checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
 
 [[package]]
 name = "futures-task"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
 
 [[package]]
 name = "futures-util"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2134,15 +2005,6 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "generational-arena"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7"
-dependencies = [
- "cfg-if",
-]
-
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -2173,25 +2035,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
 dependencies = [
  "cfg-if",
- "js-sys",
  "libc",
  "r-efi 5.3.0",
  "wasip2",
- "wasm-bindgen",
 ]
 
 [[package]]
 name = "getrandom"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "r-efi 6.0.0",
  "rand_core 0.10.1",
- "wasip2",
- "wasip3",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -2202,9 +2062,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
 
 [[package]]
 name = "h2"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -2248,9 +2108,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.16.1"
+version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
 dependencies = [
  "allocator-api2",
  "equivalent",
@@ -2258,14 +2118,8 @@ dependencies = [
 ]
 
 [[package]]
-name = "hashbrown"
-version = "0.17.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
-
-[[package]]
-name = "heck"
-version = "0.5.0"
+name = "heck"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
@@ -2277,9 +2131,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
 [[package]]
 name = "http"
-version = "1.4.0"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
 dependencies = [
  "bytes",
  "itoa",
@@ -2287,9 +2141,9 @@ dependencies = [
 
 [[package]]
 name = "http-body"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
 dependencies = [
  "bytes",
  "http",
@@ -2297,9 +2151,9 @@ dependencies = [
 
 [[package]]
 name = "http-body-util"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
 dependencies = [
  "bytes",
  "futures-core",
@@ -2322,24 +2176,24 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da"
+checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"
 dependencies = [
  "typenum",
 ]
 
 [[package]]
 name = "hyper"
-version = "1.9.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -2515,18 +2369,6 @@ dependencies = [
  "zerovec",
 ]
 
-[[package]]
-name = "id-arena"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-
-[[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
 [[package]]
 name = "idna"
 version = "1.1.0"
@@ -2556,15 +2398,13 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
 dependencies = [
  "equivalent",
  "hashbrown 0.17.1",
- "serde",
- "serde_core",
 ]
 
 [[package]]
 name = "insta"
-version = "1.47.2"
+version = "1.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e"
+checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
 dependencies = [
  "console",
  "once_cell",
@@ -2593,6 +2433,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -2601,32 +2450,25 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jobserver"
-version = "0.1.34"
+version = "0.1.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
 dependencies = [
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.99"
+version = "0.3.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
+checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
 dependencies = [
  "cfg-if",
  "futures-util",
- "once_cell",
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "leb128fmt"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -2698,28 +2540,28 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
 [[package]]
 name = "libloading"
-version = "0.7.4"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
 dependencies = [
  "cfg-if",
- "winapi",
+ "windows-link",
 ]
 
 [[package]]
 name = "liblzma"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899"
+checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba"
 dependencies = [
  "liblzma-sys",
 ]
 
 [[package]]
 name = "liblzma-sys"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6"
+checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5"
 dependencies = [
  "cc",
  "libc",
@@ -2755,9 +2597,18 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.29"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
+[[package]]
+name = "lru"
+version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
+dependencies = [
+ "hashbrown 0.17.1",
+]
 
 [[package]]
 name = "lru-slab"
@@ -2802,9 +2653,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.8.0"
+version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
 [[package]]
 name = "memory-stats"
@@ -2834,9 +2685,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.2.0"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
+checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
 dependencies = [
  "libc",
  "wasi",
@@ -2860,13 +2711,12 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.6"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
 dependencies = [
  "num-integer",
  "num-traits",
- "serde",
 ]
 
 [[package]]
@@ -2944,12 +2794,12 @@ dependencies = [
  "httparse",
  "humantime",
  "hyper",
- "itertools",
+ "itertools 0.14.0",
  "md-5 0.10.6",
  "parking_lot",
  "percent-encoding",
  "quick-xml",
- "rand 0.10.1",
+ "rand 0.10.2",
  "reqwest",
  "ring",
  "rustls-pki-types",
@@ -3104,7 +2954,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3121,9 +2971,9 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
 [[package]]
 name = "portable-atomic"
-version = "1.13.1"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
 
 [[package]]
 name = "potential_utf"
@@ -3150,7 +3000,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.117",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "proc-macro-crate"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
+dependencies = [
+ "toml_edit",
 ]
 
 [[package]]
@@ -3164,9 +3023,9 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -3174,12 +3033,12 @@ dependencies = [
 
 [[package]]
 name = "prost-build"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
+checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
  "heck",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "multimap",
  "petgraph",
@@ -3189,28 +3048,28 @@ dependencies = [
  "pulldown-cmark",
  "pulldown-cmark-to-cmark",
  "regex",
- "syn 2.0.117",
+ "syn 2.0.119",
  "tempfile",
 ]
 
 [[package]]
 name = "prost-derive"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
 dependencies = [
  "anyhow",
- "itertools",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "prost-types"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
+checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
 dependencies = [
  "prost",
 ]
@@ -3247,7 +3106,7 @@ dependencies = [
 
 [[package]]
 name = "pyballista"
-version = "53.0.0"
+version = "54.0.0"
 dependencies = [
  "async-trait",
  "ballista",
@@ -3259,7 +3118,7 @@ dependencies = [
  "datafusion-python",
  "log",
  "pyo3",
- "pyo3-build-config",
+ "pyo3-build-config 0.29.0",
  "pyo3-log",
  "tokio",
  "tonic",
@@ -3274,7 +3133,7 @@ dependencies = [
  "libc",
  "once_cell",
  "portable-atomic",
- "pyo3-build-config",
+ "pyo3-build-config 0.28.3",
  "pyo3-ffi",
  "pyo3-macros",
 ]
@@ -3298,6 +3157,16 @@ name = "pyo3-build-config"
 version = "0.28.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
+dependencies = [
+ "python3-dll-a",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
 dependencies = [
  "target-lexicon",
 ]
@@ -3309,14 +3178,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
 dependencies = [
  "libc",
- "pyo3-build-config",
+ "pyo3-build-config 0.28.3",
 ]
 
 [[package]]
 name = "pyo3-log"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26c2ec80932c5c3b2d4fbc578c9b56b2d4502098587edb8bef5b6bfcad43682e"
+checksum = "f64083bd3a16a353d9d62335808e8e13d0552d2a2b83fdb084496192dcfa9fcd"
 dependencies = [
  "arc-swap",
  "log",
@@ -3332,7 +3201,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3343,16 +3212,19 @@ checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
 dependencies = [
  "heck",
  "proc-macro2",
- "pyo3-build-config",
+ "pyo3-build-config 0.28.3",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
-name = "quad-rand"
-version = "0.2.3"
+name = "python3-dll-a"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
+checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b"
+dependencies = [
+ "cc",
+]
 
 [[package]]
 name = "quick-xml"
@@ -3366,9 +3238,9 @@ dependencies = [
 
 [[package]]
 name = "quinn"
-version = "0.11.9"
+version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
+checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
 dependencies = [
  "bytes",
  "cfg_aliases",
@@ -3386,14 +3258,15 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.14"
+version = "0.11.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
+checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
 dependencies = [
  "bytes",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "lru-slab",
- "rand 0.9.4",
+ "rand 0.10.2",
+ "rand_pcg",
  "ring",
  "rustc-hash",
  "rustls",
@@ -3407,23 +3280,23 @@ dependencies = [
 
 [[package]]
 name = "quinn-udp"
-version = "0.5.14"
+version = "0.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
+checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
 dependencies = [
  "cfg_aliases",
  "libc",
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.45"
+version = "1.0.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
 dependencies = [
  "proc-macro2",
 ]
@@ -3442,9 +3315,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
 
 [[package]]
 name = "rand"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
 dependencies = [
  "rand_chacha",
  "rand_core 0.9.5",
@@ -3452,12 +3325,12 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
+checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
 dependencies = [
  "chacha20",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "rand_core 0.10.1",
 ]
 
@@ -3486,6 +3359,15 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
 
+[[package]]
+name = "rand_pcg"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
+dependencies = [
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "recursive"
 version = "0.1.1"
@@ -3503,7 +3385,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
 dependencies = [
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3517,9 +3399,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.12.3"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -3529,35 +3411,20 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.14"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
 dependencies = [
  "aho-corasick",
  "memchr",
  "regex-syntax",
 ]
 
-[[package]]
-name = "regex-lite"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
-
 [[package]]
 name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
-
-[[package]]
-name = "repr_offset"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea"
-dependencies = [
- "tstr",
-]
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "reqwest"
@@ -3592,7 +3459,7 @@ dependencies = [
  "tokio-rustls",
  "tokio-util",
  "tower",
- "tower-http",
+ "tower-http 0.6.11",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -3617,9 +3484,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -3645,9 +3512,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.40"
+version = "0.23.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
+checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
 dependencies = [
  "once_cell",
  "ring",
@@ -3659,9 +3526,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-native-certs"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
 dependencies = [
  "openssl-probe",
  "rustls-pki-types",
@@ -3671,9 +3538,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
 dependencies = [
  "web-time",
  "zeroize",
@@ -3692,9 +3559,9 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.22"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
 
 [[package]]
 name = "ryu"
@@ -3763,42 +3630,32 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
 ]
 
-[[package]]
-name = "serde_bytes"
-version = "0.11.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
-dependencies = [
- "serde",
- "serde_core",
-]
-
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 3.0.0",
 ]
 
 [[package]]
@@ -3837,22 +3694,39 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "sha1"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.11.3",
+]
+
 [[package]]
 name = "sha2"
-version = "0.10.9"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
 dependencies = [
  "cfg-if",
- "cpufeatures 0.2.17",
- "digest 0.10.7",
+ "cpufeatures",
+ "digest 0.11.3",
 ]
 
+[[package]]
+name = "sha2-const-stable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9"
+
 [[package]]
 name = "shlex"
-version = "1.3.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
 
 [[package]]
 name = "signal-hook-registry"
@@ -3866,9 +3740,9 @@ dependencies = [
 
 [[package]]
 name = "simd-adler32"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
 
 [[package]]
 name = "simdutf8"
@@ -3896,21 +3770,21 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "smallvec"
-version = "1.15.1"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snap"
-version = "1.1.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
+checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
 
 [[package]]
 name = "socket2"
-version = "0.6.3"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
 dependencies = [
  "libc",
  "windows-sys 0.61.2",
@@ -3918,9 +3792,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "recursive",
@@ -3935,7 +3809,41 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "stabby"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec"
+dependencies = [
+ "rustversion",
+ "stabby-abi",
+]
+
+[[package]]
+name = "stabby-abi"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4"
+dependencies = [
+ "rustc_version",
+ "rustversion",
+ "sha2-const-stable",
+ "stabby-macros",
+]
+
+[[package]]
+name = "stabby-macros"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3957,28 +3865,16 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "strum"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
-
 [[package]]
 name = "strum_macros"
-version = "0.27.2"
+version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
+checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3989,9 +3885,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
-version = "1.0.109"
+version = "2.0.119"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4000,9 +3896,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.117"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+checksum = "f2fac314a64dc9a36e61a9eb4261a5e9bbfbc922b27e518af97bc32b926cf967"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4026,7 +3922,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4056,7 +3952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -4064,22 +3960,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 3.0.0",
 ]
 
 [[package]]
@@ -4114,9 +4010,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
+checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -4129,9 +4025,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.52.3"
+version = "1.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
 dependencies = [
  "bytes",
  "libc",
@@ -4146,13 +4042,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.7.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
+checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4190,6 +4086,36 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.25.13+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
+dependencies = [
+ "winnow",
+]
+
 [[package]]
 name = "tonic"
 version = "0.14.6"
@@ -4228,7 +4154,7 @@ dependencies = [
  "prettyplease",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4253,7 +4179,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "tempfile",
  "tonic-build",
 ]
@@ -4295,6 +4221,21 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "tower-http"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
+dependencies = [
+ "bitflags",
+ "bytes",
+ "http",
+ "percent-encoding",
+ "pin-project-lite",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "tower-layer"
 version = "0.3.3"
@@ -4327,7 +4268,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4345,44 +4286,20 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "tstr"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7"
-dependencies = [
- "tstr_proc_macros",
-]
-
-[[package]]
-name = "tstr_proc_macros"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a"
-
 [[package]]
 name = "twox-hash"
 version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
-
-[[package]]
-name = "typed-arena"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
+dependencies = [
+ "rand 0.9.5",
+]
 
 [[package]]
 name = "typenum"
-version = "1.20.0"
+version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
-
-[[package]]
-name = "typewit"
-version = "1.15.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
 [[package]]
 name = "unicase"
@@ -4398,9 +4315,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.13.2"
+version = "1.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
 
 [[package]]
 name = "unicode-width"
@@ -4408,12 +4325,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -4440,13 +4351,12 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
 [[package]]
 name = "uuid"
-version = "1.23.1"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "js-sys",
- "serde_core",
  "wasm-bindgen",
 ]
 
@@ -4483,27 +4393,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
 name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
-dependencies = [
- "wit-bindgen 0.57.1",
-]
-
-[[package]]
-name = "wasip3"
-version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
+version = "1.0.4+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
 dependencies = [
- "wit-bindgen 0.51.0",
+ "wit-bindgen",
 ]
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.122"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
+checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -4514,9 +4415,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.72"
+version = "0.4.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
+checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -4524,9 +4425,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.122"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
+checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -4534,48 +4435,26 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.122"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
+checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
 dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.122"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
+checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
 dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "wasm-encoder"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
-dependencies = [
- "leb128fmt",
- "wasmparser",
-]
-
-[[package]]
-name = "wasm-metadata"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
-dependencies = [
- "anyhow",
- "indexmap",
- "wasm-encoder",
- "wasmparser",
-]
-
 [[package]]
 name = "wasm-streams"
 version = "0.4.2"
@@ -4589,23 +4468,11 @@ dependencies = [
  "web-sys",
 ]
 
-[[package]]
-name = "wasmparser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
-dependencies = [
- "bitflags",
- "hashbrown 0.15.5",
- "indexmap",
- "semver",
-]
-
 [[package]]
 name = "web-sys"
-version = "0.3.99"
+version = "0.3.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
+checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -4705,7 +4572,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4716,7 +4583,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4759,16 +4626,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.6",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.60.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
-dependencies = [
- "windows-targets 0.53.5",
+ "windows-targets",
 ]
 
 [[package]]
@@ -4786,31 +4644,14 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.6",
- "windows_aarch64_msvc 0.52.6",
- "windows_i686_gnu 0.52.6",
- "windows_i686_gnullvm 0.52.6",
- "windows_i686_msvc 0.52.6",
- "windows_x86_64_gnu 0.52.6",
- "windows_x86_64_gnullvm 0.52.6",
- "windows_x86_64_msvc 0.52.6",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.53.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
-dependencies = [
- "windows-link",
- "windows_aarch64_gnullvm 0.53.1",
- "windows_aarch64_msvc 0.53.1",
- "windows_i686_gnu 0.53.1",
- "windows_i686_gnullvm 0.53.1",
- "windows_i686_msvc 0.53.1",
- "windows_x86_64_gnu 0.53.1",
- "windows_x86_64_gnullvm 0.53.1",
- "windows_x86_64_msvc 0.53.1",
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
 ]
 
 [[package]]
@@ -4828,84 +4669,42 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
-
 [[package]]
 name = "windows_i686_gnullvm"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
-[[package]]
-name = "windows_i686_gnullvm"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.52.6"
@@ -4913,18 +4712,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
-name = "windows_x86_64_msvc"
-version = "0.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
-
-[[package]]
-name = "wit-bindgen"
-version = "0.51.0"
+name = "winnow"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
 dependencies = [
- "wit-bindgen-rust-macro",
+ "memchr",
 ]
 
 [[package]]
@@ -4933,85 +4726,6 @@ version = "0.57.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 
-[[package]]
-name = "wit-bindgen-core"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
-dependencies = [
- "anyhow",
- "heck",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-bindgen-rust"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
-dependencies = [
- "anyhow",
- "heck",
- "indexmap",
- "prettyplease",
- "syn 2.0.117",
- "wasm-metadata",
- "wit-bindgen-core",
- "wit-component",
-]
-
-[[package]]
-name = "wit-bindgen-rust-macro"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
-dependencies = [
- "anyhow",
- "prettyplease",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
- "wit-bindgen-core",
- "wit-bindgen-rust",
-]
-
-[[package]]
-name = "wit-component"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
-dependencies = [
- "anyhow",
- "bitflags",
- "indexmap",
- "log",
- "serde",
- "serde_derive",
- "serde_json",
- "wasm-encoder",
- "wasm-metadata",
- "wasmparser",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-parser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
-dependencies = [
- "anyhow",
- "id-arena",
- "indexmap",
- "log",
- "semver",
- "serde",
- "serde_derive",
- "serde_json",
- "unicode-xid",
- "wasmparser",
-]
-
 [[package]]
 name = "writeable"
 version = "0.6.3"
@@ -5020,9 +4734,9 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
 
 [[package]]
 name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -5037,28 +4751,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "synstructure",
 ]
 
 [[package]]
 name = "zerocopy"
-version = "0.8.48"
+version = "0.8.54"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
+checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.48"
+version = "0.8.54"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
+checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5078,15 +4792,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "synstructure",
 ]
 
 [[package]]
 name = "zeroize"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
 
 [[package]]
 name = "zerotrie"
@@ -5118,20 +4832,20 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "zlib-rs"
-version = "0.6.3"
+version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
+checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"
 
 [[package]]
 name = "zmij"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
 
 [[package]]
 name = "zstd"
diff --git a/python/Cargo.toml b/python/Cargo.toml
index 8531c0f451..0712605831 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -17,11 +17,11 @@
 
 [package]
 name = "pyballista"
-version = "53.0.0"
+version = "54.0.0"
 homepage = "https://datafusion.apache.org/ballista/"
 repository = "https://github.com/apache/datafusion-ballista"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
-description = "Apache Arrow Ballista Python Client"
+description = "Apache DataFusion Ballista Python Client"
 readme = "README.md"
 license = "Apache-2.0"
 edition = "2021"
@@ -31,13 +31,13 @@ publish = false
 
 [dependencies]
 async-trait = "0.1.89"
-ballista = { path = "../ballista/client", version = "53.0.0" }
-ballista-core = { path = "../ballista/core", version = "53.0.0" }
-ballista-executor = { path = "../ballista/executor", version = "53.0.0", default-features = false }
-ballista-scheduler = { path = "../ballista/scheduler", version = "53.0.0", default-features = false }
-datafusion = { version = "53", features = ["avro"] }
-datafusion-proto = { version = "53" }
-datafusion-python = { version = "53", default-features = false }
+ballista = { path = "../ballista/client", version = "54.0.0" }
+ballista-core = { path = "../ballista/core", version = "54.0.0" }
+ballista-executor = { path = "../ballista/executor", version = "54.0.0", default-features = false }
+ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", default-features = false }
+datafusion = { version = "54", features = ["avro"] }
+datafusion-proto = { version = "54" }
+datafusion-python = { version = "54", default-features = false }
 log = { version = "0.4" }
 pyo3 = { version = "0.28", features = ["extension-module", "abi3", "abi3-py310"] }
 pyo3-log = "0.13"
@@ -49,4 +49,4 @@ crate-type = ["cdylib"]
 name = "ballista"
 
 [build-dependencies]
-pyo3-build-config = "0.28"
+pyo3-build-config = "0.29"
diff --git a/python/pyproject.toml b/python/pyproject.toml
index bbf077fd8c..081c504b4d 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -21,7 +21,7 @@ build-backend = "maturin"
 
 [project]
 name = "ballista"
-description = "Python client for Apache Arrow Ballista Distributed SQL Query Engine"
+description = "Python client for Apache DataFusion Ballista Distributed SQL Query Engine"
 readme = "README.md"
 license = {file = "LICENSE.txt"}
 requires-python = ">=3.10"
@@ -40,8 +40,8 @@ classifiers = [
     "Programming Language :: Rust",
 ]
 dependencies = [
-    "pyarrow>=21.0.0",
-    "datafusion==53",
+    "pyarrow>=23.0.1",
+    "datafusion==54",
     "typing-extensions;python_version<'3.13'",
 ]
 dynamic = ["version"]
diff --git a/python/src/cluster.rs b/python/src/cluster.rs
index 13891f067f..baec020dad 100644
--- a/python/src/cluster.rs
+++ b/python/src/cluster.rs
@@ -141,7 +141,7 @@ pub struct PyExecutor {
 //        which forked it. This will be investigated further
 #[pymethods]
 impl PyExecutor {
-    #[pyo3(signature = (bind_port=None, bind_host =None, scheduler_host = None, scheduler_port = None, concurrent_tasks = None))]
+    #[pyo3(signature = (bind_port=None, bind_host =None, scheduler_host = None, scheduler_port = None, vcores = None))]
     #[new]
     pub fn new(
         _py: Python,
@@ -149,7 +149,7 @@ impl PyExecutor {
         bind_host: Option<String>,
         scheduler_host: Option<String>,
         scheduler_port: Option<u16>,
-        concurrent_tasks: Option<u16>,
+        vcores: Option<u16>,
     ) -> PyResult<Self> {
         let mut config = ExecutorProcessConfig::default();
         if let Some(port) = bind_port {
@@ -168,8 +168,8 @@ impl PyExecutor {
             config.scheduler_host = host;
         }
 
-        if let Some(concurrent_tasks) = concurrent_tasks {
-            config.concurrent_tasks = concurrent_tasks as usize
+        if let Some(vcores) = vcores {
+            config.vcores = vcores as usize
         }
 
         let config = Arc::new(config);
@@ -248,12 +248,12 @@ impl PyExecutor {
 
     pub fn __repr__(&self) -> String {
         format!(
-            "BallistaExecutor(address={}:{}, scheduler={}:{}, concurrent_tasks={} listening={})",
+            "BallistaExecutor(address={}:{}, scheduler={}:{}, vcores={} listening={})",
             self.config.bind_host,
             self.config.port,
             self.config.scheduler_host,
             self.config.scheduler_port,
-            self.config.concurrent_tasks,
+            self.config.vcores,
             self.handle.is_some()
         )
     }
diff --git a/python/uv.lock b/python/uv.lock
index 9852555f5f..54493c14b0 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -52,9 +52,9 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "datafusion", specifier = "==53" },
+    { name = "datafusion", specifier = "==54" },
     { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=8.0.0" },
-    { name = "pyarrow", specifier = ">=21.0.0" },
+    { name = "pyarrow", specifier = ">=23.0.1" },
     { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
 provides-extras = ["jupyter"]
@@ -250,6 +250,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" },
 ]
 
+[[package]]
+name = "cloudpickle"
+version = "3.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" },
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -261,84 +270,85 @@ wheels = [
 
 [[package]]
 name = "cryptography"
-version = "46.0.3"
+version = "48.0.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
     { name = "typing-extensions", marker = "python_full_version < '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/1d/42/9c391dd801d6cf0d561b5890549d4b27bafcc53b39c31a817e69d87c625b/cryptography-46.0.3-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a", size = 7225004, upload-time = "2025-10-15T23:16:52.239Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/67/38769ca6b65f07461eb200e85fc1639b438bdc667be02cf7f2cd6a64601c/cryptography-46.0.3-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc", size = 4296667, upload-time = "2025-10-15T23:16:54.369Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" },
-    { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" },
-    { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" },
-    { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" },
-    { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" },
-    { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" },
-    { url = "https://files.pythonhosted.org/packages/96/92/8a6a9525893325fc057a01f654d7efc2c64b9de90413adcf605a85744ff4/cryptography-46.0.3-cp311-abi3-win32.whl", hash = "sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018", size = 3055988, upload-time = "2025-10-15T23:17:14.65Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/bf/80fbf45253ea585a1e492a6a17efcb93467701fa79e71550a430c5e60df0/cryptography-46.0.3-cp311-abi3-win_amd64.whl", hash = "sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb", size = 3514451, upload-time = "2025-10-15T23:17:16.142Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/af/9b302da4c87b0beb9db4e756386a7c6c5b8003cd0e742277888d352ae91d/cryptography-46.0.3-cp311-abi3-win_arm64.whl", hash = "sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c", size = 2928007, upload-time = "2025-10-15T23:17:18.04Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/e2/a510aa736755bffa9d2f75029c229111a1d02f8ecd5de03078f4c18d91a3/cryptography-46.0.3-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:00a5e7e87938e5ff9ff5447ab086a5706a957137e6e433841e9d24f38a065217", size = 7158012, upload-time = "2025-10-15T23:17:19.982Z" },
-    { url = "https://files.pythonhosted.org/packages/73/dc/9aa866fbdbb95b02e7f9d086f1fccfeebf8953509b87e3f28fff927ff8a0/cryptography-46.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8daeb2d2174beb4575b77482320303f3d39b8e81153da4f0fb08eb5fe86a6c5", size = 4288728, upload-time = "2025-10-15T23:17:21.527Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" },
-    { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" },
-    { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" },
-    { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" },
-    { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" },
-    { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" },
-    { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/30/27654c1dbaf7e4a3531fa1fc77986d04aefa4d6d78259a62c9dc13d7ad36/cryptography-46.0.3-cp314-cp314t-win32.whl", hash = "sha256:8a6e050cb6164d3f830453754094c086ff2d0b2f3a897a1d9820f6139a1f0914", size = 3022339, upload-time = "2025-10-15T23:17:40.888Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/30/640f34ccd4d2a1bc88367b54b926b781b5a018d65f404d409aba76a84b1c/cryptography-46.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:760f83faa07f8b64e9c33fc963d790a2edb24efb479e3520c14a45741cd9b2db", size = 3494315, upload-time = "2025-10-15T23:17:42.769Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/8b/88cc7e3bd0a8e7b861f26981f7b820e1f46aa9d26cc482d0feba0ecb4919/cryptography-46.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:516ea134e703e9fe26bcd1277a4b59ad30586ea90c365a87781d7887a646fe21", size = 2919331, upload-time = "2025-10-15T23:17:44.468Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/23/45fe7f376a7df8daf6da3556603b36f53475a99ce4faacb6ba2cf3d82021/cryptography-46.0.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936", size = 7218248, upload-time = "2025-10-15T23:17:46.294Z" },
-    { url = "https://files.pythonhosted.org/packages/27/32/b68d27471372737054cbd34c84981f9edbc24fe67ca225d389799614e27f/cryptography-46.0.3-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683", size = 4294089, upload-time = "2025-10-15T23:17:48.269Z" },
-    { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" },
-    { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" },
-    { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" },
-    { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" },
-    { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" },
-    { url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/cd/1a8633802d766a0fa46f382a77e096d7e209e0817892929655fe0586ae32/cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32", size = 3689163, upload-time = "2025-10-15T23:18:13.821Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/59/6b26512964ace6480c3e54681a9859c974172fb141c38df11eadd8416947/cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c", size = 3429474, upload-time = "2025-10-15T23:18:15.477Z" },
-    { url = "https://files.pythonhosted.org/packages/06/8a/e60e46adab4362a682cf142c7dcb5bf79b782ab2199b0dcb81f55970807f/cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea", size = 3698132, upload-time = "2025-10-15T23:18:17.056Z" },
-    { url = "https://files.pythonhosted.org/packages/da/38/f59940ec4ee91e93d3311f7532671a5cef5570eb04a144bf203b58552d11/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b", size = 4243992, upload-time = "2025-10-15T23:18:18.695Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/0c/35b3d92ddebfdfda76bb485738306545817253d0a3ded0bfe80ef8e67aa5/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb", size = 4409944, upload-time = "2025-10-15T23:18:20.597Z" },
-    { url = "https://files.pythonhosted.org/packages/99/55/181022996c4063fc0e7666a47049a1ca705abb9c8a13830f074edb347495/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:9394673a9f4de09e28b5356e7fff97d778f8abad85c9d5ac4a4b7e25a0de7717", size = 4242957, upload-time = "2025-10-15T23:18:22.18Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/af/72cd6ef29f9c5f731251acadaeb821559fe25f10852f44a63374c9ca08c1/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:94cd0549accc38d1494e1f8de71eca837d0509d0d44bf11d158524b0e12cebf9", size = 4409447, upload-time = "2025-10-15T23:18:24.209Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/c3/e90f4a4feae6410f914f8ebac129b9ae7a8c92eb60a638012dde42030a9d/cryptography-46.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6b5063083824e5509fdba180721d55909ffacccc8adbec85268b48439423d78c", size = 3438528, upload-time = "2025-10-15T23:18:26.227Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" },
+    { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" },
+    { url = "https://files.pythonhosted.org/packages/89/c6/24266ac10c47f6cd2a865f4446062b466da1d1f10b27189eac00e61bf0c9/cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", size = 5300085, upload-time = "2026-06-09T22:31:58.703Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/bb/cc4b78784f97efc8c5874c2a9743708d172be6663024b34a0467885ae0c8/cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", size = 4746137, upload-time = "2026-06-09T22:31:31.01Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/52/0c44de3f5267f8fbe8e835138017522a333436166e406f0db9b9e6e3033f/cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", size = 4333867, upload-time = "2026-06-09T22:32:28.096Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/2e/772d7adbfa931537bc401640b7cac9976bff689bda187833e5d63b428e49/cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", size = 4701805, upload-time = "2026-06-09T22:31:38.284Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/a3/b06844f303873493c963caf581c04df31c7035e0c1b0f02c4814d319ec80/cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", size = 5258461, upload-time = "2026-06-09T22:31:04.187Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" },
+    { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" },
+    { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" },
+    { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" },
+    { url = "https://files.pythonhosted.org/packages/24/dc/d8ca05ffea724eec6d232ea6f18e74c269eb6bdfdcc9bfba689790d1325f/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:e361afba8918070d376df76f408a4f67fec0ee9cff81a99e48fe9a233ef59e17", size = 5290385, upload-time = "2026-06-09T22:31:15.212Z" },
+    { url = "https://files.pythonhosted.org/packages/03/8c/3be6cb4da181f5bb6c19cf560c2359d60644a6b5fc5b57854e528f47b296/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d069066deead00ac7f090be101be875a06855908f7ec004c27b8fefb4acfb411", size = 4737082, upload-time = "2026-06-09T22:32:22.66Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/f6/d5f60a5a1434dbfd949e227fd0065d194c7e6b6ac526b17f5c06152b8231/cryptography-48.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:09f73a725d582cef64b91281a322cd798d14a33b2b6f2b7ad9531dc336d84c02", size = 4325328, upload-time = "2026-06-09T22:32:10.777Z" },
+    { url = "https://files.pythonhosted.org/packages/17/b7/ba75dd947a14b6ad907b01ae8f6b5b348cdd1b48142f0063dee9e20c1d9d/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:15254441469dd6bf027039453288e2072124f8b6603563f5d759e1c9b69273fa", size = 4694530, upload-time = "2026-06-09T22:31:53.105Z" },
+    { url = "https://files.pythonhosted.org/packages/62/29/50d6b9e8aff12d8b67afaeb3569335e32dc83a5723e3bbded24fdac9f809/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:8ace4507d1e6533c125f4fac754f8bb8b6a74c08e92179dabd7e16571a3efbf3", size = 5245046, upload-time = "2026-06-09T22:31:25.774Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" },
+    { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" },
+    { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" },
+    { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/0b/aa68b221dde92d09cb29a024ede17550ee21e77a404e59fc093c82bb51e1/cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", size = 5289970, upload-time = "2026-06-09T22:31:20.368Z" },
+    { url = "https://files.pythonhosted.org/packages/78/13/fba657f958d2af66ea959a4ba01212632089249d34af1ae48054136344d7/cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", size = 4731873, upload-time = "2026-06-09T22:31:22.253Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/4c/9a964756d24a26b3e34dfcb16f961b89838786e6700b635b0d1e3adff4b6/cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", size = 4330804, upload-time = "2026-06-09T22:31:36.56Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/0f/a10f3a6eb12950a10e3a874070283aa2dd5875b2bfd15fad8a3e17b3f13e/cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46", size = 4696217, upload-time = "2026-06-09T22:31:13.351Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/6f/5cd12f951165ea73ef85266775d97e4c763b2474ccfd816dd69d3a18d6f8/cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", size = 5245252, upload-time = "2026-06-09T22:32:02.193Z" },
+    { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" },
+    { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" },
+    { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" },
+    { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" },
+    { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" },
 ]
 
 [[package]]
 name = "datafusion"
-version = "53.0.0"
+version = "54.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "cloudpickle" },
     { name = "pyarrow" },
     { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" },
-    { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" },
+    { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" },
+    { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" },
+    { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" },
+    { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" },
+    { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" },
+    { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" },
 ]
 
 [[package]]
@@ -583,59 +593,59 @@ wheels = [
 
 [[package]]
 name = "pyarrow"
-version = "23.0.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/01/33/ffd9c3eb087fa41dd79c3cf20c4c0ae3cdb877c4f8e1107a446006344924/pyarrow-23.0.0.tar.gz", hash = "sha256:180e3150e7edfcd182d3d9afba72f7cf19839a497cc76555a8dce998a8f67615", size = 1167185, upload-time = "2026-01-18T16:19:42.218Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ae/2f/23e042a5aa99bcb15e794e14030e8d065e00827e846e53a66faec73c7cd6/pyarrow-23.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:cbdc2bf5947aa4d462adcf8453cf04aee2f7932653cb67a27acd96e5e8528a67", size = 34281861, upload-time = "2026-01-18T16:13:34.332Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/65/1651933f504b335ec9cd8f99463718421eb08d883ed84f0abd2835a16cad/pyarrow-23.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:4d38c836930ce15cd31dce20114b21ba082da231c884bdc0a7b53e1477fe7f07", size = 35825067, upload-time = "2026-01-18T16:13:42.549Z" },
-    { url = "https://files.pythonhosted.org/packages/84/ec/d6fceaec050c893f4e35c0556b77d4cc9973fcc24b0a358a5781b1234582/pyarrow-23.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4222ff8f76919ecf6c716175a0e5fddb5599faeed4c56d9ea41a2c42be4998b2", size = 44458539, upload-time = "2026-01-18T16:13:52.975Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/d9/369f134d652b21db62fe3ec1c5c2357e695f79eb67394b8a93f3a2b2cffa/pyarrow-23.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:87f06159cbe38125852657716889296c83c37b4d09a5e58f3d10245fd1f69795", size = 47535889, upload-time = "2026-01-18T16:14:03.693Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/95/f37b6a252fdbf247a67a78fb3f61a529fe0600e304c4d07741763d3522b1/pyarrow-23.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1675c374570d8b91ea6d4edd4608fa55951acd44e0c31bd146e091b4005de24f", size = 48157777, upload-time = "2026-01-18T16:14:12.483Z" },
-    { url = "https://files.pythonhosted.org/packages/ab/ab/fb94923108c9c6415dab677cf1f066d3307798eafc03f9a65ab4abc61056/pyarrow-23.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:247374428fde4f668f138b04031a7e7077ba5fa0b5b1722fdf89a017bf0b7ee0", size = 50580441, upload-time = "2026-01-18T16:14:20.187Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/78/897ba6337b517fc8e914891e1bd918da1c4eb8e936a553e95862e67b80f6/pyarrow-23.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:de53b1bd3b88a2ee93c9af412c903e57e738c083be4f6392288294513cd8b2c1", size = 27530028, upload-time = "2026-01-18T16:14:27.353Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/c0/57fe251102ca834fee0ef69a84ad33cc0ff9d5dfc50f50b466846356ecd7/pyarrow-23.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5574d541923efcbfdf1294a2746ae3b8c2498a2dc6cd477882f6f4e7b1ac08d3", size = 34276762, upload-time = "2026-01-18T16:14:34.128Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/4e/24130286548a5bc250cbed0b6bbf289a2775378a6e0e6f086ae8c68fc098/pyarrow-23.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:2ef0075c2488932e9d3c2eb3482f9459c4be629aa673b725d5e3cf18f777f8e4", size = 35821420, upload-time = "2026-01-18T16:14:40.699Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/55/a869e8529d487aa2e842d6c8865eb1e2c9ec33ce2786eb91104d2c3e3f10/pyarrow-23.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:65666fc269669af1ef1c14478c52222a2aa5c907f28b68fb50a203c777e4f60c", size = 44457412, upload-time = "2026-01-18T16:14:49.051Z" },
-    { url = "https://files.pythonhosted.org/packages/36/81/1de4f0edfa9a483bbdf0082a05790bd6a20ed2169ea12a65039753be3a01/pyarrow-23.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4d85cb6177198f3812db4788e394b757223f60d9a9f5ad6634b3e32be1525803", size = 47534285, upload-time = "2026-01-18T16:14:56.748Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/04/464a052d673b5ece074518f27377861662449f3c1fdb39ce740d646fd098/pyarrow-23.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1a9ff6fa4141c24a03a1a434c63c8fa97ce70f8f36bccabc18ebba905ddf0f17", size = 48157913, upload-time = "2026-01-18T16:15:05.114Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/1b/32a4de9856ee6688c670ca2def588382e573cce45241a965af04c2f61687/pyarrow-23.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:84839d060a54ae734eb60a756aeacb62885244aaa282f3c968f5972ecc7b1ecc", size = 50582529, upload-time = "2026-01-18T16:15:12.846Z" },
-    { url = "https://files.pythonhosted.org/packages/db/c7/d6581f03e9b9e44ea60b52d1750ee1a7678c484c06f939f45365a45f7eef/pyarrow-23.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a149a647dbfe928ce8830a713612aa0b16e22c64feac9d1761529778e4d4eaa5", size = 27542646, upload-time = "2026-01-18T16:15:18.89Z" },
-    { url = "https://files.pythonhosted.org/packages/3d/bd/c861d020831ee57609b73ea721a617985ece817684dc82415b0bc3e03ac3/pyarrow-23.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5961a9f646c232697c24f54d3419e69b4261ba8a8b66b0ac54a1851faffcbab8", size = 34189116, upload-time = "2026-01-18T16:15:28.054Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/23/7725ad6cdcbaf6346221391e7b3eecd113684c805b0a95f32014e6fa0736/pyarrow-23.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:632b3e7c3d232f41d64e1a4a043fb82d44f8a349f339a1188c6a0dd9d2d47d8a", size = 35803831, upload-time = "2026-01-18T16:15:33.798Z" },
-    { url = "https://files.pythonhosted.org/packages/57/06/684a421543455cdc2944d6a0c2cc3425b028a4c6b90e34b35580c4899743/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:76242c846db1411f1d6c2cc3823be6b86b40567ee24493344f8226ba34a81333", size = 44436452, upload-time = "2026-01-18T16:15:41.598Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/6f/8f9eb40c2328d66e8b097777ddcf38494115ff9f1b5bc9754ba46991191e/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b73519f8b52ae28127000986bf228fda781e81d3095cd2d3ece76eb5cf760e1b", size = 47557396, upload-time = "2026-01-18T16:15:51.252Z" },
-    { url = "https://files.pythonhosted.org/packages/10/6e/f08075f1472e5159553501fde2cc7bc6700944bdabe49a03f8a035ee6ccd/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:068701f6823449b1b6469120f399a1239766b117d211c5d2519d4ed5861f75de", size = 48147129, upload-time = "2026-01-18T16:16:00.299Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/82/d5a680cd507deed62d141cc7f07f7944a6766fc51019f7f118e4d8ad0fb8/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1801ba947015d10e23bca9dd6ef5d0e9064a81569a89b6e9a63b59224fd060df", size = 50596642, upload-time = "2026-01-18T16:16:08.502Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/26/4f29c61b3dce9fa7780303b86895ec6a0917c9af927101daaaf118fbe462/pyarrow-23.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:52265266201ec25b6839bf6bd4ea918ca6d50f31d13e1cf200b4261cd11dc25c", size = 27660628, upload-time = "2026-01-18T16:16:15.28Z" },
-    { url = "https://files.pythonhosted.org/packages/66/34/564db447d083ec7ff93e0a883a597d2f214e552823bfc178a2d0b1f2c257/pyarrow-23.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:ad96a597547af7827342ffb3c503c8316e5043bb09b47a84885ce39394c96e00", size = 34184630, upload-time = "2026-01-18T16:16:22.141Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/3a/3999daebcb5e6119690c92a621c4d78eef2ffba7a0a1b56386d2875fcd77/pyarrow-23.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:b9edf990df77c2901e79608f08c13fbde60202334a4fcadb15c1f57bf7afee43", size = 35796820, upload-time = "2026-01-18T16:16:29.441Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/ee/39195233056c6a8d0976d7d1ac1cd4fe21fb0ec534eca76bc23ef3f60e11/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:36d1b5bc6ddcaff0083ceec7e2561ed61a51f49cce8be079ee8ed406acb6fdef", size = 44438735, upload-time = "2026-01-18T16:16:38.79Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/41/6a7328ee493527e7afc0c88d105ecca69a3580e29f2faaeac29308369fd7/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4292b889cd224f403304ddda8b63a36e60f92911f89927ec8d98021845ea21be", size = 47557263, upload-time = "2026-01-18T16:16:46.248Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/ee/34e95b21ee84db494eae60083ddb4383477b31fb1fd19fd866d794881696/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dfd9e133e60eaa847fd80530a1b89a052f09f695d0b9c34c235ea6b2e0924cf7", size = 48153529, upload-time = "2026-01-18T16:16:53.412Z" },
-    { url = "https://files.pythonhosted.org/packages/52/88/8a8d83cea30f4563efa1b7bf51d241331ee5cd1b185a7e063f5634eca415/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832141cc09fac6aab1cd3719951d23301396968de87080c57c9a7634e0ecd068", size = 50598851, upload-time = "2026-01-18T16:17:01.133Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/4c/2929c4be88723ba025e7b3453047dc67e491c9422965c141d24bab6b5962/pyarrow-23.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:7a7d067c9a88faca655c71bcc30ee2782038d59c802d57950826a07f60d83c4c", size = 27577747, upload-time = "2026-01-18T16:18:02.413Z" },
-    { url = "https://files.pythonhosted.org/packages/64/52/564a61b0b82d72bd68ec3aef1adda1e3eba776f89134b9ebcb5af4b13cb6/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:ce9486e0535a843cf85d990e2ec5820a47918235183a5c7b8b97ed7e92c2d47d", size = 34446038, upload-time = "2026-01-18T16:17:07.861Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/c9/232d4f9855fd1de0067c8a7808a363230d223c83aeee75e0fe6eab851ba9/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:075c29aeaa685fd1182992a9ed2499c66f084ee54eea47da3eb76e125e06064c", size = 35921142, upload-time = "2026-01-18T16:17:15.401Z" },
-    { url = "https://files.pythonhosted.org/packages/96/f2/60af606a3748367b906bb82d41f0032e059f075444445d47e32a7ff1df62/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:799965a5379589510d888be3094c2296efd186a17ca1cef5b77703d4d5121f53", size = 44490374, upload-time = "2026-01-18T16:17:23.93Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/2d/7731543050a678ea3a413955a2d5d80d2a642f270aa57a3cb7d5a86e3f46/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ef7cac8fe6fccd8b9e7617bfac785b0371a7fe26af59463074e4882747145d40", size = 47527896, upload-time = "2026-01-18T16:17:33.393Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/90/f3342553b7ac9879413aed46500f1637296f3c8222107523a43a1c08b42a/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15a414f710dc927132dd67c361f78c194447479555af57317066ee5116b90e9e", size = 48210401, upload-time = "2026-01-18T16:17:42.012Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/da/9862ade205ecc46c172b6ce5038a74b5151c7401e36255f15975a45878b2/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e0d2e6915eca7d786be6a77bf227fbc06d825a75b5b5fe9bcbef121dec32685", size = 50579677, upload-time = "2026-01-18T16:17:50.241Z" },
-    { url = "https://files.pythonhosted.org/packages/c2/4c/f11f371f5d4740a5dafc2e11c76bcf42d03dfdb2d68696da97de420b6963/pyarrow-23.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4b317ea6e800b5704e5e5929acb6e2dc13e9276b708ea97a39eb8b345aa2658b", size = 27631889, upload-time = "2026-01-18T16:17:56.55Z" },
-    { url = "https://files.pythonhosted.org/packages/97/bb/15aec78bcf43a0c004067bd33eb5352836a29a49db8581fc56f2b6ca88b7/pyarrow-23.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:20b187ed9550d233a872074159f765f52f9d92973191cd4b93f293a19efbe377", size = 34213265, upload-time = "2026-01-18T16:18:07.904Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/6c/deb2c594bbba41c37c5d9aa82f510376998352aa69dfcb886cb4b18ad80f/pyarrow-23.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:18ec84e839b493c3886b9b5e06861962ab4adfaeb79b81c76afbd8d84c7d5fda", size = 35819211, upload-time = "2026-01-18T16:18:13.94Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/e5/ee82af693cb7b5b2b74f6524cdfede0e6ace779d7720ebca24d68b57c36b/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e438dd3f33894e34fd02b26bd12a32d30d006f5852315f611aa4add6c7fab4bc", size = 44502313, upload-time = "2026-01-18T16:18:20.367Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/86/95c61ad82236495f3c31987e85135926ba3ec7f3819296b70a68d8066b49/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a244279f240c81f135631be91146d7fa0e9e840e1dfed2aba8483eba25cd98e6", size = 47585886, upload-time = "2026-01-18T16:18:27.544Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/6e/a72d901f305201802f016d015de1e05def7706fff68a1dedefef5dc7eff7/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c4692e83e42438dba512a570c6eaa42be2f8b6c0f492aea27dec54bdc495103a", size = 48207055, upload-time = "2026-01-18T16:18:35.425Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/e5/5de029c537630ca18828db45c30e2a78da03675a70ac6c3528203c416fe3/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae7f30f898dfe44ea69654a35c93e8da4cef6606dc4c72394068fd95f8e9f54a", size = 50619812, upload-time = "2026-01-18T16:18:43.553Z" },
-    { url = "https://files.pythonhosted.org/packages/59/8d/2af846cd2412e67a087f5bda4a8e23dfd4ebd570f777db2e8686615dafc1/pyarrow-23.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:5b86bb649e4112fb0614294b7d0a175c7513738876b89655605ebb87c804f861", size = 28263851, upload-time = "2026-01-18T16:19:38.567Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/7f/caab863e587041156f6786c52e64151b7386742c8c27140f637176e9230e/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ebc017d765d71d80a3f8584ca0566b53e40464586585ac64176115baa0ada7d3", size = 34463240, upload-time = "2026-01-18T16:18:49.755Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/fa/3a5b8c86c958e83622b40865e11af0857c48ec763c11d472c87cd518283d/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:0800cc58a6d17d159df823f87ad66cefebf105b982493d4bad03ee7fab84b993", size = 35935712, upload-time = "2026-01-18T16:18:55.626Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/08/17a62078fc1a53decb34a9aa79cf9009efc74d63d2422e5ade9fed2f99e3/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3a7c68c722da9bb5b0f8c10e3eae71d9825a4b429b40b32709df5d1fa55beb3d", size = 44503523, upload-time = "2026-01-18T16:19:03.958Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/70/84d45c74341e798aae0323d33b7c39194e23b1abc439ceaf60a68a7a969a/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:bd5556c24622df90551063ea41f559b714aa63ca953db884cfb958559087a14e", size = 47542490, upload-time = "2026-01-18T16:19:11.208Z" },
-    { url = "https://files.pythonhosted.org/packages/61/d9/d1274b0e6f19e235de17441e53224f4716574b2ca837022d55702f24d71d/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54810f6e6afc4ffee7c2e0051b61722fbea9a4961b46192dcfae8ea12fa09059", size = 48233605, upload-time = "2026-01-18T16:19:19.544Z" },
-    { url = "https://files.pythonhosted.org/packages/39/07/e4e2d568cb57543d84482f61e510732820cddb0f47c4bb7df629abfed852/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:14de7d48052cf4b0ed174533eafa3cfe0711b8076ad70bede32cf59f744f0d7c", size = 50603979, upload-time = "2026-01-18T16:19:26.717Z" },
-    { url = "https://files.pythonhosted.org/packages/72/9c/47693463894b610f8439b2e970b82ef81e9599c757bf2049365e40ff963c/pyarrow-23.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:427deac1f535830a744a4f04a6ac183a64fcac4341b3f618e693c41b7b98d2b0", size = 28338905, upload-time = "2026-01-18T16:19:32.93Z" },
+version = "23.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390, upload-time = "2026-02-16T10:08:08.654Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/8e/4be5617b4aaae0287f621ad31c6036e5f63118cfca0dc57d42121ff49b51/pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c", size = 35853761, upload-time = "2026-02-16T10:08:17.811Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/08/3e56a18819462210432ae37d10f5c8eed3828be1d6c751b6e6a2e93c286a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258", size = 44493116, upload-time = "2026-02-16T10:08:25.792Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/82/c40b68001dbec8a3faa4c08cd8c200798ac732d2854537c5449dc859f55a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2", size = 47564532, upload-time = "2026-02-16T10:08:34.27Z" },
+    { url = "https://files.pythonhosted.org/packages/20/bc/73f611989116b6f53347581b02177f9f620efdf3cd3f405d0e83cdf53a83/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5", size = 48183685, upload-time = "2026-02-16T10:08:42.889Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/cc/6c6b3ecdae2a8c3aced99956187e8302fc954cc2cca2a37cf2111dad16ce/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222", size = 50605582, upload-time = "2026-02-16T10:08:51.641Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/94/d359e708672878d7638a04a0448edf7c707f9e5606cee11e15aaa5c7535a/pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d", size = 27521148, upload-time = "2026-02-16T10:08:58.077Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" },
+    { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" },
+    { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" },
+    { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" },
+    { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" },
+    { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" },
+    { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" },
+    { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" },
+    { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" },
+    { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" },
+    { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" },
+    { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" },
+    { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" },
+    { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" },
+    { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" },
+    { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" },
 ]
 
 [[package]]
@@ -675,11 +685,14 @@ wheels = [
 
 [[package]]
 name = "pyjwt"
-version = "2.12.0"
+version = "2.13.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a8/10/e8192be5f38f3e8e7e046716de4cae33d56fd5ae08927a823bb916be36c1/pyjwt-2.12.0.tar.gz", hash = "sha256:2f62390b667cd8257de560b850bb5a883102a388829274147f1d724453f8fb02", size = 102511, upload-time = "2026-03-12T17:15:30.831Z" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/15/70/70f895f404d363d291dcf62c12c85fdd47619ad9674ac0f53364d035925a/pyjwt-2.12.0-py3-none-any.whl", hash = "sha256:9bb459d1bdd0387967d287f5656bf7ec2b9a26645d1961628cda1764e087fd6e", size = 29700, upload-time = "2026-03-12T17:15:29.257Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" },
 ]
 
 [package.optional-dependencies]