From 1d4c30afdead8002da3a160d4fc3e25467831417 Mon Sep 17 00:00:00 2001 From: devoopsman45 Date: Thu, 18 Jun 2026 23:12:57 -0400 Subject: [PATCH 1/2] feat: add docker-compose.quick.yml and fix onboarding docs Add docker-compose.quick.yml that pulls pre-built GHCR images so users can start a 1-scheduler + 2-executor cluster with Docker as the only prerequisite (~2 min vs ~20 min cold Rust build). Key details: - Uses --advertise-flight-sql-endpoint so clients only connect to scheduler:50050; no direct executor port access needed from the host - Health checks on both services with depends_on for correct startup order - 2 executor replicas by default, easily scaled with --scale Also fix two doc gaps that caused silent failures: - quick-start.md: restructured with two labelled paths (Docker eval vs build from source), expected log output, troubleshooting section, and compatibility gap warning - docker-compose.md: document the missing cargo build --release prerequisite for docker-compose.yml, and note that no CLI image is published to GHCR Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01TKhWssogfJHUnDTz4qfpiA --- docker-compose.quick.yml | 82 +++++++ .../user-guide/deployment/docker-compose.md | 50 +++-- .../user-guide/deployment/quick-start.md | 207 +++++++++++------- 3 files changed, 237 insertions(+), 102 deletions(-) create mode 100644 docker-compose.quick.yml diff --git a/docker-compose.quick.yml b/docker-compose.quick.yml new file mode 100644 index 0000000000..0ecc695c77 --- /dev/null +++ b/docker-compose.quick.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Quick-start cluster using pre-built images from GHCR. +# No local build required — Docker is the only prerequisite. +# +# Runs the last stable release. To test against unreleased changes, +# use docker-compose.yml (requires `cargo build --release` first). +# +# Usage: +# docker compose -f docker-compose.quick.yml up +# +# Connect from Rust: +# SessionContext::remote("df://localhost:50050").await? +# +# Connect from the CLI: +# cargo run -p ballista-cli -- --host localhost --port 50050 +# +# To make local data available inside executors, uncomment and +# set the volume path under ballista-executor: +# volumes: +# - /absolute/path/to/your/data:/data:ro + +services: + ballista-scheduler: + image: ghcr.io/apache/datafusion-ballista-scheduler:latest + # --advertise-flight-sql-endpoint enables the scheduler to proxy all + # result fetching so clients only ever connect to port 50050. + # Without this flag, clients would need direct access to each + # executor's Arrow Flight port, which breaks in Docker networking. + command: > + --bind-host 0.0.0.0 + --external-host ballista-scheduler + --advertise-flight-sql-endpoint + ports: + - "50050:50050" + environment: + - RUST_LOG=ballista=info,ballista_scheduler=info + healthcheck: + test: ["CMD", "bash", "-c", " + --bind-host 0.0.0.0 + --scheduler-host ballista-scheduler + --concurrent-tasks 4 + --work-dir /work + environment: + - RUST_LOG=ballista=info,ballista_executor=info + # Uncomment to mount local data for queries: + # volumes: + # - /absolute/path/to/your/data:/data:ro + depends_on: + ballista-scheduler: + condition: service_healthy + healthcheck: + test: ["CMD", "bash", "-c", " [!IMPORTANT] +> Ballista and DataFusion are developed independently. A given Ballista release may not be compatible +> with the latest DataFusion version. Check the [compatibility matrix](../configs.md) before integrating. + +--- + +## Path A: Evaluate with Docker (~2 min) + +The only prerequisite is [Docker](https://docs.docker.com/get-docker/) with Compose v2. + +This uses pre-built images from GHCR that are published on each stable release. The `latest` tag +tracks the most recent release, not the `main` branch. + +```shell +docker compose -f docker-compose.quick.yml up +``` + +You should see output similar to: + +``` +ballista-scheduler-1 | Ballista Scheduler v53.0.0 listening on 0.0.0.0:50050 +ballista-executor-1 | Executor registration succeed +ballista-executor-2 | Executor registration succeed +``` + +Two executors start by default. The scheduler listens on `localhost:50050`. + +**Connect from Rust:** + +```rust +let ctx = SessionContext::remote("df://localhost:50050").await?; +``` + +**Connect from the CLI** (requires a local build — no pre-built CLI image is published): + +```shell +cargo run -p ballista-cli -- --host localhost --port 50050 +``` + +**To make local data available inside the executors**, uncomment and set the `volumes` block +in `docker-compose.quick.yml`: + +```yaml +ballista-executor: + volumes: + - /absolute/path/to/your/data:/data:ro +``` + +Then reference `/data/yourfile.parquet` in your queries. The path must be the same inside +every executor container. + +**Tear down:** + +```shell +docker compose -f docker-compose.quick.yml down +``` + +--- + +## Path B: Build from source (~20 min) + +Use this path if you need to test local code changes or run against the `main` branch. + +**Prerequisites:** - [Rust](https://www.rust-lang.org/tools/install) - [Protobuf Compiler](https://protobuf.dev/downloads/) -## Build the project - -From the root of the project, build release binaries. +**Step 1:** Build release binaries from the repository root: ```shell cargo build --release ``` -Start a Ballista scheduler process in a new terminal session. +**Step 2:** Start the scheduler in a new terminal: ```shell RUST_LOG=info ./target/release/ballista-scheduler ``` -Start one or more Ballista executor processes in new terminal sessions. When starting more than one -executor, a unique port number must be specified for each executor. +**Step 3:** Start one or more executors, each in a new terminal. When running multiple +executors, each needs a unique pair of ports: ```shell RUST_LOG=info ./target/release/ballista-executor -c 2 -p 50051 --bind-grpc-port 50052 +``` +```shell RUST_LOG=info ./target/release/ballista-executor -c 2 -p 50053 --bind-grpc-port 50054 ``` +> **Two-port model:** each executor exposes an Arrow Flight port (data, `-p`) and a gRPC +> control port (`--bind-grpc-port`). Both must be reachable by the scheduler. + +--- + ## Running the examples -The examples can be run using the `cargo run --bin` syntax. Open a new terminal session and run the following commands. +Examples live in the `examples/` directory and connect to `localhost:50050` by default. -### Distributed SQL Example +### Distributed SQL example ```bash cd examples cargo run --release --example remote-sql ``` -#### Source code for distributed SQL example +### Distributed DataFrame example -```rust -use ballista::prelude::*; -use ballista_examples::test_util; -use datafusion::{ - execution::SessionStateBuilder, - prelude::{CsvReadOptions, SessionConfig, SessionContext}, -}; - -/// This example demonstrates executing a simple query against an Arrow data source (CSV) and -/// fetching results, using SQL -#[tokio::main] -async fn main() -> Result<()> { - let config = SessionConfig::new_with_ballista() - .with_target_partitions(4) - .with_ballista_job_name("Remote SQL Example"); - - let state = SessionStateBuilder::new() - .with_config(config) - .with_default_features() - .build(); - - let ctx = SessionContext::remote_with_state("df://localhost:50050", state).await?; - - let test_data = test_util::examples_test_data(); - - ctx.register_csv( - "test", - &format!("{test_data}/aggregate_test_100.csv"), - CsvReadOptions::new(), - ) - .await?; - - let df = ctx - .sql( - "SELECT c1, MIN(c12), MAX(c12) \ - FROM test \ - WHERE c11 > 0.1 AND c11 < 0.9 \ - GROUP BY c1", - ) - .await?; - - df.show().await?; - - Ok(()) -} +```bash +cd examples +cargo run --release --example remote-dataframe ``` -### Distributed DataFrame Example +### Standalone (single-process) example + +No cluster needed — scheduler and executor run in the same process: ```bash cd examples -cargo run --release --example remote-dataframe +cargo run --release --example standalone-sql ``` -#### Source code for distributed DataFrame example +--- -```rust -use ballista::prelude::*; -use ballista_examples::test_util; -use datafusion::{ - prelude::{col, lit, ParquetReadOptions, SessionContext}, -}; - -#[tokio::main] -async fn main() -> Result<()> { - // creating SessionContext with default settings - let ctx = SessionContext::remote("df://localhost:50050").await?; - - let test_data = test_util::examples_test_data(); - let filename = format!("{test_data}/alltypes_plain.parquet"); - - let df = ctx - .read_parquet(filename, ParquetReadOptions::default()) - .await? - .select_columns(&["id", "bool_col", "timestamp_col"])? - .filter(col("id").gt(lit(1)))?; - - df.show().await?; - - Ok(()) -} +## Troubleshooting + +**`protoc` not found during build** + +Install the protobuf compiler for your OS, then retry `cargo build --release`: + +```shell +# Ubuntu / Debian +sudo apt install protobuf-compiler + +# macOS +brew install protobuf ``` + +**Executor can't reach the scheduler** + +The executor connects to the scheduler at startup. Make sure the scheduler is running before +starting executors. Check that the scheduler address (`--scheduler-host`, default `localhost`) +is reachable from the executor's network. + +**Port conflict** + +If port 50050 is already in use, start the scheduler with `--bind-port ` and update +your client connection string accordingly. + +**Docker executor containers exit immediately** + +Check `docker compose logs`. The most common cause is the scheduler health check failing +because the scheduler itself hasn't started yet — `depends_on: condition: service_healthy` +handles this in `docker-compose.quick.yml`. From 0092c8cc864536ae00eb2e3a938a7b1d408d4e82 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 19 Jun 2026 10:18:20 +0300 Subject: [PATCH 2/2] Add AI config files --- .cursor/rules.md | 5 +++++ .gemini/rules.md | 5 +++++ AGENTS.md | 5 +++++ CLAUDE.md | 5 +++++ 4 files changed, 20 insertions(+) create mode 100644 .cursor/rules.md create mode 100644 .gemini/rules.md create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/.cursor/rules.md b/.cursor/rules.md new file mode 100644 index 0000000000..4ee2be4ad3 --- /dev/null +++ b/.cursor/rules.md @@ -0,0 +1,5 @@ +When making a code review never link to any issue or a pull request +in any GitHub repository! + +Do not review AI agents' config files like CLAUDE.md, AGENTS.md or rules.md! + diff --git a/.gemini/rules.md b/.gemini/rules.md new file mode 100644 index 0000000000..4ee2be4ad3 --- /dev/null +++ b/.gemini/rules.md @@ -0,0 +1,5 @@ +When making a code review never link to any issue or a pull request +in any GitHub repository! + +Do not review AI agents' config files like CLAUDE.md, AGENTS.md or rules.md! + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..4ee2be4ad3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +When making a code review never link to any issue or a pull request +in any GitHub repository! + +Do not review AI agents' config files like CLAUDE.md, AGENTS.md or rules.md! + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..4ee2be4ad3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +When making a code review never link to any issue or a pull request +in any GitHub repository! + +Do not review AI agents' config files like CLAUDE.md, AGENTS.md or rules.md! +