diff --git a/Cargo.toml b/Cargo.toml index b6164f89d31e..d26446c11167 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,13 +191,20 @@ strip = false # Retain debug info for flamegraphs inherits = "dev" incremental = false -# ci turns off debug info, etc for dependencies to allow for smaller binaries making caching more effective +# ci turns off debug info, etc. for dependencies to allow for smaller binaries making caching more effective [profile.ci.package."*"] debug = false debug-assertions = false strip = "debuginfo" incremental = false +# release inherited profile keeping debug information and symbols +# for mem/cpu profiling +[profile.profiling] +inherits = "release" +debug = true +strip = false + [workspace.lints.clippy] # Detects large stack-allocated futures that may cause stack overflow crashes (see threshold in clippy.toml) large_futures = "warn" diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh index 5be825eb0daf..5d3ad3446ddb 100755 --- a/benchmarks/bench.sh +++ b/benchmarks/bench.sh @@ -412,7 +412,10 @@ run_tpch() { echo "Running tpch benchmark..." # Optional query filter to run specific query QUERY=$([ -n "$ARG3" ] && echo "--query $ARG3" || echo "") + # debug the target command + set -x $CARGO_COMMAND --bin tpch -- benchmark datafusion --iterations 5 --path "${TPCH_DIR}" --prefer_hash_join "${PREFER_HASH_JOIN}" --format parquet -o "${RESULTS_FILE}" $QUERY + set +x } # Runs the tpch in memory @@ -427,9 +430,13 @@ run_tpch_mem() { RESULTS_FILE="${RESULTS_DIR}/tpch_mem_sf${SCALE_FACTOR}.json" echo "RESULTS_FILE: ${RESULTS_FILE}" echo "Running tpch_mem benchmark..." + # Optional query filter to run specific query QUERY=$([ -n "$ARG3" ] && echo "--query $ARG3" || echo "") + # debug the target command + set -x # -m means in memory $CARGO_COMMAND --bin tpch -- benchmark datafusion --iterations 5 --path "${TPCH_DIR}" --prefer_hash_join "${PREFER_HASH_JOIN}" -m --format parquet -o "${RESULTS_FILE}" $QUERY + set +x } # Runs the cancellation benchmark diff --git a/docs/source/library-user-guide/profiling.md b/docs/source/library-user-guide/profiling.md index 40fae6f44705..61e848a2b7d9 100644 --- a/docs/source/library-user-guide/profiling.md +++ b/docs/source/library-user-guide/profiling.md @@ -21,7 +21,7 @@ The section contains examples how to perform CPU profiling for Apache DataFusion on different operating systems. -## Building a flamegraph +## Building a flame graph [Video: how to CPU profile DataFusion with a Flamegraph](https://youtu.be/2z11xtYw_xs) @@ -82,6 +82,43 @@ CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --root --bench sql_planner -- [Video: how to CPU profile DataFusion with XCode Instruments](https://youtu.be/P3dXH61Kr5U) -## Linux +## Profiling using Samply cross platform profiler -## Windows +There is an opportunity to build flamegraphs, call trees and stack charts on any platform using +[Samply](https://github.com/mstange/samply) + +Install Samply profiler + +```shell +cargo install --locked samply +``` + +More Samply [installation options](https://github.com/mstange/samply?tab=readme-ov-file#installation) + +Run the profiler + +```shell +samply record --profile profiling ./my-application my-arguments +``` + +### Profile the benchmark + +[Set up benchmarks](https://github.com/apache/datafusion/blob/main/benchmarks/README.md#running-the-benchmarks) if not yet done + +Example: Profile Q22 query from TPC-H benchmark. +Note: `--profile profiling` to profile release optimized artifact with debug symbols + +```shell +cargo build --profile profiling --bin tpch +samply record ./target/profiling/tpch benchmark datafusion --iterations 5 --path datafusion/benchmarks/data/tpch_sf10 --prefer_hash_join true --format parquet -o datafusion/benchmarks/results/dev2/tpch_sf10.json --query 22 +``` + +After sampling has completed the Samply starts a local server and navigates to the profiler + +```shell +Local server listening at http://127.0.0.1:3000 +``` + +![img.png](samply_profiler.png) + +Note: The Firefox profiler cannot be opened in Safari, please use Chrome or Firefox instead diff --git a/docs/source/library-user-guide/samply_profiler.png b/docs/source/library-user-guide/samply_profiler.png new file mode 100644 index 000000000000..08b990745856 Binary files /dev/null and b/docs/source/library-user-guide/samply_profiler.png differ