diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md index 530cdf1b..468cbfb6 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md @@ -146,7 +146,10 @@ mvn clean package -Pudf-native-examples The build will automatically: - Extract `libcudf.so` from the rapids-4-spark jar -- Clone cuDF repository for headers (shallow clone) +- Read the embedded `spark-rapids-jni` and `cudf-java` version metadata from the jar +- Download the matching `spark-rapids-jni` `cudf-pins` files +- Clone the cuDF repository at the revision recorded in the jar +- Configure rapids-cmake with the matching `rapids-cmake` sha and package override file - Build only your UDF native code against the prebuilt library **Native ABI compatibility note:** @@ -160,12 +163,11 @@ the UDF may fail with undefined symbols or crash when Spark loads the native library. When changing the rapids-4-spark jar version, rebuild the native UDFs with -matching cuDF/RMM/CCCL headers and libraries. For snapshot or locally built -jars, make sure `cudf.git.branch`, the rapids-cmake branch in -`src/main/cpp/CMakeLists.txt`, and the RMM/CCCL versions resolved by -rapids-cmake correspond to the same source state used to build the jar. After -changing these values, remove `target/native-deps` and `target/cudf-repo` -before rebuilding so stale headers or libraries are not reused. +matching cuDF/RMM/CCCL headers and libraries. In prebuilt mode, the Maven build +derives those native dependency pins from the `spark-rapids-jni` revision +recorded in the jar. After changing jar versions, remove `target/native-deps`, +`target/cudf-repo`, `target/cudf-pins`, and `target/cpp-build` before +rebuilding so stale headers, pins, or libraries are not reused. **Or manually extract first:** ```bash @@ -217,7 +219,7 @@ You can customize the build by passing Maven system properties via `-D | `BUILD_UDF_BENCHMARKS` | `OFF` | Build benchmark executables | | `PER_THREAD_DEFAULT_STREAM` | `ON` | Enable per-thread default CUDA streams | | `CUDF_ENABLE_ARROW_S3` | `OFF` | Enable Arrow S3 support in cuDF | -| `cudf.git.branch` | `main` | cuDF git branch to clone for headers | +| `cudf.git.branch` | `main` | cuDF git branch to clone for source builds or fallback paths | | `skipCudfExtraction` | `false` | Skip extracting cuDF dependencies from jar | **Example usage:** diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/clone-cudf-repo.sh b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/clone-cudf-repo.sh index 1f80b82f..97987a1f 100755 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/clone-cudf-repo.sh +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/clone-cudf-repo.sh @@ -22,11 +22,11 @@ # headers needed for compiling native UDF code. # # Usage: -# clone-cudf-repo.sh +# clone-cudf-repo.sh # # Arguments: # target_directory - Directory where cuDF repo will be cloned -# branch_name - Git branch to clone/checkout +# git_ref - Git branch, tag, or commit to checkout # # Exit codes: # 0 - Success @@ -38,79 +38,88 @@ set -o pipefail # Parse arguments if [ $# -ne 2 ]; then - echo "ERROR: Usage: $0 " >&2 + echo "ERROR: Usage: $0 " >&2 exit 1 fi CUDF_DIR="$1" -BRANCH="$2" +GIT_REF="$2" +GIT=(git -c "safe.directory=$CUDF_DIR") echo "================================================" echo "cuDF Repository Management" echo " Target directory: $CUDF_DIR" -echo " Branch: $BRANCH" +echo " Git ref: $GIT_REF" echo "================================================" # Check if repository already exists if [ ! -d "$CUDF_DIR/.git" ]; then # Repository doesn't exist - clone it - echo "Cloning cuDF repository ($BRANCH branch)..." + echo "Cloning cuDF repository..." - git clone --depth 1 --branch "$BRANCH" \ - https://github.com/rapidsai/cudf.git "$CUDF_DIR" || { - echo "ERROR: Failed to clone cuDF from branch $BRANCH" >&2 + "${GIT[@]}" clone --filter=blob:none --no-checkout https://github.com/rapidsai/cudf.git "$CUDF_DIR" || { + echo "ERROR: Failed to clone cuDF repository" >&2 echo "Please check:" >&2 echo " 1. Network connectivity to GitHub" >&2 - echo " 2. Branch '$BRANCH' exists in cuDF repository" >&2 exit 1 } - + + cd "$CUDF_DIR" || { + echo "ERROR: Cannot access directory $CUDF_DIR" >&2 + exit 1 + } + + "${GIT[@]}" fetch --depth 1 origin "$GIT_REF" || { + echo "ERROR: Failed to fetch cuDF ref $GIT_REF from origin" >&2 + echo "Please check:" >&2 + echo " 1. Network connectivity to GitHub" >&2 + echo " 2. Ref '$GIT_REF' exists in cuDF repository" >&2 + exit 1 + } + + "${GIT[@]}" checkout --detach FETCH_HEAD || { + echo "ERROR: Failed to checkout cuDF ref $GIT_REF" >&2 + exit 1 + } + echo "✓ Successfully cloned cuDF repository" else # Repository exists - verify and update if needed - echo "cuDF repository exists, verifying branch..." + echo "cuDF repository exists, verifying ref..." cd "$CUDF_DIR" || { echo "ERROR: Cannot access directory $CUDF_DIR" >&2 exit 1 } - - # Get current branch - CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") - - if [ "$CURRENT_BRANCH" != "$BRANCH" ]; then - # Branch mismatch - fetch and switch to correct branch - echo "Branch mismatch detected:" - echo " Current branch: $CURRENT_BRANCH" - echo " Expected branch: $BRANCH" - echo "Fetching and switching to $BRANCH..." - - git fetch --depth 1 origin "$BRANCH" || { - echo "ERROR: Failed to fetch branch $BRANCH from origin" >&2 - echo "Please check:" >&2 - echo " 1. Network connectivity to GitHub" >&2 - echo " 2. Branch '$BRANCH' exists in cuDF repository" >&2 - exit 1 - } - - git checkout "$BRANCH" || { - echo "ERROR: Failed to checkout branch $BRANCH" >&2 - exit 1 - } - - git reset --hard "origin/$BRANCH" || { - echo "ERROR: Failed to reset to origin/$BRANCH" >&2 + + CURRENT_COMMIT=$("${GIT[@]}" rev-parse HEAD 2>/dev/null || echo "unknown") + + "${GIT[@]}" fetch --depth 1 origin "$GIT_REF" || { + echo "ERROR: Failed to fetch cuDF ref $GIT_REF from origin" >&2 + echo "Please check:" >&2 + echo " 1. Network connectivity to GitHub" >&2 + echo " 2. Ref '$GIT_REF' exists in cuDF repository" >&2 + exit 1 + } + + FETCHED_COMMIT=$("${GIT[@]}" rev-parse FETCH_HEAD) + + if [ "$CURRENT_COMMIT" != "$FETCHED_COMMIT" ]; then + echo "cuDF ref mismatch detected:" + echo " Current commit: $CURRENT_COMMIT" + echo " Expected commit: $FETCHED_COMMIT ($GIT_REF)" + "${GIT[@]}" checkout --detach FETCH_HEAD || { + echo "ERROR: Failed to checkout cuDF ref $GIT_REF" >&2 exit 1 } - - echo "✓ Switched to branch $BRANCH" + echo "✓ Switched cuDF repository to $GIT_REF" else - echo "✓ Already on correct branch ($BRANCH)" + echo "✓ Already on correct cuDF ref ($GIT_REF)" fi fi echo "================================================" echo "✓ cuDF repository ready at: $CUDF_DIR" -echo " Branch: $BRANCH" +echo " Git ref: $GIT_REF" echo "================================================" exit 0 diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-cudf-libs.sh b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-cudf-libs.sh index a00081b3..f5c28780 100755 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-cudf-libs.sh +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-cudf-libs.sh @@ -28,6 +28,7 @@ # ./extract-cudf-libs.sh # # Environment Variables (optional, will use pom.xml values if not set): +# RAPIDS_JAR_PATH - explicit rapids-4-spark jar path # RAPIDS4SPARK_VERSION - rapids-4-spark version (e.g., 26.02.0 or 26.06.0-SNAPSHOT) # SCALA_VERSION - Scala binary version (e.g., 2.12, 2.13) # CUDA_VERSION - CUDA version (e.g., cuda11, cuda12) @@ -40,7 +41,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TARGET_DIR="$SCRIPT_DIR/target" +TARGET_DIR="${TARGET_DIR:-$SCRIPT_DIR/target}" NATIVE_DEPS_DIR="$TARGET_DIR/native-deps" CUDF_REPO_DIR="$TARGET_DIR/cudf-repo" POM_FILE="$SCRIPT_DIR/pom.xml" @@ -50,6 +51,11 @@ POM_FILE="$SCRIPT_DIR/pom.xml" extract_pom_property() { local property_name="$1" local value + + if [ ! -f "$POM_FILE" ]; then + echo "" + return + fi # Use xmllint if available (more reliable) if command -v xmllint >/dev/null 2>&1; then @@ -109,10 +115,16 @@ JAR_PATH_WITH_CLASSIFIER="$MAVEN_REPO/com/nvidia/rapids-4-spark_${SCALA_VERSION} JAR_PATH_NO_CLASSIFIER="$MAVEN_REPO/com/nvidia/rapids-4-spark_${SCALA_VERSION}/${RAPIDS4SPARK_VERSION}/rapids-4-spark_${SCALA_VERSION}-${RAPIDS4SPARK_VERSION}.jar" echo "Looking for rapids-4-spark jar..." +if [ -n "${RAPIDS_JAR_PATH:-}" ]; then + echo " Explicit path: $RAPIDS_JAR_PATH" +fi echo " Pattern 1 (with classifier): $JAR_PATH_WITH_CLASSIFIER" echo " Pattern 2 (no classifier): $JAR_PATH_NO_CLASSIFIER" -if [ -f "$JAR_PATH_WITH_CLASSIFIER" ]; then +if [ -n "${RAPIDS_JAR_PATH:-}" ] && [ -f "$RAPIDS_JAR_PATH" ]; then + JAR_PATH="$RAPIDS_JAR_PATH" + echo "✓ Found jar (explicit path): $JAR_PATH" +elif [ -f "$JAR_PATH_WITH_CLASSIFIER" ]; then JAR_PATH="$JAR_PATH_WITH_CLASSIFIER" echo "✓ Found jar (with classifier): $JAR_PATH" elif [ -f "$JAR_PATH_NO_CLASSIFIER" ]; then @@ -122,6 +134,9 @@ else echo "" echo "ERROR: rapids-4-spark jar not found!" echo "Tried:" + if [ -n "${RAPIDS_JAR_PATH:-}" ]; then + echo " $RAPIDS_JAR_PATH" + fi echo " $JAR_PATH_WITH_CLASSIFIER" echo " $JAR_PATH_NO_CLASSIFIER" echo "" @@ -232,15 +247,22 @@ fi echo "✓ Successfully extracted libraries to: $NATIVE_DEPS_DIR" ls -lh "$NATIVE_DEPS_DIR" +PINS_DIR="$TARGET_DIR/cudf-pins" +PINS_PROPERTIES="$TARGET_DIR/jar-native-deps.properties" +"$SCRIPT_DIR/resolve-jni-cudf-pins.sh" "$JAR_PATH" "$PINS_DIR" "$PINS_PROPERTIES" +RESOLVED_CUDF_REF=$(awk -F= '$1 == "jar.cudf.revision" {print $2; exit}' "$PINS_PROPERTIES") +if [ -z "$RESOLVED_CUDF_REF" ]; then + echo "ERROR: Failed to resolve cuDF revision from $PINS_PROPERTIES" >&2 + exit 1 +fi + # Clone cuDF repo for headers (shallow clone) if [ ! -d "$CUDF_REPO_DIR/.git" ]; then echo "Cloning cuDF repository for headers..." - git clone --depth 1 --branch "$CUDF_BRANCH" https://github.com/rapidsai/cudf.git "$CUDF_REPO_DIR" - echo "✓ Cloned cuDF repo to: $CUDF_REPO_DIR" else echo "✓ cuDF repo already exists at: $CUDF_REPO_DIR" - echo " (Delete it to re-clone: rm -rf \"$CUDF_REPO_DIR\")" fi +"$SCRIPT_DIR/clone-cudf-repo.sh" "$CUDF_REPO_DIR" "$RESOLVED_CUDF_REF" echo "" echo "==================================================" diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml index 7ac5a7c3..ddc52b9d 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml @@ -413,6 +413,16 @@ message="Failed to extract libcudf.so from jar"/> + + + + + + + + + @@ -421,7 +431,7 @@ - + @@ -451,6 +461,7 @@ compile + + + + diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/resolve-jni-cudf-pins.sh b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/resolve-jni-cudf-pins.sh new file mode 100755 index 00000000..7f7f7e97 --- /dev/null +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/resolve-jni-cudf-pins.sh @@ -0,0 +1,177 @@ +#!/bin/bash +# +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +JAR_PATH="$1" +PINS_DIR="$2" +PROPERTIES_FILE="$3" + +if [ ! -f "$JAR_PATH" ]; then + echo "ERROR: rapids-4-spark jar not found: $JAR_PATH" >&2 + exit 1 +fi + +mkdir -p "$PINS_DIR" +mkdir -p "$(dirname "$PROPERTIES_FILE")" + +read_property_from_jar() { + local entry="$1" + local property="$2" + if command -v python3 >/dev/null 2>&1; then + python3 - "$JAR_PATH" "$entry" "$property" <<'PY' +import sys +import zipfile + +jar_path, entry, prop = sys.argv[1:] +try: + with zipfile.ZipFile(jar_path) as jar: + data = jar.read(entry).decode("utf-8", errors="replace") +except Exception: + sys.exit(0) + +for line in data.splitlines(): + key, sep, value = line.partition("=") + if sep and key == prop: + print(value) + break +PY + elif command -v unzip >/dev/null 2>&1; then + unzip -p "$JAR_PATH" "$entry" 2>/dev/null | awk -F= -v key="$property" '$1 == key {print $2; exit}' + else + echo "ERROR: python3 or unzip is required to read $entry from $JAR_PATH" >&2 + exit 1 + fi +} + +normalize_github_raw_base() { + local repo_url="$1" + repo_url="${repo_url%.git}" + case "$repo_url" in + https://github.com/*) + echo "${repo_url/github.com/raw.githubusercontent.com}" + ;; + *) + echo "" + ;; + esac +} + +download_file() { + local url="$1" + local output="$2" + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$url" -o "$output" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$output" + else + echo "ERROR: curl or wget is required to download $url" >&2 + exit 1 + fi +} + +JNI_INFO="spark-rapids-jni-version-info.properties" +CUDF_INFO="cudf-java-version-info.properties" + +JNI_REVISION="$(read_property_from_jar "$JNI_INFO" revision || true)" +JNI_URL="$(read_property_from_jar "$JNI_INFO" url || true)" +CUDF_REVISION="$(read_property_from_jar "$CUDF_INFO" revision || true)" + +if [ -z "$JNI_REVISION" ] || [ -z "$JNI_URL" ]; then + echo "ERROR: Failed to read spark-rapids-jni revision/url from $JAR_PATH" >&2 + echo "Expected $JNI_INFO in the jar." >&2 + exit 1 +fi + +if [ -z "$CUDF_REVISION" ]; then + echo "ERROR: Failed to read cuDF revision from $JAR_PATH" >&2 + echo "Expected $CUDF_INFO in the jar." >&2 + exit 1 +fi + +RAW_BASE="$(normalize_github_raw_base "$JNI_URL")" +if [ -z "$RAW_BASE" ]; then + echo "ERROR: Unsupported spark-rapids-jni URL for automatic pin lookup: $JNI_URL" >&2 + echo "Only github.com URLs can be resolved automatically." >&2 + exit 1 +fi + +VERSIONS_FILE="$PINS_DIR/versions.json" +RAPIDS_CMAKE_SHA_FILE="$PINS_DIR/rapids-cmake.sha" +RAPIDS_CMAKE_FILE="$PINS_DIR/RAPIDS.cmake" + +download_file "$RAW_BASE/$JNI_REVISION/thirdparty/cudf-pins/versions.json" "$VERSIONS_FILE" +download_file "$RAW_BASE/$JNI_REVISION/thirdparty/cudf-pins/rapids-cmake.sha" "$RAPIDS_CMAKE_SHA_FILE" + +python3 - "$VERSIONS_FILE" <<'PY' +import json +import sys + +versions_file = sys.argv[1] +with open(versions_file, encoding="utf-8") as fh: + data = json.load(fh) + +packages = data.get("packages") +if not isinstance(packages, dict) or not packages: + raise SystemExit(f"ERROR: {versions_file} does not contain a non-empty packages map") + +missing_metadata = [] +for name, package in sorted(packages.items()): + if "version" not in package: + missing_metadata.append(f"{name}: missing version") + has_git_source = "git_url" in package and "git_tag" in package + has_url_source = "url" in package and "url_hash" in package + if not (has_git_source or has_url_source): + missing_metadata.append(f"{name}: missing pinned git/url source") + +if missing_metadata: + raise SystemExit("ERROR: invalid cudf-pins metadata:\n " + "\n ".join(missing_metadata)) + +required = ["CCCL", "rmm"] +missing_required = [name for name in required if name not in packages] +if missing_required: + raise SystemExit("ERROR: cudf-pins missing required packages: " + ", ".join(missing_required)) +PY + +RAPIDS_CMAKE_SHA="$(tr -d '[:space:]' < "$RAPIDS_CMAKE_SHA_FILE")" +if [ -z "$RAPIDS_CMAKE_SHA" ]; then + echo "ERROR: rapids-cmake sha file is empty: $RAPIDS_CMAKE_SHA_FILE" >&2 + exit 1 +fi + +download_file "https://raw.githubusercontent.com/rapidsai/rapids-cmake/$RAPIDS_CMAKE_SHA/RAPIDS.cmake" \ + "$RAPIDS_CMAKE_FILE" + +cat > "$PROPERTIES_FILE" <