feat: add stable install tests

.github/workflows/test.yaml

@@ -28,3 +28,50 @@ jobs:
       script: "ci/test_conda_nightly_env.sh"
       # just using the workflow to get the matrix, this isn't actually building anything we want to upload
       upload-artifacts: false
+  test-stable-install-pip-cuda-12:
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - amd64
+          - arm64
+    with:
+      build_type: pull-request
+      node_type: "cpu4"
+      arch: "${{matrix.arch}}"
+      container_image: "rapidsai/citestwheel:cuda12.9.1-ubuntu24.04-py3.14"
+      script: |
+        ./ci/stable_install/install_and_test_pip.sh --cuda cu12
+  test-stable-install-pip-cuda-13:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - amd64
+          - arm64
+    with:
+      build_type: pull-request
+      node_type: "cpu4"
+      arch: "${{matrix.arch}}"
+      container_image: "rapidsai/citestwheel:cuda13.2.0-ubuntu24.04-py3.14"
+      script: |
+        ./ci/stable_install/install_and_test_pip.sh --cuda cu13
+  test-stable-install-conda:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - amd64
+          - arm64
+    with:
+      build_type: pull-request
+      node_type: "cpu4"
+      arch: "${{matrix.arch}}"
+      container_image: "rapidsai/ci-conda:26.04-latest"
+      script: |
+        ./ci/stable_install/install_and_test_conda.sh

ci/release/update-version.sh

@@ -94,6 +94,7 @@ function sed_runner() {
 
 sed_runner "/RAPIDS_VER=/ s/[0-9][0-9].[0-9][0-9]/${NEXT_SHORT_TAG}/" ci/conda-pack.sh
 sed_runner "/RAPIDS_VERSION=/ s/[0-9][0-9].[0-9][0-9]/${NEXT_SHORT_TAG}/" ci/test_conda_nightly_env.sh
+sed_runner "/STABLE_RAPIDS_VERSION=/ s/[0-9][0-9].[0-9][0-9]/${NEXT_SHORT_TAG}/" ci/stable_install/install_and_test_*.sh
 
 # CI files - context-aware branch references
 for FILE in .github/workflows/*.yaml; do

ci/stable_install/bootstrap/pip.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright (c) 2026, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+echo "installing 'uv'"
+curl -LsSf https://astral.sh/uv/install.sh | sh
+echo "done installing 'uv'"
+
+source "$HOME"/.local/bin/env
+
+rapids-logger "Removing nightly PyPI index"
+pip config --global unset global.extra-index-url
+
+rapids-logger "Setting pip global retries to 10"
+pip config --global set global.retries 10

ci/stable_install/install_and_test_conda.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright (c) 2026, NVIDIA CORPORATION.
+
+# By default, this script attempts to install the latest stable RAPIDS version
+# in a clean conda environment for all supported versions of Python and CUDA
+#
+# The Python version, CUDA version, and RAPIDS version can be overridden by
+# supplying the value to test to the appropriate flag (--python, --rapids, and
+# --cuda, respectively).
+
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+source "${SCRIPT_DIR}/test_imports.sh"
+
+STABLE_RAPIDS_VERSION="26.4.*"
+SUPPORTED_PYTHON_VERSIONS=(3.11 3.12 3.13 3.14)
+SUPPORTED_CUDA_VERSIONS=("12.2" "12.9" "13.0" "13.1")
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --python)
+      shift
+      SUPPORTED_PYTHON_VERSIONS=("$1")
+      shift
+      ;;
+    --rapids)
+      shift
+      STABLE_RAPIDS_VERSION="$1"
+      shift
+      ;;
+    --cuda)
+      shift
+      SUPPORTED_CUDA_VERSIONS=("$1")
+      shift
+      ;;
+    -*)
+      rapids-echo-stderr "Unknown flag: $1. Supported flags: --python, --rapids, --cuda"
+      exit 1
+      ;;
+  esac
+done
+
+. /opt/conda/etc/profile.d/conda.sh
+
+for CUDA_VERSION in "${SUPPORTED_CUDA_VERSIONS[@]}"; do
+    for PY_VER in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
+        envName="rapids_${PY_VER}_${CUDA_VERSION}"
+
+        rapids-logger "Testing stable version install with Python $PY_VER and CUDA version $CUDA_VERSION"
+
+        # use `-O` to override channels so we don't include `rapidsai-nightly`
+        conda create -n "$envName" -O -c rapidsai -c conda-forge -y \
+          rapids="$STABLE_RAPIDS_VERSION" python="$PY_VER" "cuda-version==${CUDA_VERSION}"
+
+        conda activate "$envName"
+
+        testImports cudf dask_cudf cuml pylibraft raft_dask cugraph nx_cugraph cuxfilter cuvs # cucim
+
+        conda deactivate
+        conda env remove -n "$envName"
+    done
+done

ci/stable_install/install_and_test_pip.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright (c) 2026, NVIDIA CORPORATION.
+
+# By default, this script attempts to install the latest stable RAPIDS version
+# in a clean Python venv for all supported versions of Python and CUDA
+#
+# The Python version, CUDA version, and RAPIDS version can be overridden by
+# supplying the value to test to the appropriate flag (--python, --rapids, and
+# --cuda, respectively).
+#
+# Note that the CUDA version should be specified as a -cu* suffix, e.g. `--cuda cu13`
+# and will iterate over all supported minor CUDA versions, so `cu13` will test (13.0, 13.1)
+
+set -euo pipefail
+
+STABLE_RAPIDS_VERSION="26.4.*"
+SUPPORTED_PYTHON_VERSIONS=(3.11 3.12 3.13 3.14)
+SUPPORTED_CUDA_VERSIONS=("cu12" "cu13")
+CUDA12_MINOR_VERSIONS=('>=12.2,<12.4' '==12.9')
+CUDA13_MINOR_VERSIONS=('==13.0' '==13.1')
+
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+source "${SCRIPT_DIR}/bootstrap/pip.sh"
+source "${SCRIPT_DIR}/test_imports.sh"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --python)
+      shift
+      SUPPORTED_PYTHON_VERSIONS=("$1")
+      shift
+      ;;
+    --rapids)
+      shift
+      STABLE_RAPIDS_VERSION="$1"
+      shift
+      ;;
+    --cuda)
+      shift
+      SUPPORTED_CUDA_VERSIONS=("$1")
+      shift
+      ;;
+    -*)
+      rapids-echo-stderr "Unknown flag: $1. Supported flags: --python, --rapids, --cuda"
+      exit 1
+      ;;
+  esac
+done
+
+function createPyEnv {
+    PY_VER=$1
+    INSTALL_DIR=$2
+    pushd "$INSTALL_DIR"
+
+    rapids-logger "Creating virtualenv for Python $PY_VER"
+    uv venv --python="$PY_VER" --seed
+    source "${INSTALL_DIR}/.venv/bin/activate"
+}
+
+
+for CUDA_SUFFIX in "${SUPPORTED_CUDA_VERSIONS[@]}"; do
+
+    case "${CUDA_SUFFIX}" in
+        cu12) CUDA_MINOR_VERSIONS=("${CUDA12_MINOR_VERSIONS[@]}") ;;
+        cu13) CUDA_MINOR_VERSIONS=("${CUDA13_MINOR_VERSIONS[@]}") ;;
+    esac
+
+
+
+    for cuda_major_minor in "${CUDA_MINOR_VERSIONS[@]}"; do
+        rapids-logger "Using cuda-toolkit for CUDA ${cuda_major_minor}"
+        PIP_INSTALL_PYPI=(
+            "cudf-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}"
+            "dask-cudf-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}"
+            "cuml-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}"
+            "pylibraft-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}"
+            "raft-dask-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}"
+            "cuda-toolkit[cublas,cufft,curand,cusolver,cusparse,nvcc,nvrtc]${cuda_major_minor}"
+        )
+
+
+      for PY_VER in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
+
+          INSTALL_DIR=$(mktemp -d)
+          createPyEnv "$PY_VER" "$INSTALL_DIR"
+
+          rapids-logger "Downloading NVIDIA PyPI only wheels for Python $PY_VER and CUDA $cuda_major_minor"
+
+          WHEELS_DIR=$(mktemp -d)
+          pip download \
+            --isolated \
+            --index-url https://pypi.nvidia.com \
+            --prefer-binary \
+            --no-deps \
+            -d "${WHEELS_DIR}" \
+            "cucim-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "cugraph-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "cuvs-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "cuxfilter-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "libcugraph-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "libcuvs-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "nx-cugraph-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}" \
+            "pylibcugraph-${CUDA_SUFFIX}==${STABLE_RAPIDS_VERSION}"
+
+          rapids-logger "Testing stable version install with Python $PY_VER and CUDA $cuda_major_minor"
+
+          pip install \
+            --isolated \
+            --index-url https://pypi.org/simple \
+            --prefer-binary \
+            "${PIP_INSTALL_PYPI[@]}" \
+            "${WHEELS_DIR}"/*.whl
+
+          # can't import cucim on CPU-only
+          testImports cudf dask_cudf cuml pylibraft raft_dask cugraph nx_cugraph cuxfilter cuvs # cucim
+
+          popd
+
+          rapids-logger "Removing environment in $INSTALL_DIR/.venv"
+          rm -rf "$INSTALL_DIR/.venv"
+        done
+    done
+done

ci/stable_install/test_imports.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright (c) 2026, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Patterns in import output that we want to flag as a failure even when Python exits 0
+declare -a IMPORT_ERROR_PATTERNS=(
+    "dlopen error"
+    "cannot open shared object file"
+    "missing cuda symbols"
+    "initialization failed"
+)
+IMPORT_ERROR_PATTERN=$(printf "%s|" "${IMPORT_ERROR_PATTERNS[@]}")
+# Strip off the trailing `|` from joining the patterns together
+IMPORT_ERROR_PATTERN="${IMPORT_ERROR_PATTERN%|}"
+
+function runImport {
+    local cmd="$1"
+    local label="$2"
+    local output exit_code=0
+    output=$(python -c "$cmd" 2>&1) || exit_code=$?
+    [[ -n "$output" ]] && echo "$output"
+    if [[ $exit_code -ne 0 ]]; then
+        rapids-logger "Test failed for: $label"
+        return 1
+    # here we grep over the combined output to look for errors that we want to
+    # flag even if Python would exit cleanly
+    # -E is for extended regex support for the pattern1|pattern2 matching
+    elif echo "$output" | grep -qi -E "${IMPORT_ERROR_PATTERN}"; then
+        rapids-logger "Test failed for: $label (error pattern detected in output)"
+        return 1
+    else
+        rapids-logger "Passed"
+        return 0
+    fi
+}
+
+function testImports {
+    local -a imports=()
+    local failures=0
+    while [[ $# -gt 0 ]]; do
+        rapids-logger "Standalone import test for $1"
+        runImport "import $1" "$1" || failures=$((failures + 1))
+        # add import to array for combined import test before shifting
+        imports+=("$1")
+        shift
+    done
+    local import_cmd
+    import_cmd=$(printf "import %s; " "${imports[@]}")
+    rapids-logger "Combined import test for: ${imports[*]}"
+    runImport "${import_cmd}" "${imports[*]}" || failures=$((failures + 1))
+    return $failures
+}