WIP: [DO NOT MERGE] introduce libcuml wheels

.github/workflows/build.yaml

@@ -66,15 +66,38 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-build-libcuml:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_libcuml.sh
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+  wheel-publish-libcuml:
+    needs: wheel-build-libcuml
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libcuml
+      package-type: cpp
   wheel-build-cuml:
+    needs: wheel-build-libcuml
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel.sh
+      script: ci/build_wheel_cuml.sh
       # Note that this approach to cloning repos obviates any modification to
       # the CMake variables in get_cumlprims_mg.cmake since CMake will just use
       # the clone as is.

.github/workflows/pr.yaml

@@ -25,6 +25,7 @@ jobs:
       - conda-notebook-tests
       - docs-build
       - telemetry-setup
+      - wheel-build-libcuml
       - wheel-build-cuml
       - wheel-tests-cuml
       - devcontainer
@@ -166,13 +167,28 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
-  wheel-build-cuml:
+  wheel-build-libcuml:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-      script: ci/build_wheel.sh
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_libcuml.sh
+      extra-repo: rapidsai/cumlprims_mg
+      extra-repo-sha: branch-25.02
+      extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+  wheel-build-cuml:
+    needs: [checks, wheel-build-libcuml]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_cuml.sh
       extra-repo: rapidsai/cumlprims_mg
       extra-repo-sha: branch-25.02
       extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY

build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2019-2024, NVIDIA CORPORATION.
+# Copyright (c) 2019-2025, NVIDIA CORPORATION.
 
 # cuml build script
 
@@ -288,11 +288,6 @@ if (! hasArg --configure-only) && (completeBuild || hasArg cuml || hasArg pydocs
     # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS
     SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g')
 
-    # Append `-DFIND_CUML_CPP=ON` to CUML_EXTRA_CMAKE_ARGS unless a user specified the option.
-    if [[ "${SKBUILD_EXTRA_CMAKE_ARGS}" != *"DFIND_CUML_CPP"* ]]; then
-        SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUML_CPP=ON"
-    fi
-
     SKBUILD_CMAKE_ARGS="-DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL};${SKBUILD_EXTRA_CMAKE_ARGS}" \
         python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuml
 

ci/build_wheel.sh

@@ -1,55 +1,26 @@
 #!/bin/bash
-# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-package_dir="python/cuml"
+package_name=$1
+package_dir=$2
 
 source rapids-configure-sccache
 source rapids-date-string
 
-RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-
 rapids-generate-version > ./VERSION
 
-cd ${package_dir}
-
-case "${RAPIDS_CUDA_VERSION}" in
-  12.*)
-    EXCLUDE_ARGS=(
-      --exclude "libcuvs.so"
-      --exclude "libcublas.so.12"
-      --exclude "libcublasLt.so.12"
-      --exclude "libcufft.so.11"
-      --exclude "libcurand.so.10"
-      --exclude "libcusolver.so.11"
-      --exclude "libcusparse.so.12"
-      --exclude "libnvJitLink.so.12"
-    )
-    EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON"
-    ;;
-  11.*)
-    EXCLUDE_ARGS=(
-      --exclude "libcuvs.so"
-    )
-    EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF"
-    ;;
-esac
+cd "${package_dir}"
 
 sccache --zero-stats
 
-SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/;-DUSE_CUVS_WHEEL=ON${EXTRA_CMAKE_ARGS}" \
-  python -m pip wheel . \
+rapids-logger "Building '${package_name}' wheel"
+python -m pip wheel \
     -w dist \
     -v \
     --no-deps \
-    --disable-pip-version-check
+    --disable-pip-version-check \
+    .
 
 sccache --show-adv-stats
-
-mkdir -p final_dist
-python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/*
-
-../../ci/validate_wheel.sh final_dist
-
-RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist

ci/build_wheel_cuml.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="cuml"
+package_dir="python/cuml"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# Download the libcuml wheel built in the previous step and make it
+# available for pip to find.
+RAPIDS_PY_WHEEL_NAME="libcuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcuml_dist
+
+rapids-logger "Generating build requirements"
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_build_${package_name}" \
+  --file-key "py_rapids_build_${package_name}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \
+| tee /tmp/requirements-build.txt
+
+rapids-logger "Installing build requirements"
+python -m pip install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt \
+    /tmp/libcuml_dist/libcuml_*.whl
+
+# build with '--no-build-isolation', for better sccache hit rate
+# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735)
+export PIP_NO_BUILD_ISOLATION=0
+
+EXCLUDE_ARGS=(
+  --exclude "libcuml++.so"
+  --exclude "libcumlprims_mg.so"
+  --exclude "libcuvs.so"
+)
+
+case "${RAPIDS_CUDA_VERSION}" in
+  12.*)
+    EXCLUDE_ARGS+=(
+      --exclude "libcublas.so.12"
+      --exclude "libcublasLt.so.12"
+      --exclude "libcufft.so.11"
+      --exclude "libcurand.so.10"
+      --exclude "libcusolver.so.11"
+      --exclude "libcusparse.so.12"
+      --exclude "libnvJitLink.so.12"
+    )
+    EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=ON"
+    ;;
+  11.*)
+    EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=OFF"
+    ;;
+esac
+
+export SKBUILD_CMAKE_ARGS="-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/;${EXTRA_CMAKE_ARGS};-DSINGLEGPU=OFF;-DUSE_LIBCUML_WHEEL=ON"
+./ci/build_wheel.sh "${package_name}" "${package_dir}"
+
+mkdir -p ${package_dir}/final_dist
+python -m auditwheel repair \
+    "${EXCLUDE_ARGS[@]}" \
+    -w ${package_dir}/final_dist \
+    ${package_dir}/dist/*
+
+./ci/validate_wheel.sh ${package_dir} final_dist
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${package_dir}/final_dist"

ci/build_wheel_libcuml.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Copyright (c) 2025, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="libcuml"
+package_dir="python/libcuml"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+rapids-logger "Generating build requirements"
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_build_${package_name}" \
+  --file-key "py_rapids_build_${package_name}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \
+| tee /tmp/requirements-build.txt
+
+rapids-logger "Installing build requirements"
+python -m pip install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt
+
+# build with '--no-build-isolation', for better sccache hit rate
+# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735)
+export PIP_NO_BUILD_ISOLATION=0
+
+# NOTE: 'libcumlprims_mg.so' is marked as '--exclude' here because auditwheel doesn't detect it,
+#       but it really is intentionally included in 'libcuml' wheels
+EXCLUDE_ARGS=(
+  --exclude "libcumlprims_mg.so"
+  --exclude "libcuvs.so"
+  --exclude "libraft.so"
+)
+
+# Avoid ever vendoring CUDA libraries into wheels.
+#
+# On CUDA 11 builds these excludes should technically be unnecessary because
+# there cuml and its dependencies statically link again these libraries, but
+# this is here unconditionally to ensure those wheels don't accidentally pick up
+# these libraries transitively.
+EXCLUDE_ARGS+=(
+  --exclude "libcublas.so.12"
+  --exclude "libcublasLt.so.12"
+  --exclude "libcufft.so.11"
+  --exclude "libcurand.so.10"
+  --exclude "libcusolver.so.11"
+  --exclude "libcusparse.so.12"
+  --exclude "libnvJitLink.so.12"
+)
+
+case "${RAPIDS_CUDA_VERSION}" in
+  12.*)
+    EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=ON"
+    ;;
+  11.*)
+    EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=OFF"
+    ;;
+esac
+
+export SKBUILD_CMAKE_ARGS="-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/;${EXTRA_CMAKE_ARGS}"
+./ci/build_wheel.sh "${package_name}" "${package_dir}"
+
+mkdir -p ${package_dir}/final_dist
+python -m auditwheel repair \
+    "${EXCLUDE_ARGS[@]}" \
+    -w ${package_dir}/final_dist \
+    ${package_dir}/dist/*
+
+./ci/validate_wheel.sh ${package_dir} final_dist
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp "${package_dir}/final_dist"

ci/test_wheel.sh

@@ -6,9 +6,11 @@ set -euo pipefail
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="libcuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
+  ./dist/libcuml*.whl \
   "$(echo ./dist/cuml*.whl)[test]"
 
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}

ci/validate_wheel.sh

@@ -1,14 +1,45 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-wheel_dir_relative_path=$1
+package_dir=$1
+wheel_dir_relative_path=$2
+
+RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
+
+# some packages are much larger on CUDA 11 than on CUDA 12
+PYDISTCHECK_ARGS=()
+if [[ "${package_dir}" == "python/libcuml" ]]; then
+    if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then
+        PYDISTCHECK_ARGS+=(
+            --max-allowed-size-compressed '1.0G'
+        )
+    else
+        PYDISTCHECK_ARGS+=(
+            --max-allowed-size-compressed '500M'
+        )
+    fi
+elif [[ "${package_dir}" == "python/cuml" ]]; then
+    # TODO(jameslamb): why are the CUDA 11 wheels so big???
+    if [[ "${RAPIDS_CUDA_MAJOR}" == "11" ]]; then
+        PYDISTCHECK_ARGS+=(
+            --max-allowed-size-compressed '1.5G'
+        )
+    else
+        PYDISTCHECK_ARGS+=(
+            --max-allowed-size-compressed '75M'
+        )
+    fi
+fi
+
+cd "${package_dir}"
 
 rapids-logger "validate packages with 'pydistcheck'"
 
 pydistcheck \
     --inspect \
+    "${PYDISTCHECK_ARGS[@]}" \
     "$(echo ${wheel_dir_relative_path}/*.whl)"
 
 rapids-logger "validate packages with 'twine'"

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -42,7 +42,7 @@ dependencies:
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
 - libcuvs==25.2.*,>=0.0.0a0
-- libraft-headers==25.2.*,>=0.0.0a0
+- libraft==25.2.*,>=0.0.0a0
 - librmm==25.2.*,>=0.0.0a0
 - nbsphinx
 - ninja

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -39,7 +39,7 @@ dependencies:
 - libcusolver-dev
 - libcusparse-dev
 - libcuvs==25.2.*,>=0.0.0a0
-- libraft-headers==25.2.*,>=0.0.0a0
+- libraft==25.2.*,>=0.0.0a0
 - librmm==25.2.*,>=0.0.0a0
 - nbsphinx
 - ninja