Support non-CTK Nvidia libraries

.github/workflows/test-wheel-linux.yml

@@ -321,16 +321,15 @@ jobs:
             pip install $(ls cuda_python*.whl)[all]
           fi
 
-      - name: Install cuda.pathfinder nvidia_wheels_cu13
-        if: startsWith(matrix.CUDA_VER, '13.')
+      - name: Install cuda.pathfinder extra wheels for testing
         run: |
+          set -euo pipefail
           pushd cuda_pathfinder
-          pip install -v .[nvidia_wheels_cu13]
-          pip freeze
+          pip install --only-binary=:all: -v ".[nvidia_wheels_cu${TEST_CUDA_MAJOR},nvidia_wheels_host]"
+          pip list
           popd
 
       - name: Run cuda.pathfinder tests with all_must_work
-        if: startsWith(matrix.CUDA_VER, '13.')
         env:
           CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work
         run: run-tests pathfinder

.github/workflows/test-wheel-windows.yml

@@ -288,17 +288,15 @@ jobs:
             pip install "$((Get-ChildItem -Filter cuda_python*.whl).FullName)[all]"
           }
 
-      - name: Install cuda.pathfinder nvidia_wheels_cu13
-        if: startsWith(matrix.CUDA_VER, '13.')
+      - name: Install cuda.pathfinder extra wheels for testing
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
           pushd cuda_pathfinder
-          pip install -v .[nvidia_wheels_cu13]
-          pip freeze
+          pip install --only-binary=:all: -v ".[nvidia_wheels_cu${TEST_CUDA_MAJOR},nvidia_wheels_host]"
+          pip list
           popd
 
       - name: Run cuda.pathfinder tests with all_must_work
-        if: startsWith(matrix.CUDA_VER, '13.')
         env:
           CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work
         shell: bash --noprofile --norc -xeuo pipefail {0}

ci/tools/env-vars

@@ -69,6 +69,7 @@ elif [[ "${1}" == "test" ]]; then
   echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV
   echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
   echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV
+  echo "TEST_CUDA_MAJOR=${TEST_CUDA_MAJOR}" >> $GITHUB_ENV
 fi
 
 echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV

cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py

@@ -10,6 +10,8 @@
 from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError
 from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import (
     IS_WINDOWS,
+    SITE_PACKAGES_LIBDIRS_LINUX,
+    SITE_PACKAGES_LIBDIRS_WINDOWS,
     is_suppressed_dll_file,
 )
 from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs, find_sub_dirs_all_sitepackages
@@ -28,22 +30,25 @@ def _no_such_file_in_sub_dirs(
 def _find_so_using_nvidia_lib_dirs(
     libname: str, so_basename: str, error_messages: list[str], attachments: list[str]
 ) -> Optional[str]:
-    file_wild = so_basename + "*"
-    nvidia_sub_dirs_list: list[tuple[str, ...]] = [("nvidia", "*", "lib")]  # works also for CTK 13 nvvm
-    if libname == "nvvm":
-        nvidia_sub_dirs_list.append(("nvidia", "*", "nvvm", "lib64"))  # CTK 12
-    for nvidia_sub_dirs in nvidia_sub_dirs_list:
-        for lib_dir in find_sub_dirs_all_sitepackages(nvidia_sub_dirs):
-            # First look for an exact match
-            so_name = os.path.join(lib_dir, so_basename)
-            if os.path.isfile(so_name):
-                return so_name
-            # Look for a versioned library
-            # Using sort here mainly to make the result deterministic.
-            for so_name in sorted(glob.glob(os.path.join(lib_dir, file_wild))):
+    rel_dirs = SITE_PACKAGES_LIBDIRS_LINUX.get(libname)
+    if rel_dirs is not None:
+        sub_dirs_searched = []
+        file_wild = so_basename + "*"
+        for rel_dir in rel_dirs:
+            sub_dir = tuple(rel_dir.split(os.path.sep))
+            for abs_dir in find_sub_dirs_all_sitepackages(sub_dir):
+                # First look for an exact match
+                so_name = os.path.join(abs_dir, so_basename)
                 if os.path.isfile(so_name):
                     return so_name
-    _no_such_file_in_sub_dirs(nvidia_sub_dirs, file_wild, error_messages, attachments)
+                # Look for a versioned library
+                # Using sort here mainly to make the result deterministic.
+                for so_name in sorted(glob.glob(os.path.join(abs_dir, file_wild))):
+                    if os.path.isfile(so_name):
+                        return so_name
+            sub_dirs_searched.append(sub_dir)
+        for sub_dir in sub_dirs_searched:
+            _no_such_file_in_sub_dirs(sub_dir, file_wild, error_messages, attachments)
     return None
 
 
@@ -59,18 +64,18 @@ def _find_dll_under_dir(dirpath: str, file_wild: str) -> Optional[str]:
 def _find_dll_using_nvidia_bin_dirs(
     libname: str, lib_searched_for: str, error_messages: list[str], attachments: list[str]
 ) -> Optional[str]:
-    nvidia_sub_dirs_list: list[tuple[str, ...]] = [
-        ("nvidia", "*", "bin"),  # CTK 12
-        ("nvidia", "*", "bin", "*"),  # CTK 13, e.g. site-packages\nvidia\cu13\bin\x86_64\
-    ]
-    if libname == "nvvm":
-        nvidia_sub_dirs_list.append(("nvidia", "*", "nvvm", "bin"))  # Only for CTK 12
-    for nvidia_sub_dirs in nvidia_sub_dirs_list:
-        for bin_dir in find_sub_dirs_all_sitepackages(nvidia_sub_dirs):
-            dll_name = _find_dll_under_dir(bin_dir, lib_searched_for)
-            if dll_name is not None:
-                return dll_name
-    _no_such_file_in_sub_dirs(nvidia_sub_dirs, lib_searched_for, error_messages, attachments)
+    rel_dirs = SITE_PACKAGES_LIBDIRS_WINDOWS.get(libname)
+    if rel_dirs is not None:
+        sub_dirs_searched = []
+        for rel_dir in rel_dirs:
+            sub_dir = tuple(rel_dir.split(os.path.sep))
+            for abs_dir in find_sub_dirs_all_sitepackages(sub_dir):
+                dll_name = _find_dll_under_dir(abs_dir, lib_searched_for)
+                if dll_name is not None:
+                    return dll_name
+            sub_dirs_searched.append(sub_dir)
+        for sub_dir in sub_dirs_searched:
+            _no_such_file_in_sub_dirs(sub_dir, lib_searched_for, error_messages, attachments)
     return None
 
 

cuda_pathfinder/cuda/pathfinder/_dynamic_libs/load_dl_linux.py

@@ -8,7 +8,10 @@
 from typing import Optional, cast
 
 from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL
-from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import SUPPORTED_LINUX_SONAMES
+from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import (
+    LIBNAMES_REQUIRING_RTLD_DEEPBIND,
+    SUPPORTED_LINUX_SONAMES,
+)
 
 CDLL_MODE = os.RTLD_NOW | os.RTLD_GLOBAL
 
@@ -138,6 +141,13 @@ def check_if_already_loaded_from_elsewhere(libname: str, _have_abs_path: bool) -
     return None
 
 
+def _load_lib(libname: str, filename: str) -> ctypes.CDLL:
+    cdll_mode = CDLL_MODE
+    if libname in LIBNAMES_REQUIRING_RTLD_DEEPBIND:
+        cdll_mode |= os.RTLD_DEEPBIND
+    return ctypes.CDLL(filename, cdll_mode)
+
+
 def load_with_system_search(libname: str) -> Optional[LoadedDL]:
     """Try to load a library using system search paths.
 
@@ -152,13 +162,14 @@ def load_with_system_search(libname: str) -> Optional[LoadedDL]:
     """
     for soname in get_candidate_sonames(libname):
         try:
-            handle = ctypes.CDLL(soname, CDLL_MODE)
+            handle = _load_lib(libname, soname)
+        except OSError:
+            pass
+        else:
             abs_path = abs_path_for_dynamic_library(libname, handle)
             if abs_path is None:
                 raise RuntimeError(f"No expected symbol for {libname=!r}")
             return LoadedDL(abs_path, False, handle._handle)
-        except OSError:
-            pass
     return None
 
 
@@ -196,7 +207,7 @@ def load_with_abs_path(libname: str, found_path: str) -> LoadedDL:
     """
     _work_around_known_bugs(libname, found_path)
     try:
-        handle = ctypes.CDLL(found_path, CDLL_MODE)
+        handle = _load_lib(libname, found_path)
     except OSError as e:
         raise RuntimeError(f"Failed to dlopen {found_path}: {e}") from e
     return LoadedDL(found_path, False, handle._handle)

cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py

@@ -63,7 +63,7 @@
 SUPPORTED_LIBNAMES = SUPPORTED_LIBNAMES_WINDOWS if IS_WINDOWS else SUPPORTED_LIBNAMES_LINUX
 
 # Based on ldd output for Linux x86_64 nvidia-*-cu12 wheels (12.8.1)
-DIRECT_DEPENDENCIES = {
+DIRECT_DEPENDENCIES_CTK = {
     "cublas": ("cublasLt",),
     "cufftw": ("cufft",),
     # "cufile_rdma": ("cufile",),
@@ -82,6 +82,10 @@
     "npps": ("nppc",),
     "nvblas": ("cublas", "cublasLt"),
 }
+DIRECT_DEPENDENCIES = DIRECT_DEPENDENCIES_CTK | {
+    "mathdx": ("nvrtc",),
+    "cufftMp": ("nvshmem_host",),
+}
 
 # Based on these released files:
 #   cuda_11.0.3_450.51.06_linux.run
@@ -104,7 +108,7 @@
 #   cuda_12.9.1_575.57.08_linux.run
 #   cuda_13.0.0_580.65.06_linux.run
 # Generated with toolshed/build_pathfinder_sonames.py
-SUPPORTED_LINUX_SONAMES = {
+SUPPORTED_LINUX_SONAMES_CTK = {
     "cublas": (
         "libcublas.so.11",
         "libcublas.so.12",
@@ -232,6 +236,13 @@
         "libnvvm.so.4",
     ),
 }
+SUPPORTED_LINUX_SONAMES_OTHER = {
+    "cufftMp": ("libcufftMp.so.11",),
+    "mathdx": ("libmathdx.so.0",),
+    "nvpl_fftw": ("libnvpl_fftw.so.0",),
+    "nvshmem_host": ("libnvshmem_host.so.3",),
+}
+SUPPORTED_LINUX_SONAMES = SUPPORTED_LINUX_SONAMES_CTK | SUPPORTED_LINUX_SONAMES_OTHER
 
 # Based on these released files:
 #   cuda_11.0.3_451.82_win10.exe
@@ -254,7 +265,7 @@
 #   cuda_12.9.1_576.57_windows.exe
 #   cuda_13.0.0_windows.exe
 # Generated with toolshed/build_pathfinder_dlls.py
-SUPPORTED_WINDOWS_DLLS = {
+SUPPORTED_WINDOWS_DLLS_CTK = {
     "cublas": (
         "cublas64_11.dll",
         "cublas64_12.dll",
@@ -384,12 +395,91 @@
         "nvvm70.dll",
     ),
 }
+SUPPORTED_WINDOWS_DLLS_OTHER = {
+    "mathdx": ("mathdx64_0.dll",),
+}
+SUPPORTED_WINDOWS_DLLS = SUPPORTED_WINDOWS_DLLS_CTK | SUPPORTED_WINDOWS_DLLS_OTHER
 
 LIBNAMES_REQUIRING_OS_ADD_DLL_DIRECTORY = (
     "cufft",
     "nvrtc",
 )
 
+LIBNAMES_REQUIRING_RTLD_DEEPBIND = ("cufftMp",)
+
+# Based on output of toolshed/make_site_packages_libdirs_linux.py
+SITE_PACKAGES_LIBDIRS_LINUX_CTK = {
+    "cublas": ("nvidia/cu13/lib", "nvidia/cublas/lib"),
+    "cublasLt": ("nvidia/cu13/lib", "nvidia/cublas/lib"),
+    "cudart": ("nvidia/cu13/lib", "nvidia/cuda_runtime/lib"),
+    "cufft": ("nvidia/cu13/lib", "nvidia/cufft/lib"),
+    "cufftw": ("nvidia/cu13/lib", "nvidia/cufft/lib"),
+    "cufile": ("nvidia/cu13/lib", "nvidia/cufile/lib"),
+    # "cufile_rdma": ("nvidia/cu13/lib", "nvidia/cufile/lib"),
+    "curand": ("nvidia/cu13/lib", "nvidia/curand/lib"),
+    "cusolver": ("nvidia/cu13/lib", "nvidia/cusolver/lib"),
+    "cusolverMg": ("nvidia/cu13/lib", "nvidia/cusolver/lib"),
+    "cusparse": ("nvidia/cu13/lib", "nvidia/cusparse/lib"),
+    "nppc": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppial": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppicc": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppidei": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppif": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppig": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppim": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppist": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppisu": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nppitc": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "npps": ("nvidia/cu13/lib", "nvidia/npp/lib"),
+    "nvJitLink": ("nvidia/cu13/lib", "nvidia/nvjitlink/lib"),
+    "nvblas": ("nvidia/cu13/lib", "nvidia/cublas/lib"),
+    "nvfatbin": ("nvidia/cu13/lib", "nvidia/nvfatbin/lib"),
+    "nvjpeg": ("nvidia/cu13/lib", "nvidia/nvjpeg/lib"),
+    "nvrtc": ("nvidia/cu13/lib", "nvidia/cuda_nvrtc/lib"),
+    "nvvm": ("nvidia/cu13/lib", "nvidia/cuda_nvcc/nvvm/lib64"),
+}
+SITE_PACKAGES_LIBDIRS_LINUX_OTHER = {
+    "cufftMp": ("nvidia/cufftmp/cu12/lib",),
+    "mathdx": ("nvidia/cu13/lib", "nvidia/cu12/lib"),
+    "nvpl_fftw": ("nvpl/lib",),
+    "nvshmem_host": ("nvidia/nvshmem/lib",),
+}
+SITE_PACKAGES_LIBDIRS_LINUX = SITE_PACKAGES_LIBDIRS_LINUX_CTK | SITE_PACKAGES_LIBDIRS_LINUX_OTHER
+
+# Based on output of toolshed/make_site_packages_libdirs_windows.py
+SITE_PACKAGES_LIBDIRS_WINDOWS_CTK = {
+    "cublas": ("nvidia/cu13/bin/x86_64", "nvidia/cublas/bin"),
+    "cublasLt": ("nvidia/cu13/bin/x86_64", "nvidia/cublas/bin"),
+    "cudart": ("nvidia/cu13/bin/x86_64", "nvidia/cuda_runtime/bin"),
+    "cufft": ("nvidia/cu13/bin/x86_64", "nvidia/cufft/bin"),
+    "cufftw": ("nvidia/cu13/bin/x86_64", "nvidia/cufft/bin"),
+    "curand": ("nvidia/cu13/bin/x86_64", "nvidia/curand/bin"),
+    "cusolver": ("nvidia/cu13/bin/x86_64", "nvidia/cusolver/bin"),
+    "cusolverMg": ("nvidia/cu13/bin/x86_64", "nvidia/cusolver/bin"),
+    "cusparse": ("nvidia/cu13/bin/x86_64", "nvidia/cusparse/bin"),
+    "nppc": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppial": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppicc": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppidei": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppif": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppig": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppim": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppist": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppisu": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nppitc": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "npps": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"),
+    "nvJitLink": ("nvidia/cu13/bin/x86_64", "nvidia/nvjitlink/bin"),
+    "nvblas": ("nvidia/cu13/bin/x86_64", "nvidia/cublas/bin"),
+    "nvfatbin": ("nvidia/cu13/bin/x86_64", "nvidia/nvfatbin/bin"),
+    "nvjpeg": ("nvidia/cu13/bin/x86_64", "nvidia/nvjpeg/bin"),
+    "nvrtc": ("nvidia/cu13/bin/x86_64", "nvidia/cuda_nvrtc/bin"),
+    "nvvm": ("nvidia/cu13/bin/x86_64", "nvidia/cuda_nvcc/nvvm/bin"),
+}
+SITE_PACKAGES_LIBDIRS_WINDOWS_OTHER = {
+    "mathdx": ("nvidia/cu13/bin/x86_64", "nvidia/cu12/bin"),
+}
+SITE_PACKAGES_LIBDIRS_WINDOWS = SITE_PACKAGES_LIBDIRS_WINDOWS_CTK | SITE_PACKAGES_LIBDIRS_WINDOWS_OTHER
+
 
 def is_suppressed_dll_file(path_basename: str) -> bool:
     if path_basename.startswith("nvrtc"):

cuda_pathfinder/cuda/pathfinder/_utils/find_site_packages_dll.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import collections
+import functools
+import importlib.metadata
+
+
+@functools.cache
+def find_all_dll_files_via_metadata() -> dict[str, tuple[str, ...]]:
+    results: collections.defaultdict[str, list[str]] = collections.defaultdict(list)
+
+    # sort dists for deterministic output
+    for dist in sorted(importlib.metadata.distributions(), key=lambda d: (d.metadata.get("Name", ""), d.version)):
+        files = dist.files
+        if not files:
+            continue
+        for relpath in sorted(files, key=lambda p: str(p)):  # deterministic
+            relname = relpath.name.lower()
+            if not relname.endswith(".dll"):
+                continue
+            abs_path = str(dist.locate_file(relpath))
+            results[relname].append(abs_path)
+
+    # plain dicts; sort inner list for stability
+    return {k: tuple(sorted(v)) for k, v in results.items()}

cuda_pathfinder/cuda/pathfinder/_utils/find_site_packages_so.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import collections
+import functools
+import importlib.metadata
+import re
+
+_SO_RE = re.compile(r"\.so(?:$|\.)")  # matches libfoo.so or libfoo.so.1.2.3
+
+
+def split_so_version_suffix(so_filename: str) -> tuple[str, str]:
+    idx = so_filename.rfind(".so")
+    assert idx > 0, so_filename
+    idx += 3
+    return (so_filename[:idx], so_filename[idx:])
+
+
+@functools.cache
+def find_all_so_files_via_metadata() -> dict[str, dict[str, tuple[str, ...]]]:
+    results: collections.defaultdict[str, collections.defaultdict[str, list[str]]] = collections.defaultdict(
+        lambda: collections.defaultdict(list)
+    )
+
+    # sort dists for deterministic output
+    for dist in sorted(importlib.metadata.distributions(), key=lambda d: (d.metadata.get("Name", ""), d.version)):
+        files = dist.files
+        if not files:
+            continue
+        for relpath in sorted(files, key=lambda p: str(p)):  # deterministic
+            relname = relpath.name
+            if not _SO_RE.search(relname):
+                continue
+            so_basename, so_version_suffix = split_so_version_suffix(relname)
+            abs_path = str(dist.locate_file(relpath))
+            results[so_basename][so_version_suffix].append(abs_path)
+
+    # plain dicts; sort inner lists for stability
+    return {k: {kk: tuple(sorted(vv)) for kk, vv in v.items()} for k, v in results.items()}

cuda_pathfinder/cuda/pathfinder/_version.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-__version__ = "1.1.1a2"
+__version__ = "1.1.1a3"