[MLIR][Python] use FetchContent_Declare for nanobind and remove pybind

Author: makslevental

mlir/cmake/modules/AddMLIRPython.cmake

@@ -123,12 +123,12 @@ function(mlir_generate_type_stubs)
     "IMPORT_PATHS;DEPENDS_TARGETS;OUTPUTS;DEPENDS_TARGET_SRC_DEPS"
     ${ARGN})
 
-  # for people doing find_package(nanobind)
+  # for people installing a distro (e.g., pip install) of nanobind
   if(EXISTS ${nanobind_DIR}/../src/stubgen.py)
     set(NB_STUBGEN "${nanobind_DIR}/../src/stubgen.py")
   elseif(EXISTS ${nanobind_DIR}/../stubgen.py)
     set(NB_STUBGEN "${nanobind_DIR}/../stubgen.py")
-  # for people using FetchContent_Declare and FetchContent_MakeAvailable
+  # for people using nanobind git source tree (e.g., FetchContent_Declare and FetchContent_MakeAvailable)
   elseif(EXISTS ${nanobind_SOURCE_DIR}/src/stubgen.py)
     set(NB_STUBGEN "${nanobind_SOURCE_DIR}/src/stubgen.py")
   elseif(EXISTS ${nanobind_SOURCE_DIR}/stubgen.py)
@@ -203,11 +203,10 @@ endfunction()
 #   EMBED_CAPI_LINK_LIBS: Dependent CAPI libraries that this extension depends
 #     on. These will be collected for all extensions and put into an
 #     aggregate dylib that is linked against.
-#   PYTHON_BINDINGS_LIBRARY: Either pybind11 or nanobind.
 function(declare_mlir_python_extension name)
   cmake_parse_arguments(ARG
     ""
-    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT;PYTHON_BINDINGS_LIBRARY"
+    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT"
     "SOURCES;PRIVATE_LINK_LIBS;EMBED_CAPI_LINK_LIBS"
     ${ARGN})
 
@@ -216,20 +215,15 @@ function(declare_mlir_python_extension name)
   endif()
   set(_install_destination "src/python/${name}")
 
-  if(NOT ARG_PYTHON_BINDINGS_LIBRARY)
-    set(ARG_PYTHON_BINDINGS_LIBRARY "pybind11")
-  endif()
-
   add_library(${name} INTERFACE)
   set_target_properties(${name} PROPERTIES
     # Yes: Leading-lowercase property names are load bearing and the recommended
     # way to do this: https://gitlab.kitware.com/cmake/cmake/-/issues/19261
-    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS;mlir_python_BINDINGS_LIBRARY"
+    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS"
     mlir_python_SOURCES_TYPE extension
     mlir_python_EXTENSION_MODULE_NAME "${ARG_MODULE_NAME}"
     mlir_python_EMBED_CAPI_LINK_LIBS "${ARG_EMBED_CAPI_LINK_LIBS}"
     mlir_python_DEPENDS ""
-    mlir_python_BINDINGS_LIBRARY "${ARG_PYTHON_BINDINGS_LIBRARY}"
   )
 
   # Set the interface source and link_libs properties of the target
@@ -318,14 +312,12 @@ function(add_mlir_python_modules name)
     elseif(_source_type STREQUAL "extension")
       # Native CPP extension.
       get_target_property(_module_name ${sources_target} mlir_python_EXTENSION_MODULE_NAME)
-      get_target_property(_bindings_library ${sources_target} mlir_python_BINDINGS_LIBRARY)
       # Transform relative source to based on root dir.
       set(_extension_target "${modules_target}.extension.${_module_name}.dso")
       add_mlir_python_extension(${_extension_target} "${_module_name}"
         INSTALL_COMPONENT ${modules_target}
         INSTALL_DIR "${ARG_INSTALL_PREFIX}/_mlir_libs"
         OUTPUT_DIRECTORY "${ARG_ROOT_PREFIX}/_mlir_libs"
-        PYTHON_BINDINGS_LIBRARY ${_bindings_library}
         LINK_LIBS PRIVATE
           ${sources_target}
           ${ARG_COMMON_CAPI_LINK_LIBS}
@@ -730,78 +722,69 @@ endfunction()
 function(add_mlir_python_extension libname extname)
   cmake_parse_arguments(ARG
   ""
-  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY;PYTHON_BINDINGS_LIBRARY"
+  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY"
   "SOURCES;LINK_LIBS"
   ${ARGN})
   if(ARG_UNPARSED_ARGUMENTS)
     message(FATAL_ERROR "Unhandled arguments to add_mlir_python_extension(${libname}, ... : ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
   # The extension itself must be compiled with RTTI and exceptions enabled.
-  # Also, some warning classes triggered by pybind11 are disabled.
+  # Also, some warning classes triggered by nanobind are disabled.
   set(eh_rtti_enable)
   if (MSVC)
     set(eh_rtti_enable /EHsc /GR)
   elseif(LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL)
     set(eh_rtti_enable -frtti -fexceptions)
   endif ()
 
-  # The actual extension library produces a shared-object or DLL and has
-  # sources that must be compiled in accordance with pybind11 needs (RTTI and
-  # exceptions).
-  if(NOT DEFINED ARG_PYTHON_BINDINGS_LIBRARY OR ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "pybind11")
-    pybind11_add_module(${libname}
-      ${ARG_SOURCES}
-    )
-  elseif(ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "nanobind")
-    nanobind_add_module(${libname}
-      NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}
-      FREE_THREADED
-      ${ARG_SOURCES}
-    )
+  nanobind_add_module(${libname}
+    NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}
+    FREE_THREADED
+    ${ARG_SOURCES}
+  )
 
-    if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES
-        AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
-      # Avoid some warnings from upstream nanobind.
-      # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let
-      # the super project handle compile options as it wishes.
-      get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)
-      target_compile_options(${NB_LIBRARY_TARGET_NAME}
-        PRIVATE
-          -Wall -Wextra -Wpedantic
-          -Wno-c++98-compat-extra-semi
-          -Wno-cast-qual
-          -Wno-covered-switch-default
-          -Wno-deprecated-literal-operator
-          -Wno-nested-anon-types
-          -Wno-unused-parameter
-          -Wno-zero-length-array
-          ${eh_rtti_enable})
-
-      target_compile_options(${libname}
-        PRIVATE
-          -Wall -Wextra -Wpedantic
-          -Wno-c++98-compat-extra-semi
-          -Wno-cast-qual
-          -Wno-covered-switch-default
-          -Wno-deprecated-literal-operator
-          -Wno-nested-anon-types
-          -Wno-unused-parameter
-          -Wno-zero-length-array
-          ${eh_rtti_enable})
-    endif()
+  if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES
+      AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
+    # Avoid some warnings from upstream nanobind.
+    # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let
+    # the super project handle compile options as it wishes.
+    get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)
+    target_compile_options(${NB_LIBRARY_TARGET_NAME}
+      PRIVATE
+        -Wall -Wextra -Wpedantic
+        -Wno-c++98-compat-extra-semi
+        -Wno-cast-qual
+        -Wno-covered-switch-default
+        -Wno-deprecated-literal-operator
+        -Wno-nested-anon-types
+        -Wno-unused-parameter
+        -Wno-zero-length-array
+        ${eh_rtti_enable})
+
+    target_compile_options(${libname}
+      PRIVATE
+        -Wall -Wextra -Wpedantic
+        -Wno-c++98-compat-extra-semi
+        -Wno-cast-qual
+        -Wno-covered-switch-default
+        -Wno-deprecated-literal-operator
+        -Wno-nested-anon-types
+        -Wno-unused-parameter
+        -Wno-zero-length-array
+        ${eh_rtti_enable})
+  endif()
 
-    if(APPLE)
-      # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
-      # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
-      # for downstream users that do not do something like `-undefined dynamic_lookup`.
-      # Same for the rest.
-      target_link_options(${libname} PUBLIC
-        "LINKER:-U,_PyClassMethod_New"
-        "LINKER:-U,_PyCode_Addr2Location"
-        "LINKER:-U,_PyFrame_GetLasti"
-      )
-    endif()
+  if(APPLE)
+    # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
+    # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
+    # for downstream users that do not do something like `-undefined dynamic_lookup`.
+    # Same for the rest.
+    target_link_options(${libname} PUBLIC
+      "LINKER:-U,_PyClassMethod_New"
+      "LINKER:-U,_PyCode_Addr2Location"
+      "LINKER:-U,_PyFrame_GetLasti"
+    )
   endif()
 
   target_compile_options(${libname} PRIVATE ${eh_rtti_enable})
@@ -839,11 +822,11 @@ function(add_mlir_python_extension libname extname)
   if(WIN32)
     # On Windows, pyconfig.h (and by extension python.h) hardcode the version of the
     # python library which will be used for linkage depending on the flavor of the build.
-    # pybind11 has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG
-    # is not passed in as a compile definition, pybind11 undefs _DEBUG when including
+    # nanobind has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG
+    # is not passed in as a compile definition, nanobind undefs _DEBUG when including
     # python.h, so that the release python library would be used).
-    # Since mlir uses pybind11, we can leverage their workaround by never directly
-    # pyconfig.h or python.h and instead relying on the pybind11 headers to include the
+    # Since mlir uses nanobind, we can leverage their workaround by never directly
+    # pyconfig.h or python.h and instead relying on the nanobind headers to include the
     # necessary python headers. This results in mlir always linking against the
     # release python library via the (undocumented) cmake property Python3_LIBRARY_RELEASE.
     target_link_libraries(${libname} PRIVATE ${Python3_LIBRARY_RELEASE})

mlir/cmake/modules/MLIRDetectPythonEnv.cmake

@@ -46,81 +46,52 @@ macro(mlir_configure_python_dev_packages)
     message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
     message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
     message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
-    mlir_detect_pybind11_install()
-    find_package(pybind11 2.10 CONFIG REQUIRED)
-    message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
-    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
-                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
-                  "extension = '${PYTHON_MODULE_EXTENSION}")
-
-    mlir_detect_nanobind_install()
-    find_package(nanobind 2.9 CONFIG REQUIRED)
-    message(STATUS "Found nanobind v${nanobind_VERSION}: ${nanobind_INCLUDE_DIR}")
-    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
-                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
-                  "extension = '${PYTHON_MODULE_EXTENSION}")
-  endif()
-endmacro()
-
-# Detects a pybind11 package installed in the current python environment
-# and sets variables to allow it to be found. This allows pybind11 to be
-# installed via pip, which typically yields a much more recent version than
-# the OS install, which will be available otherwise.
-function(mlir_detect_pybind11_install)
-  if(pybind11_DIR)
-    message(STATUS "Using explicit pybind11 cmake directory: ${pybind11_DIR} (-Dpybind11_DIR to change)")
-  else()
-    message(STATUS "Checking for pybind11 in python path...")
-    execute_process(
-      COMMAND "${Python3_EXECUTABLE}"
-      -c "import pybind11;print(pybind11.get_cmake_dir(), end='')"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      RESULT_VARIABLE STATUS
-      OUTPUT_VARIABLE PACKAGE_DIR
-      ERROR_QUIET)
-    if(NOT STATUS EQUAL "0")
-      message(STATUS "not found (install via 'pip install pybind11' or set pybind11_DIR)")
-      return()
-    endif()
-    message(STATUS "found (${PACKAGE_DIR})")
-    set(pybind11_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
-  endif()
-endfunction()
-
-
-# Detects a nanobind package installed in the current python environment
-# and sets variables to allow it to be found. This allows nanobind to be
-# installed via pip, which typically yields a much more recent version than
-# the OS install, which will be available otherwise.
-function(mlir_detect_nanobind_install)
-  if(nanobind_DIR)
-    message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
-  else()
-    message(STATUS "Checking for nanobind in python path...")
-    execute_process(
-      COMMAND "${Python3_EXECUTABLE}"
-      -c "import nanobind;print(nanobind.cmake_dir(), end='')"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      RESULT_VARIABLE STATUS
-      OUTPUT_VARIABLE PACKAGE_DIR
-      ERROR_QUIET)
-    if(NOT STATUS EQUAL "0")
-      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
-      return()
+    message(STATUS "Python extension suffix for modules: '${Python3_SOABI}'")
+    if(nanobind_DIR)
+      message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
+      find_package(nanobind 2.9 CONFIG REQUIRED)
+    else()
+      include(FetchContent)
+#      # nanobind uses tsl-robin-map and since we're not using GIT for nanobind we need to
+#      # get tsl-robin-map manually too.
+#      FetchContent_Declare(
+#        tsl-robin-map
+#        OVERRIDE_FIND_PACKAGE
+#        # timestamp the files with the extraction time instead of archive time
+#        DOWNLOAD_EXTRACT_TIMESTAMP FALSE
+#        URL https://github.com/Tessil/robin-map/archive/refs/tags/v1.4.0.tar.gz
+#        # put this last otherwise windows parses everything after the hash as part of the hash???
+#        URL_HASH MD5=d56a879c94e021c55d8956e37deb3e4f
+#      )
+#      FetchContent_MakeAvailable(tsl-robin-map)
+#      # Tell nanobind to use find_project(tsl-robin-map) instead of searching its
+#      # submodule folder.
+#      set(NB_USE_SUBMODULE_DEPS OFF CACHE INTERNAL "Switch off Nanobind submodule deps")
+#      FetchContent_Declare(
+#        nanobind
+#        OVERRIDE_FIND_PACKAGE
+#        # timestamp the files with the extraction time instead of archive time
+#        DOWNLOAD_EXTRACT_TIMESTAMP FALSE
+#        URL https://github.com/wjakob/nanobind/archive/refs/tags/v2.9.0.tar.gz
+#        # put this last otherwise windows parses everything after the hash as part of the hash???
+#        URL_HASH MD5=df0e9de9d5fd817df264584be4917fd0
+#      )
+#      FetchContent_MakeAvailable(nanobind)
+      FetchContent_Declare(
+        nanobind
+        GIT_REPOSITORY https://github.com/wjakob/nanobind.git
+        GIT_TAG        v2.9.0
+        GIT_SHALLOW    TRUE
+        OVERRIDE_FIND_PACKAGE
+      )
+      FetchContent_MakeAvailable(nanobind)
+      if(CMAKE_VERSION VERSION_LESS "3.24.0")
+        # OVERRIDE_FIND_PACKAGE not implemented so have to set the dirs manually.
+        set(nanobind_DIR "${nanobind_SOURCE_DIR}/cmake" CACHE INTERNAL "")
+      endif()
+      # no PACKAGE_VERSION in this path but we know it's 2.9 of course.
+      find_package(nanobind CONFIG REQUIRED)
     endif()
-    message(STATUS "found (${PACKAGE_DIR})")
-    set(nanobind_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
-    execute_process(
-      COMMAND "${Python3_EXECUTABLE}"
-      -c "import nanobind;print(nanobind.include_dir(), end='')"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      RESULT_VARIABLE STATUS
-      OUTPUT_VARIABLE PACKAGE_DIR
-      ERROR_QUIET)
-    if(NOT STATUS EQUAL "0")
-      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
-      return()
-    endif()
-    set(nanobind_INCLUDE_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
+    message(STATUS "Found nanobind: ${NB_DIR}")
   endif()
-endfunction()
+endmacro()

mlir/examples/standalone/python/CMakeLists.txt

@@ -16,27 +16,10 @@ declare_mlir_dialect_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir_standalone"
   TD_FILE dialects/StandaloneOps.td
   SOURCES
-    dialects/standalone_pybind11.py
     dialects/standalone_nanobind.py
     _mlir_libs/_standaloneDialectsNanobind/py.typed
   DIALECT_NAME standalone)
 
-
-declare_mlir_python_extension(StandalonePythonSources.Pybind11Extension
-  MODULE_NAME _standaloneDialectsPybind11
-  ADD_TO_PARENT StandalonePythonSources
-  SOURCES
-    StandaloneExtensionPybind11.cpp
-  PRIVATE_LINK_LIBS
-    LLVMSupport
-  EMBED_CAPI_LINK_LIBS
-    MLIRCAPIIR
-    MLIRCAPIArith
-    MLIRCAPITransforms
-    StandaloneCAPI
-  PYTHON_BINDINGS_LIBRARY pybind11
-)
-
 declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
   MODULE_NAME _standaloneDialectsNanobind
   ADD_TO_PARENT StandalonePythonSources
@@ -49,7 +32,6 @@ declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
     MLIRCAPIArith
     MLIRCAPITransforms
     StandaloneCAPI
-  PYTHON_BINDINGS_LIBRARY nanobind
 )