diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d292c209a6..950d9bc6998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ print_configured_options() include(tools/cmake/Utils.cmake) include(CMakeDependentOption) +include(ExternalProject) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -260,6 +261,8 @@ cmake_dependent_option( "NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF ) +add_subdirectory(third-party) + if(EXECUTORCH_BUILD_EXTENSION_TRAINING) set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON) set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON) @@ -454,81 +457,6 @@ if(EXECUTORCH_USE_CPP_CODE_COVERAGE) endif() endif() -# -# flatc: Flatbuffer commandline tool to generate .h files from .fbs files -# -cmake_dependent_option( - EXECUTORCH_BUILD_FLATC "Build the flatc executable." ON - "NOT FLATC_EXECUTABLE" OFF -) - -set(FLATBUFFERS_BUILD_FLATC OFF CACHE BOOL "") -set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "") -set(FLATBUFFERS_BUILD_FLATLIB OFF CACHE BOOL "") -set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "") -set(FLATBUFFERS_INSTALL OFF CACHE BOOL "") -# exir lets users set the alignment of tensor data embedded in the flatbuffer, -# and some users need an alignment larger than the default, which is typically -# 32. -set(FLATBUFFERS_MAX_ALIGNMENT 1024) - -if(EXECUTORCH_BUILD_FLATC) - if(FLATC_EXECUTABLE) - # We could ignore this, but it could lead to confusion about which `flatc` - # is actually being used. - message( - FATAL_ERROR "May not set both EXECUTORCH_BUILD_FLATC and FLATC_EXECUTABLE" - ) - endif() - - # Build flatc for the *host* to generate files as part of the build step. - include(ExternalProject) - ExternalProject_Add( - flatbuffers - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/third-party/flatbuffers - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/flatbuffers - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/flatbuffers - CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON - -DFLATBUFFERS_BUILD_FLATHASH=${FLATBUFFERS_BUILD_FLATHASH} - -DFLATBUFFERS_BUILD_FLATLIB=${FLATBUFFERS_BUILD_FLATLIB} - -DFLATBUFFERS_BUILD_TESTS=${FLATBUFFERS_BUILD_TESTS} - -DFLATBUFFERS_INSTALL=${FLATBUFFERS_INSTALL} - -DCMAKE_CXX_FLAGS="-DFLATBUFFERS_MAX_ALIGNMENT=${FLATBUFFERS_MAX_ALIGNMENT}" - # If building for iOS, "unset" these variables to rely on the host (macOS) defaults. - $<$,$>>:-DCMAKE_OSX_SYSROOT=> - INSTALL_COMMAND "" - BUILD_BYPRODUCTS /flatc - ) - ExternalProject_Get_Property(flatbuffers BINARY_DIR) - if(WIN32) - # flatbuffers does not use CMAKE_BUILD_TYPE. Internally, the build forces Release - # config, but from CMake's perspective the build type is always Debug. - set(FLATC_EXECUTABLE ${BINARY_DIR}/$/flatc.exe) - elseif(CMAKE_GENERATOR STREQUAL "Xcode") - set(FLATC_EXECUTABLE ${BINARY_DIR}/$/flatc) - else() - set(FLATC_EXECUTABLE ${BINARY_DIR}/flatc) - endif() - set(FLATC_EXECUTABLE_BUILT_FROM_SOURCE YES) -endif() - -if(NOT FLATC_EXECUTABLE) - message( - WARNING "FLATC_EXECUTABLE not specified, looking for flatc" - ) - find_program(FLATC_EXECUTABLE flatc) - - if(NOT FLATC_EXECUTABLE) - message(FATAL_ERROR "FLATC_EXECUTABLE must be set when EXECUTORCH_BUILD_FLATC is disabled.") - endif() -endif() - -add_executable(flatc IMPORTED GLOBAL) -set_target_properties(flatc PROPERTIES IMPORTED_LOCATION ${FLATC_EXECUTABLE}) - -if(FLATC_EXECUTABLE_BUILT_FROM_SOURCE) - add_dependencies(flatc flatbuffers) -endif() # # program_schema: Generated .h files from schema/*.fbs inputs diff --git a/backends/apple/mps/CMakeLists.txt b/backends/apple/mps/CMakeLists.txt index 132307c5c78..f8119fbea97 100644 --- a/backends/apple/mps/CMakeLists.txt +++ b/backends/apple/mps/CMakeLists.txt @@ -42,7 +42,7 @@ endforeach() add_custom_command( OUTPUT ${_mps_schema__outputs} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o + flatc --cpp --cpp-std c++11 --scoped-enums -o "${_mps_schema__include_dir}/executorch/backends/apple/mps" ${_mps_schema__srcs} WORKING_DIRECTORY ${EXECUTORCH_ROOT} diff --git a/backends/qualcomm/CMakeLists.txt b/backends/qualcomm/CMakeLists.txt index 1b7c8891a4e..d0ea30011a9 100644 --- a/backends/qualcomm/CMakeLists.txt +++ b/backends/qualcomm/CMakeLists.txt @@ -43,7 +43,7 @@ set(qcir_schema_include_dir ${CMAKE_CURRENT_LIST_DIR}/aot/ir) set(qcir_schema_output ${qcir_schema_include_dir}/qcir_generated.h) add_custom_command( OUTPUT qcir_schema_output - COMMAND ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o + COMMAND flatc --cpp --cpp-std c++11 --scoped-enums -o ${qcir_schema_include_dir} ${qcir_schema_include_dir}/qcir.fbs DEPENDS flatc COMMENT "Generating qualcomm ir schema headers" @@ -94,7 +94,7 @@ endforeach() add_custom_command( OUTPUT ${_qnn_schema__outputs} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o + flatc --cpp --cpp-std c++11 --scoped-enums -o "${_qnn_schema__include_dir}/executorch/backends/qualcomm" ${_qnn_schema__srcs} WORKING_DIRECTORY ${EXECUTORCH_SOURCE_DIR} diff --git a/backends/vulkan/CMakeLists.txt b/backends/vulkan/CMakeLists.txt index c17ca0b4ecf..ae3ec4b4e24 100644 --- a/backends/vulkan/CMakeLists.txt +++ b/backends/vulkan/CMakeLists.txt @@ -85,7 +85,7 @@ set(GENERATED_HEADER add_custom_command( OUTPUT ${GENERATED_HEADER} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o + flatc --cpp --cpp-std c++11 --scoped-enums -o "${SCHEMA_INCLUDE_DIR}/executorch/backends/vulkan/serialization/" ${_vulkan_schema__srcs} WORKING_DIRECTORY ${EXECUTORCH_ROOT} DEPENDS flatc diff --git a/backends/xnnpack/CMakeLists.txt b/backends/xnnpack/CMakeLists.txt index 250190a5712..bae557037f1 100644 --- a/backends/xnnpack/CMakeLists.txt +++ b/backends/xnnpack/CMakeLists.txt @@ -92,7 +92,7 @@ endif() add_custom_command( OUTPUT ${_xnnpack_schema__outputs} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o + flatc --cpp --cpp-std c++11 --scoped-enums -o "${_xnnpack_schema__include_dir}/executorch/backends/xnnpack/serialization" ${_xnnpack_schema__srcs} COMMAND ${MV_COMMAND} diff --git a/devtools/CMakeLists.txt b/devtools/CMakeLists.txt index 9dd38d3678e..f9fd2086b91 100644 --- a/devtools/CMakeLists.txt +++ b/devtools/CMakeLists.txt @@ -199,7 +199,7 @@ target_link_libraries( add_custom_command( OUTPUT ${_bundled_program_schema__outputs} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o + flatc --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o "${_bundled_schema__include_dir}/executorch/devtools/bundled_program/schema" ${_bundled_program_schema__srcs} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/devtools diff --git a/extension/flat_tensor/serialize/CMakeLists.txt b/extension/flat_tensor/serialize/CMakeLists.txt index d1ae797f8b3..39b364797b8 100644 --- a/extension/flat_tensor/serialize/CMakeLists.txt +++ b/extension/flat_tensor/serialize/CMakeLists.txt @@ -30,7 +30,7 @@ function(generate_flat_tensor_schema _schema_srcs _schema_name) add_custom_command( OUTPUT ${_schema_outputs} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o + flatc --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o "${_flat_tensor_schema__output_dir}" ${_schema_srcs} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS flatc ${_schema_srcs} @@ -45,7 +45,7 @@ function(generate_flat_tensor_schema _schema_srcs _schema_name) # and some users need an alignment larger than the default, which is typically # 32. target_compile_definitions( - ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${FLATBUFFERS_MAX_ALIGNMENT} + ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT} ) target_include_directories( diff --git a/schema/CMakeLists.txt b/schema/CMakeLists.txt index 484363acdf5..f5bb3ff3ca7 100644 --- a/schema/CMakeLists.txt +++ b/schema/CMakeLists.txt @@ -30,7 +30,7 @@ function(generate_program_schema _schema_srcs _schema_name) add_custom_command( OUTPUT ${_schema_outputs} COMMAND - ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o + flatc --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o "${_program_schema__output_dir}" ${_schema_srcs} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS flatc ${_schema_srcs} @@ -45,7 +45,7 @@ function(generate_program_schema _schema_srcs _schema_name) # and some users need an alignment larger than the default, which is typically # 32. target_compile_definitions( - ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${FLATBUFFERS_MAX_ALIGNMENT} + ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT} ) target_include_directories( diff --git a/setup.py b/setup.py index 2c5f5578bcf..033426f29ce 100644 --- a/setup.py +++ b/setup.py @@ -156,10 +156,6 @@ def llama_custom_ops(cls) -> bool: "EXECUTORCH_BUILD_KERNELS_CUSTOM_AOT", default=True ) - @classmethod - def flatc(cls) -> bool: - return cls._is_cmake_arg_enabled("EXECUTORCH_BUILD_FLATC", default=True) - class Version: """Static strings that describe the version of the pip package.""" @@ -833,23 +829,20 @@ def run(self): def get_ext_modules() -> List[Extension]: """Returns the set of extension modules to build.""" - ext_modules = [] - if ShouldBuild.flatc(): - ext_modules.extend( - [ - BuiltFile( - src_dir="%CMAKE_CACHE_DIR%/third-party/flatbuffers/%BUILD_TYPE%/", - src_name="flatc", - dst="executorch/data/bin/", - is_executable=True, - ), - BuiltFile( - src_dir="tools/wheel", - src_name="pip_data_bin_init.py.in", - dst="executorch/data/bin/__init__.py", - ), - ] - ) + + ext_modules = [ + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/third-party/flatbuffers_external_project", + src_name="flatc", + dst="executorch/data/bin/", + is_executable=True, + ), + BuiltFile( + src_dir="tools/wheel", + src_name="pip_data_bin_init.py.in", + dst="executorch/data/bin/__init__.py", + ), + ] if ShouldBuild.pybindings(): ext_modules.append( diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt new file mode 100644 index 00000000000..2aa606927c3 --- /dev/null +++ b/third-party/CMakeLists.txt @@ -0,0 +1,49 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# MARK: - flatbuffers + +# We use ExternalProject to build flatc from source to force it target the host. +# Otherwise, flatc will target the project's toolchain (i.e. iOS, or Android). +ExternalProject_Add( + flatbuffers_external_project + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_external_project + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_external_project + SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatbuffers + # Always use Make to avoid needing to codesign flatc if the project is using Xcode. + CMAKE_GENERATOR "Unix Makefiles" + CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON + -DFLATBUFFERS_INSTALL=OFF + -DFLATBUFFERS_BUILD_FLATHASH=OFF + -DFLATBUFFERS_BUILD_FLATLIB=OFF + -DFLATBUFFERS_BUILD_TESTS=OFF + -DCMAKE_CXX_FLAGS="-DFLATBUFFERS_MAX_ALIGNMENT=${EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT}" + # Unset the toolchain to build for the host instead of the toolchain set for the project. + -DCMAKE_TOOLCHAIN_FILE= + # If building for iOS, "unset" these variables to rely on the host (macOS) defaults. + $<$,$>>:-DCMAKE_OSX_SYSROOT=> + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${CMAKE_OSX_DEPLOYMENT_TARGET} + INSTALL_COMMAND "" + BUILD_BYPRODUCTS /flatc +) +ExternalProject_Get_Property(flatbuffers_external_project BINARY_DIR) +add_executable(flatc IMPORTED GLOBAL) +add_dependencies(flatc flatbuffers_external_project) +if(WIN32) + # flatbuffers does not use CMAKE_BUILD_TYPE. Internally, the build forces Release + # config, but from CMake's perspective the build type is always Debug. + set_target_properties(flatc PROPERTIES IMPORTED_LOCATION ${BINARY_DIR}/$/flatc.exe) +else() + set_target_properties(flatc PROPERTIES IMPORTED_LOCATION ${BINARY_DIR}/flatc) +endif() + +# TODO: re-enable once flatbuffers is added as a subdirectory. +# set(FLATBUFFERS_BUILD_FLATC OFF) +# set(FLATBUFFERS_INSTALL OFF) +# set(FLATBUFFERS_BUILD_FLATHASH OFF) +# set(FLATBUFFERS_BUILD_FLATLIB OFF) +# set(FLATBUFFERS_BUILD_TESTS OFF) diff --git a/tools/cmake/Utils.cmake b/tools/cmake/Utils.cmake index dda83f1794e..13ec09c59d9 100644 --- a/tools/cmake/Utils.cmake +++ b/tools/cmake/Utils.cmake @@ -29,7 +29,6 @@ function(executorch_print_configuration_summary) message(STATUS " CMAKE_TOOLCHAIN_FILE : ${CMAKE_TOOLCHAIN_FILE}") message(STATUS " BUCK2 : ${BUCK2}") message(STATUS " PYTHON_EXECUTABLE : ${PYTHON_EXECUTABLE}") - message(STATUS " FLATC_EXECUTABLE : ${FLATC_EXECUTABLE}") message(STATUS " EXECUTORCH_ENABLE_PROGRAM_VERIFICATION : " "${EXECUTORCH_ENABLE_PROGRAM_VERIFICATION}" ) @@ -77,10 +76,6 @@ function(executorch_print_configuration_summary) message(STATUS " EXECUTORCH_BUILD_EXTENSION_TRAINING : " "${EXECUTORCH_BUILD_EXTENSION_TRAINING}" ) - message( - STATUS - " EXECUTORCH_BUILD_FLATC : ${EXECUTORCH_BUILD_FLATC}" - ) message( STATUS " EXECUTORCH_BUILD_GFLAGS : ${EXECUTORCH_BUILD_GFLAGS}" diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 5fbb47b1396..ddcab5d367b 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -12,7 +12,12 @@ else() set(_is_build_type_debug ON) endif() -# MARK: - Definitions +# MARK: - Overridable Options define_overridable_option(EXECUTORCH_ENABLE_LOGGING "Build with ET_LOG_ENABLED" BOOL ${_is_build_type_debug}) define_overridable_option(EXECUTORCH_BUILD_COREML "Build the Core ML backend" BOOL OFF) +define_overridable_option( + EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT + "Exir lets users set the alignment of tensor data embedded in the flatbuffer, and some users need an alignment larger than the default, which is typically 32." + STRING 1024 +)