Remove FLATC_EXECUTABLE and the ability to bring your own flatc (#10781)

### Summary

* Remove the ability to set flatc using FLATC_EXECUTABLE and always
build it from source
  * Also remove, the now redundant, `EXECUTORCH_BUILD_FLATC`
* Move the flatc configuration into `third-party/CMakeLists.txt` to keep
things organized
* Allow `FLATBUFFERS_MAX_ALIGNMENT` to be configurable

### Test plan

CI

cc @larryliu0820

Author: jathu

CMakeLists.txt

@@ -59,6 +59,7 @@ print_configured_options()
 
 include(tools/cmake/Utils.cmake)
 include(CMakeDependentOption)
+include(ExternalProject)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -260,6 +261,8 @@ cmake_dependent_option(
   "NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF
 )
 
+add_subdirectory(third-party)
+
 if(EXECUTORCH_BUILD_EXTENSION_TRAINING)
   set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON)
   set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON)
@@ -454,81 +457,6 @@ if(EXECUTORCH_USE_CPP_CODE_COVERAGE)
   endif()
 endif()
 
-#
-# flatc: Flatbuffer commandline tool to generate .h files from .fbs files
-#
-cmake_dependent_option(
-  EXECUTORCH_BUILD_FLATC "Build the flatc executable." ON
-  "NOT FLATC_EXECUTABLE" OFF
-)
-
-set(FLATBUFFERS_BUILD_FLATC OFF CACHE BOOL "")
-set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "")
-set(FLATBUFFERS_BUILD_FLATLIB OFF CACHE BOOL "")
-set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "")
-set(FLATBUFFERS_INSTALL OFF CACHE BOOL "")
-# exir lets users set the alignment of tensor data embedded in the flatbuffer,
-# and some users need an alignment larger than the default, which is typically
-# 32.
-set(FLATBUFFERS_MAX_ALIGNMENT 1024)
-
-if(EXECUTORCH_BUILD_FLATC)
-  if(FLATC_EXECUTABLE)
-    # We could ignore this, but it could lead to confusion about which `flatc`
-    # is actually being used.
-    message(
-      FATAL_ERROR "May not set both EXECUTORCH_BUILD_FLATC and FLATC_EXECUTABLE"
-    )
-  endif()
-
-  # Build flatc for the *host* to generate files as part of the build step.
-  include(ExternalProject)
-  ExternalProject_Add(
-    flatbuffers
-    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/third-party/flatbuffers
-    BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/flatbuffers
-    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/flatbuffers
-    CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON
-               -DFLATBUFFERS_BUILD_FLATHASH=${FLATBUFFERS_BUILD_FLATHASH}
-               -DFLATBUFFERS_BUILD_FLATLIB=${FLATBUFFERS_BUILD_FLATLIB}
-               -DFLATBUFFERS_BUILD_TESTS=${FLATBUFFERS_BUILD_TESTS}
-               -DFLATBUFFERS_INSTALL=${FLATBUFFERS_INSTALL}
-               -DCMAKE_CXX_FLAGS="-DFLATBUFFERS_MAX_ALIGNMENT=${FLATBUFFERS_MAX_ALIGNMENT}"
-               # If building for iOS, "unset" these variables to rely on the host (macOS) defaults.
-               $<$<AND:$<BOOL:${CMAKE_TOOLCHAIN_IOS}>,$<BOOL:$<FILTER:${PLATFORM},EXCLUDE,^MAC>>>:-DCMAKE_OSX_SYSROOT=>
-    INSTALL_COMMAND ""
-    BUILD_BYPRODUCTS <BINARY_DIR>/flatc
-  )
-  ExternalProject_Get_Property(flatbuffers BINARY_DIR)
-  if(WIN32)
-    # flatbuffers does not use CMAKE_BUILD_TYPE. Internally, the build forces Release
-    # config, but from CMake's perspective the build type is always Debug.
-    set(FLATC_EXECUTABLE ${BINARY_DIR}/$<CONFIG>/flatc.exe)
-  elseif(CMAKE_GENERATOR STREQUAL "Xcode")
-    set(FLATC_EXECUTABLE ${BINARY_DIR}/$<CONFIG>/flatc)
-  else()
-    set(FLATC_EXECUTABLE ${BINARY_DIR}/flatc)
-  endif()
-  set(FLATC_EXECUTABLE_BUILT_FROM_SOURCE YES)
-endif()
-
-if(NOT FLATC_EXECUTABLE)
-  message(
-    WARNING "FLATC_EXECUTABLE not specified, looking for flatc"
-  )
-  find_program(FLATC_EXECUTABLE flatc)
-
-  if(NOT FLATC_EXECUTABLE)
-    message(FATAL_ERROR "FLATC_EXECUTABLE must be set when EXECUTORCH_BUILD_FLATC is disabled.")
-  endif()
-endif()
-
-add_executable(flatc IMPORTED GLOBAL)
-set_target_properties(flatc PROPERTIES IMPORTED_LOCATION ${FLATC_EXECUTABLE})
-
-if(FLATC_EXECUTABLE_BUILT_FROM_SOURCE)
-  add_dependencies(flatc flatbuffers)
-endif()
 
 #
 # program_schema: Generated .h files from schema/*.fbs inputs

backends/apple/mps/CMakeLists.txt

@@ -42,7 +42,7 @@ endforeach()
 add_custom_command(
   OUTPUT ${_mps_schema__outputs}
   COMMAND
-    ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
+    flatc --cpp --cpp-std c++11 --scoped-enums -o
     "${_mps_schema__include_dir}/executorch/backends/apple/mps"
     ${_mps_schema__srcs}
   WORKING_DIRECTORY ${EXECUTORCH_ROOT}

backends/qualcomm/CMakeLists.txt

@@ -43,7 +43,7 @@ set(qcir_schema_include_dir ${CMAKE_CURRENT_LIST_DIR}/aot/ir)
 set(qcir_schema_output ${qcir_schema_include_dir}/qcir_generated.h)
 add_custom_command(
   OUTPUT qcir_schema_output
-  COMMAND ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
+  COMMAND flatc --cpp --cpp-std c++11 --scoped-enums -o
           ${qcir_schema_include_dir} ${qcir_schema_include_dir}/qcir.fbs
   DEPENDS flatc
   COMMENT "Generating qualcomm ir schema headers"
@@ -94,7 +94,7 @@ endforeach()
 add_custom_command(
   OUTPUT ${_qnn_schema__outputs}
   COMMAND
-    ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
+    flatc --cpp --cpp-std c++11 --scoped-enums -o
     "${_qnn_schema__include_dir}/executorch/backends/qualcomm"
     ${_qnn_schema__srcs}
   WORKING_DIRECTORY ${EXECUTORCH_SOURCE_DIR}

backends/vulkan/CMakeLists.txt

@@ -85,7 +85,7 @@ set(GENERATED_HEADER
 add_custom_command(
   OUTPUT ${GENERATED_HEADER}
   COMMAND
-    ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
+    flatc --cpp --cpp-std c++11 --scoped-enums -o
     "${SCHEMA_INCLUDE_DIR}/executorch/backends/vulkan/serialization/" ${_vulkan_schema__srcs}
   WORKING_DIRECTORY ${EXECUTORCH_ROOT}
   DEPENDS flatc

backends/xnnpack/CMakeLists.txt

@@ -92,7 +92,7 @@ endif()
 add_custom_command(
   OUTPUT ${_xnnpack_schema__outputs}
   COMMAND
-    ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --scoped-enums -o
+    flatc --cpp --cpp-std c++11 --scoped-enums -o
     "${_xnnpack_schema__include_dir}/executorch/backends/xnnpack/serialization"
     ${_xnnpack_schema__srcs}
   COMMAND ${MV_COMMAND}

devtools/CMakeLists.txt

@@ -199,7 +199,7 @@ target_link_libraries(
 add_custom_command(
   OUTPUT ${_bundled_program_schema__outputs}
   COMMAND
-    ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o
+    flatc --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o
     "${_bundled_schema__include_dir}/executorch/devtools/bundled_program/schema"
     ${_bundled_program_schema__srcs}
   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/devtools

extension/flat_tensor/serialize/CMakeLists.txt

@@ -30,7 +30,7 @@ function(generate_flat_tensor_schema _schema_srcs _schema_name)
   add_custom_command(
     OUTPUT ${_schema_outputs}
     COMMAND
-      ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o
+      flatc --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o
       "${_flat_tensor_schema__output_dir}" ${_schema_srcs}
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     DEPENDS flatc ${_schema_srcs}
@@ -45,7 +45,7 @@ function(generate_flat_tensor_schema _schema_srcs _schema_name)
   # and some users need an alignment larger than the default, which is typically
   # 32.
   target_compile_definitions(
-    ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${FLATBUFFERS_MAX_ALIGNMENT}
+    ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT}
   )
 
   target_include_directories(

schema/CMakeLists.txt

@@ -30,7 +30,7 @@ function(generate_program_schema _schema_srcs _schema_name)
   add_custom_command(
     OUTPUT ${_schema_outputs}
     COMMAND
-      ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o
+      flatc --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o
       "${_program_schema__output_dir}" ${_schema_srcs}
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     DEPENDS flatc ${_schema_srcs}
@@ -45,7 +45,7 @@ function(generate_program_schema _schema_srcs _schema_name)
   # and some users need an alignment larger than the default, which is typically
   # 32.
   target_compile_definitions(
-    ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${FLATBUFFERS_MAX_ALIGNMENT}
+    ${_schema_name} INTERFACE FLATBUFFERS_MAX_ALIGNMENT=${EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT}
   )
 
   target_include_directories(

setup.py

@@ -156,10 +156,6 @@ def llama_custom_ops(cls) -> bool:
             "EXECUTORCH_BUILD_KERNELS_CUSTOM_AOT", default=True
         )
 
-    @classmethod
-    def flatc(cls) -> bool:
-        return cls._is_cmake_arg_enabled("EXECUTORCH_BUILD_FLATC", default=True)
-
 
 class Version:
     """Static strings that describe the version of the pip package."""
@@ -833,23 +829,20 @@ def run(self):
 
 def get_ext_modules() -> List[Extension]:
     """Returns the set of extension modules to build."""
-    ext_modules = []
-    if ShouldBuild.flatc():
-        ext_modules.extend(
-            [
-                BuiltFile(
-                    src_dir="%CMAKE_CACHE_DIR%/third-party/flatbuffers/%BUILD_TYPE%/",
-                    src_name="flatc",
-                    dst="executorch/data/bin/",
-                    is_executable=True,
-                ),
-                BuiltFile(
-                    src_dir="tools/wheel",
-                    src_name="pip_data_bin_init.py.in",
-                    dst="executorch/data/bin/__init__.py",
-                ),
-            ]
-        )
+
+    ext_modules = [
+        BuiltFile(
+            src_dir="%CMAKE_CACHE_DIR%/third-party/flatbuffers_external_project",
+            src_name="flatc",
+            dst="executorch/data/bin/",
+            is_executable=True,
+        ),
+        BuiltFile(
+            src_dir="tools/wheel",
+            src_name="pip_data_bin_init.py.in",
+            dst="executorch/data/bin/__init__.py",
+        ),
+    ]
 
     if ShouldBuild.pybindings():
         ext_modules.append(

third-party/CMakeLists.txt

@@ -0,0 +1,49 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# MARK: - flatbuffers
+
+# We use ExternalProject to build flatc from source to force it target the host.
+# Otherwise, flatc will target the project's toolchain (i.e. iOS, or Android).
+ExternalProject_Add(
+  flatbuffers_external_project
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_external_project
+  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_external_project
+  SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatbuffers
+  # Always use Make to avoid needing to codesign flatc if the project is using Xcode.
+  CMAKE_GENERATOR "Unix Makefiles"
+  CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON
+             -DFLATBUFFERS_INSTALL=OFF
+             -DFLATBUFFERS_BUILD_FLATHASH=OFF
+             -DFLATBUFFERS_BUILD_FLATLIB=OFF
+             -DFLATBUFFERS_BUILD_TESTS=OFF
+             -DCMAKE_CXX_FLAGS="-DFLATBUFFERS_MAX_ALIGNMENT=${EXECUTORCH_FLATBUFFERS_MAX_ALIGNMENT}"
+             # Unset the toolchain to build for the host instead of the toolchain set for the project.
+             -DCMAKE_TOOLCHAIN_FILE=
+             # If building for iOS, "unset" these variables to rely on the host (macOS) defaults.
+             $<$<AND:$<BOOL:${APPLE}>,$<BOOL:$<FILTER:${PLATFORM},EXCLUDE,^MAC>>>:-DCMAKE_OSX_SYSROOT=>
+             -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${CMAKE_OSX_DEPLOYMENT_TARGET}
+  INSTALL_COMMAND ""
+  BUILD_BYPRODUCTS <BINARY_DIR>/flatc
+)
+ExternalProject_Get_Property(flatbuffers_external_project BINARY_DIR)
+add_executable(flatc IMPORTED GLOBAL)
+add_dependencies(flatc flatbuffers_external_project)
+if(WIN32)
+  # flatbuffers does not use CMAKE_BUILD_TYPE. Internally, the build forces Release
+  # config, but from CMake's perspective the build type is always Debug.
+  set_target_properties(flatc PROPERTIES IMPORTED_LOCATION ${BINARY_DIR}/$<CONFIG>/flatc.exe)
+else()
+  set_target_properties(flatc PROPERTIES IMPORTED_LOCATION ${BINARY_DIR}/flatc)
+endif()
+
+# TODO: re-enable once flatbuffers is added as a subdirectory.
+# set(FLATBUFFERS_BUILD_FLATC OFF)
+# set(FLATBUFFERS_INSTALL OFF)
+# set(FLATBUFFERS_BUILD_FLATHASH OFF)
+# set(FLATBUFFERS_BUILD_FLATLIB OFF)
+# set(FLATBUFFERS_BUILD_TESTS OFF)