From 9f12cce2d0cda533840aab9bd4490d750cebee22 Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 13 Aug 2020 14:01:34 -0700 Subject: [PATCH 1/6] Use Ninja to build extension --- build_tools/setup_helpers/extension.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index 4223b398d2..2b4e1b3272 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -89,14 +89,14 @@ def _build_third_party(debug): build_dir.mkdir(exist_ok=True) build_env = os.environ.copy() config = 'Debug' if debug else 'Release' + extra_args = [ + '-GNinja', + ] if platform.system() == 'Windows': - extra_args = [ - '-GNinja', - ] build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: - extra_args = ['-DCMAKE_CXX_FLAGS=-fPIC'] + extra_args += ['-DCMAKE_CXX_FLAGS=-fPIC'] subprocess.run( args=[ 'cmake', From 850446278534e0e4ca9ca546ce7f3f2e33a4629e Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 13 Aug 2020 14:28:36 -0700 Subject: [PATCH 2/6] Checkpoint --- build_tools/setup_helpers/extension.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index 2b4e1b3272..01f21bb8bf 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -96,12 +96,13 @@ def _build_third_party(debug): build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: - extra_args += ['-DCMAKE_CXX_FLAGS=-fPIC'] + extra_args += ['-DCMAKE_CXX_FLAGS=-fPIC -D_GLIBCXX_USE_CXX11_ABI=0'] subprocess.run( args=[ 'cmake', '-DBUILD_SHARED_LIBS=OFF', '-DRE2_BUILD_TESTING=OFF', + # '-D_GLIBCXX_USE_CXX11_ABI=0', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', f'-DCMAKE_INSTALL_PREFIX={_TP_INSTALL_DIR}', f'-DCMAKE_BUILD_TYPE={config}', @@ -127,14 +128,15 @@ def _build_sentence_piece(debug): build_dir.mkdir(exist_ok=True) build_env = os.environ.copy() config = 'Debug' if debug else 'Release' + extra_args = ['-GNinja'] if platform.system() == 'Windows': - extra_args = ['-GNinja'] build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: extra_args = [] subprocess.run( args=['cmake', f'-DSPM_ENABLE_SHARED=OFF', f'-DCMAKE_INSTALL_PREFIX={_TP_INSTALL_DIR}', + '-DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0', f'-DCMAKE_BUILD_TYPE={config}'] + extra_args + ['..'], cwd=str(build_dir), check=True, From ae95fe44f7875665f7c2f9e20d4103781b8bd08d Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 13 Aug 2020 14:56:41 -0700 Subject: [PATCH 3/6] Checkpoint --- build_tools/setup_helpers/extension.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index 01f21bb8bf..e262d59e02 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -83,6 +83,13 @@ def _get_libraries(): 'double-conversion' ] +def _get_cxx11_abi(): + try: + import torch + value = int(torch._C._GLIBCXX_USE_CXX11_ABI) + except ImportError: + value = 0 + return '-D_GLIBCXX_USE_CXX11_ABI=' + str(value) def _build_third_party(debug): build_dir = _TP_BASE_DIR / 'build' @@ -96,13 +103,12 @@ def _build_third_party(debug): build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: - extra_args += ['-DCMAKE_CXX_FLAGS=-fPIC -D_GLIBCXX_USE_CXX11_ABI=0'] + extra_args += ['-DCMAKE_CXX_FLAGS=-fPIC ' + _get_cxx11_abi()] subprocess.run( args=[ 'cmake', '-DBUILD_SHARED_LIBS=OFF', '-DRE2_BUILD_TESTING=OFF', - # '-D_GLIBCXX_USE_CXX11_ABI=0', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', f'-DCMAKE_INSTALL_PREFIX={_TP_INSTALL_DIR}', f'-DCMAKE_BUILD_TYPE={config}', @@ -136,7 +142,7 @@ def _build_sentence_piece(debug): extra_args = [] subprocess.run( args=['cmake', f'-DSPM_ENABLE_SHARED=OFF', f'-DCMAKE_INSTALL_PREFIX={_TP_INSTALL_DIR}', - '-DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0', + '-DCMAKE_CXX_FLAGS=' + _get_cxx11_abi(), f'-DCMAKE_BUILD_TYPE={config}'] + extra_args + ['..'], cwd=str(build_dir), check=True, From d04f0a213f10f6dfed91f867a13242a817cfcf1a Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 13 Aug 2020 15:00:16 -0700 Subject: [PATCH 4/6] Checkpoint --- build_tools/setup_helpers/extension.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index e262d59e02..e9fa5c67bf 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -96,10 +96,10 @@ def _build_third_party(debug): build_dir.mkdir(exist_ok=True) build_env = os.environ.copy() config = 'Debug' if debug else 'Release' - extra_args = [ - '-GNinja', - ] if platform.system() == 'Windows': + extra_args = [ + '-GNinja', + ] build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: @@ -134,8 +134,8 @@ def _build_sentence_piece(debug): build_dir.mkdir(exist_ok=True) build_env = os.environ.copy() config = 'Debug' if debug else 'Release' - extra_args = ['-GNinja'] if platform.system() == 'Windows': + extra_args = ['-GNinja'] build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: From 64df44a5b638c5abe25105135e485817d9aec6ca Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 13 Aug 2020 15:01:12 -0700 Subject: [PATCH 5/6] Checkpoint --- build_tools/setup_helpers/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index e9fa5c67bf..64279516db 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -103,7 +103,7 @@ def _build_third_party(debug): build_env.setdefault('CC', 'cl') build_env.setdefault('CXX', 'cl') else: - extra_args += ['-DCMAKE_CXX_FLAGS=-fPIC ' + _get_cxx11_abi()] + extra_args = ['-DCMAKE_CXX_FLAGS=-fPIC ' + _get_cxx11_abi()] subprocess.run( args=[ 'cmake', From bf9c8979ab6e41e0116b702ed2563724197d421c Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 13 Aug 2020 15:15:53 -0700 Subject: [PATCH 6/6] Checkpoint --- build_tools/setup_helpers/extension.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index 64279516db..f3a2097313 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -83,6 +83,7 @@ def _get_libraries(): 'double-conversion' ] + def _get_cxx11_abi(): try: import torch @@ -91,6 +92,7 @@ def _get_cxx11_abi(): value = 0 return '-D_GLIBCXX_USE_CXX11_ABI=' + str(value) + def _build_third_party(debug): build_dir = _TP_BASE_DIR / 'build' build_dir.mkdir(exist_ok=True) @@ -120,7 +122,7 @@ def _build_third_party(debug): print('*** Command list Thirdparty ***') with open(build_dir / 'compile_commands.json', 'r') as fileobj: print(fileobj.read()) - print(f'running cmake --build', flush=True) + print('running cmake --build', flush=True) subprocess.run( args=['cmake', '--build', '.', '--target', 'install', '--config', config], cwd=str(build_dir), @@ -141,8 +143,8 @@ def _build_sentence_piece(debug): else: extra_args = [] subprocess.run( - args=['cmake', f'-DSPM_ENABLE_SHARED=OFF', f'-DCMAKE_INSTALL_PREFIX={_TP_INSTALL_DIR}', - '-DCMAKE_CXX_FLAGS=' + _get_cxx11_abi(), + args=['cmake', '-DSPM_ENABLE_SHARED=OFF', f'-DCMAKE_INSTALL_PREFIX={_TP_INSTALL_DIR}', + '-DCMAKE_CXX_FLAGS=' + _get_cxx11_abi(), f'-DCMAKE_BUILD_TYPE={config}'] + extra_args + ['..'], cwd=str(build_dir), check=True,