Use common include directory for all ports (#9983)

We already use a common "lib" directory to install all the libs.  This
change simply installs headers in the same common "sysroot".

This makes the ports install more like normal "make install" scripts
would.

Author: sbc100

ChangeLog.md

@@ -17,12 +17,17 @@ See docs/process.md for how version tagging works.
 
 Current Trunk
 -------------
-- Removed `timestamp` field from mouse, wheel, devicemotion and deviceorientation
-  events. The presence of a `timestamp` on these events was slightly arbitrary,
-  and populating this field caused a small profileable overhead that all users
-  might not care about. It is easy to get a timestamp of an event by calling
-  `emscripten_get_now()` or `emscripten_performance_now()` inside the event
-  handler function of any event.
+- All ports now install their headers into a shared directory under
+  `EM_CACHE`.  This should not really be a user visible change although one
+  side effect is that once a give ports is built its headers are then
+  universally accessible, just like the library is universally available as
+  `-l<name>`.
+- Removed `timestamp` field from mouse, wheel, devicemotion and
+  deviceorientation events. The presence of a `timestamp` on these events was
+  slightly arbitrary, and populating this field caused a small profileable
+  overhead that all users might not care about. It is easy to get a timestamp of
+  an event by calling `emscripten_get_now()` or `emscripten_performance_now()`
+  inside the event handler function of any event.
 - Add fine-grained options for specific legacy browser support,
   `MIN_FIREFOX_VERSION`, `MIN_SAFARI_VERSION`, `MIN_IE_VERSION`,
   `MIN_EDGE_VERSION`, `MIN_CHROME_VERSION`. The existing `LEGACY_VM_SUPPORT`

embuilder.py

@@ -18,9 +18,7 @@
 from tools import shared
 from tools.system_libs import Library
 
-C_BARE = '''
-        int main() {}
-      '''
+C_BARE = 'int main() {}'
 
 SYSTEM_LIBRARIES = Library.get_all_variations()
 SYSTEM_TASKS = list(SYSTEM_LIBRARIES.keys())
@@ -60,35 +58,23 @@
 force = False
 
 
-def print_help():
+def get_usage():
   all_tasks = SYSTEM_TASKS + USER_TASKS
   all_tasks.sort()
-  print('''
-Emscripten System Builder Tool
-==============================
+  return '''embuilder.py [flags]... OPERATION TARGET [TARGET]...
 
 You can use this tool to manually build parts of the emscripten system
 environment. In general emcc will build them automatically on demand, so
 you do not strictly need to use this tool, but it gives you more control
 over the process (in particular, if emcc does this automatically, and you
 are running multiple build commands in parallel, confusion can occur).
 
-Usage:
-
-  embuilder.py OPERATION TASK1 [TASK2..] [--pic] [--lto]
-
 Available operations and tasks:
 
   build %s
 
 Issuing 'embuilder.py build ALL' causes each task to be built.
 
-Flags:
-
-  --force  Force rebuild of target (by removing it first)
-  --lto    Build bitcode files, for LTO with the LLVM wasm backend
-  --pic    Build as position independent code (used by MAIN_MODULE/SIDE_MODULE)
-
 It is also possible to build native_optimizer manually by using CMake. To
 do that, run
 
@@ -97,7 +83,7 @@ def print_help():
    3. make (or mingw32-make/vcbuild/msbuild on Windows)
 
 and set up the location to the native optimizer in ~/.emscripten
-''' % '\n        '.join(all_tasks))
+''' % '\n        '.join(all_tasks)
 
 
 def build(src, result_libs, args=[]):
@@ -125,28 +111,22 @@ def build(src, result_libs, args=[]):
       shared.exit_with_error('not seeing that requested library %s has been built because file %s does not exist' % (lib, shared.Cache.get_path(lib)))
 
 
-def build_port(port_name, lib_name, params):
-  build(C_BARE, [lib_name] if lib_name else [], params)
+def build_port(port_name, lib_name, params, extra_source=''):
+  build(extra_source + '\n' + C_BARE, [lib_name] if lib_name else [], params)
 
 
 def main():
   global force
-  parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [operation] [targets ...]",
-                                   add_help=False)
-  parser.add_argument('--help', '-h', action='store_true', help='print help message')
+  parser = argparse.ArgumentParser(description=__doc__, usage=get_usage())
   parser.add_argument('--lto', action='store_true', help='build bitcode object for LTO')
   parser.add_argument('--pic', action='store_true',
                       help='build relocatable objects for suitable for dynamic linking')
   parser.add_argument('--force', action='store_true',
                       help='force rebuild of target (by removing it first)')
-  parser.add_argument('operation')
-  parser.add_argument('targets', nargs='+')
+  parser.add_argument('operation', help='currently only "build" is supported')
+  parser.add_argument('targets', nargs='+', help='see above')
   args = parser.parse_args()
 
-  if args.help:
-    print_help()
-    return 0
-
   if args.operation != 'build':
     shared.exit_with_error('unfamiliar operation: ' + args.operation)
 
@@ -215,7 +195,7 @@ def main():
     elif what == 'native_optimizer':
       build(C_BARE, ['optimizer.2.exe'], ['-O2', '-s', 'WASM=0'])
     elif what == 'icu':
-      build_port('icu', libname('libicuuc'), ['-s', 'USE_ICU=1'])
+      build_port('icu', libname('libicuuc'), ['-s', 'USE_ICU=1'], '#include "unicode/ustring.h"')
     elif what == 'zlib':
       build_port('zlib', 'libz.a', ['-s', 'USE_ZLIB=1'])
     elif what == 'bzip2':
@@ -239,9 +219,9 @@ def main():
     elif what == 'sdl2-image':
       build_port('sdl2-image', libname('libSDL2_image'), ['-s', 'USE_SDL=2', '-s', 'USE_SDL_IMAGE=2'])
     elif what == 'sdl2-image-png':
-      build_port('sdl2-image', libname('libSDL2_image'), ['-s', 'USE_SDL=2', '-s', 'USE_SDL_IMAGE=2', '-s', 'SDL2_IMAGE_FORMATS=["png"]'])
+      build_port('sdl2-image', libname('libSDL2_image_png'), ['-s', 'USE_SDL=2', '-s', 'USE_SDL_IMAGE=2', '-s', 'SDL2_IMAGE_FORMATS=["png"]'])
     elif what == 'sdl2-image-jpg':
-      build_port('sdl2-image', libname('libSDL2_image'), ['-s', 'USE_SDL=2', '-s', 'USE_SDL_IMAGE=2', '-s', 'SDL2_IMAGE_FORMATS=["jpg"]'])
+      build_port('sdl2-image', libname('libSDL2_image_jpg'), ['-s', 'USE_SDL=2', '-s', 'USE_SDL_IMAGE=2', '-s', 'SDL2_IMAGE_FORMATS=["jpg"]'])
     elif what == 'sdl2-net':
       build_port('sdl2-net', libname('libSDL2_net'), ['-s', 'USE_SDL=2', '-s', 'USE_SDL_NET=2'])
     elif what == 'sdl2-mixer':

tests/runner.py

@@ -1187,7 +1187,7 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None,
   def get_freetype_library(self):
     if '-Werror' in self.emcc_args:
       self.emcc_args.remove('-Werror')
-    return self.get_library('freetype', os.path.join('objs', '.libs', 'libfreetype.a'), configure_args=['--disable-shared'])
+    return self.get_library('freetype', os.path.join('objs', '.libs', 'libfreetype.a'), configure_args=['--disable-shared', '--without-zlib'])
 
   def get_poppler_library(self):
     # The fontconfig symbols are all missing from the poppler build

tests/test_browser.py

@@ -3221,7 +3221,9 @@ def test_cocos2d_hello(self):
     cocos2d_root = os.path.join(system_libs.Ports.get_build_dir(), 'cocos2d')
     preload_file = os.path.join(cocos2d_root, 'samples', 'HelloCpp', 'Resources') + '@'
     self.btest('cocos2d_hello.cpp', reference='cocos2d_hello.png', reference_slack=1,
-               args=['-s', 'USE_COCOS2D=3', '-s', 'ERROR_ON_UNDEFINED_SYMBOLS=0', '--std=c++11', '--preload-file', preload_file, '--use-preload-plugins'],
+               args=['-s', 'USE_COCOS2D=3', '-s', 'ERROR_ON_UNDEFINED_SYMBOLS=0', '--std=c++11',
+                     '--preload-file', preload_file, '--use-preload-plugins',
+                     '-Wno-inconsistent-missing-override'],
                message='You should see Cocos2d logo')
 
   def test_async(self):

tests/test_other.py

@@ -798,7 +798,10 @@ def verify_includes(stderr):
       includes = stderr[start:end]
       includes = [i.strip() for i in includes.splitlines()[1:-1]]
       for i in includes:
-        self.assertContained(path_from_root('system'), i)
+        if shared.Cache.dirname in i:
+          self.assertContained(shared.Cache.dirname, i)
+        else:
+          self.assertContained(path_from_root('system'), i)
 
     err = run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-v'], stderr=PIPE).stderr
     verify_includes(err)

tools/ports/boost_headers.py

@@ -25,21 +25,25 @@ def create():
 
     # includes
     source_path_include = os.path.join(ports.get_dir(), 'boost_headers', 'boost')
-    dest_path_include = os.path.join(ports.get_build_dir(), 'boost_headers', 'boost')
+    dest_path_include = os.path.join(ports.get_include_dir(), 'boost')
+    shared.try_delete(dest_path_include)
     shutil.copytree(source_path_include, dest_path_include)
 
     # write out a dummy cpp file, to create an empty library
     # this is needed as emscripted ports expect this, even if it is not used
-    open(os.path.join(ports.get_build_dir(), 'boost_headers', 'dummy.cpp'), 'w').write('static void dummy() {}')
+    dummy_file = os.path.join(ports.get_build_dir(), 'boost_headers', 'dummy.cpp')
+    shared.safe_ensure_dirs(os.path.dirname(dummy_file))
+    with open(dummy_file, 'w') as f:
+      f.write('static void dummy() {}')
 
     commands = []
     o_s = []
-    o = os.path.join(ports.get_build_dir(), 'boost_headers', 'dummy.cpp.o')
-    command = [shared.PYTHON, shared.EMCC, '-c', os.path.join(ports.get_build_dir(), 'boost_headers', 'dummy.cpp'), '-o', o]
+    obj = dummy_file + '.o'
+    command = [shared.PYTHON, shared.EMCC, '-c', dummy_file, '-o', obj]
     commands.append(command)
     ports.run_commands(commands)
     final = os.path.join(ports.get_build_dir(), 'boost_headers', libname)
-    o_s.append(o)
+    o_s.append(obj)
     ports.create_lib(final, o_s)
     return final
 
@@ -53,7 +57,7 @@ def clear(ports, shared):
 def process_args(ports, args, settings, shared):
   if settings.USE_BOOST_HEADERS == 1:
     get(ports, settings, shared)
-    args += ['-Xclang', '-isystem' + os.path.join(ports.get_build_dir(), 'boost_headers'), '-DBOOST_ALL_NO_LIB']
+    args += ['-DBOOST_ALL_NO_LIB']
   return args
 
 

tools/ports/bullet.py

@@ -26,8 +26,20 @@ def create():
 
     shutil.rmtree(dest_path, ignore_errors=True)
     shutil.copytree(source_path, dest_path)
-
     src_path = os.path.join(dest_path, 'bullet', 'src')
+    src_path = os.path.join(dest_path, 'bullet', 'src')
+
+    dest_include_path = os.path.join(ports.get_include_dir(), 'bullet')
+    for base, dirs, files in os.walk(src_path):
+      for f in files:
+        if os.path.splitext(f)[1] != '.h':
+          continue
+        fullpath = os.path.join(base, f)
+        relpath = os.path.relpath(fullpath, src_path)
+        target = os.path.join(dest_include_path, relpath)
+        shared.safe_ensure_dirs(os.path.dirname(target))
+        shutil.copyfile(fullpath, target)
+
     includes = []
     for root, dirs, files in os.walk(src_path, topdown=False):
       for dir in dirs:
@@ -47,7 +59,7 @@ def clear(ports, shared):
 def process_args(ports, args, settings, shared):
   if settings.USE_BULLET == 1:
     get(ports, settings, shared)
-    args += ['-Xclang', '-isystem' + os.path.join(shared.Cache.get_path('ports-builds'), 'bullet', 'bullet', 'src')]
+    args += ['-I' + os.path.join(ports.get_include_dir(), 'bullet')]
   return args
 
 

tools/ports/bzip2.py

@@ -43,6 +43,7 @@ def create():
 
     final = os.path.join(ports.get_build_dir(), 'bzip2', 'libbz2.a')
     ports.create_lib(final, o_s)
+    ports.install_headers(source_path)
     return final
 
   return [shared.Cache.get('libbz2.a', create, what='port')]
@@ -55,7 +56,6 @@ def clear(ports, shared):
 def process_args(ports, args, settings, shared):
   if settings.USE_BZIP2 == 1:
     get(ports, settings, shared)
-    args += ['-Xclang', '-isystem' + os.path.join(shared.Cache.get_path('ports-builds'), 'bzip2')]
   return args
 
 

tools/ports/cocos2d.py

@@ -28,9 +28,10 @@ def create():
     cocos2d_root = os.path.join(cocos2d_src, 'Cocos2d-' + TAG)
     cocos2dx_root = os.path.join(cocos2d_root, 'cocos2dx')
     cocos2dx_src = make_source_list(cocos2d_root, cocos2dx_root)
-    cocos2dx_includes = make_includes(cocos2d_root, cocos2dx_root)
+    cocos2dx_includes = make_includes(cocos2d_root)
 
     cocos2d_build = os.path.join(ports.get_build_dir(), 'cocos2d')
+    shared.try_delete(os.path.join(cocos2d_build, 'samples'))
     shutil.copytree(os.path.join(cocos2d_root, 'samples', 'Cpp'),
                     os.path.join(cocos2d_build, 'samples'))
 
@@ -41,8 +42,7 @@ def create():
       shared.safe_ensure_dirs(os.path.dirname(o))
       command = [shared.PYTHON,
                  shared.EMCC,
-                 '-c',
-                 os.path.join(cocos2dx_root, 'proj.emscripten', src),
+                 '-c', src,
                  '-Wno-overloaded-virtual',
                  '-Wno-deprecated-declarations',
                  '-D__CC_PLATFORM_FILEUTILS_CPP__',
@@ -68,6 +68,11 @@ def create():
     ports.run_commands(commands)
     final = os.path.join(cocos2d_build, libname)
     ports.create_lib(final, o_s)
+
+    for dirname in cocos2dx_includes:
+      target = os.path.join('cocos2d', os.path.relpath(dirname, cocos2d_root))
+      ports.install_header_dir(dirname, target=target)
+
     return final
 
   return [shared.Cache.get(libname, create, what='port')]
@@ -86,13 +91,8 @@ def process_dependencies(settings):
 def process_args(ports, args, settings, shared):
   if settings.USE_COCOS2D == 3:
     get(ports, settings, shared)
-    cocos2d_build = os.path.join(ports.get_dir(), 'cocos2d')
-    cocos2d_root = os.path.join(cocos2d_build, 'Cocos2d-' + TAG)
-    cocos2dx_root = os.path.join(cocos2d_root, 'cocos2dx')
-    cocos2dx_includes = make_includes(cocos2d_root, cocos2dx_root)
-    args += ['-Xclang']
-    for include in cocos2dx_includes:
-      args.append('-isystem' + include)
+    for include in make_includes(os.path.join(ports.get_include_dir(), 'cocos2d')):
+      args.append('-I' + include)
   return args
 
 
@@ -133,30 +133,30 @@ def add_makefile(makefile):
   return sources
 
 
-def make_includes(cocos2d_root, cocos2dx_root):
-  return [os.path.join(cocos2d_root, 'CocosDenshion', 'include'),
-          os.path.join(cocos2d_root, 'extensions'),
-          os.path.join(cocos2d_root, 'extensions', 'AssetsManager'),
-          os.path.join(cocos2d_root, 'extensions', 'CCArmature'),
-          os.path.join(cocos2d_root, 'extensions', 'CCBReader'),
-          os.path.join(cocos2d_root, 'extensions', 'GUI', 'CCControlExtension'),
-          os.path.join(cocos2d_root, 'extensions', 'GUI', 'CCEditBox'),
-          os.path.join(cocos2d_root, 'extensions', 'GUI', 'CCScrollView'),
-          os.path.join(cocos2d_root, 'extensions', 'network'),
-          os.path.join(cocos2d_root, 'extensions', 'Components'),
-          os.path.join(cocos2d_root, 'extensions', 'LocalStorage'),
-          os.path.join(cocos2d_root, 'extensions', 'physics_nodes'),
-          os.path.join(cocos2d_root, 'extensions', 'spine'),
-          os.path.join(cocos2d_root, 'external'),
-          os.path.join(cocos2d_root, 'external', 'chipmunk', 'include', 'chipmunk'),
-          cocos2dx_root,
-          os.path.join(cocos2dx_root, 'cocoa'),
-          os.path.join(cocos2dx_root, 'include'),
-          os.path.join(cocos2dx_root, 'kazmath', 'include'),
-          os.path.join(cocos2dx_root, 'platform'),
-          os.path.join(cocos2dx_root, 'platform', 'emscripten'),
-          os.path.join(cocos2dx_root, 'platform', 'third_party', 'linux', 'libfreetype2'),
-          os.path.join(cocos2dx_root, 'platform', 'third_party', 'common', 'etc'),
-          os.path.join(cocos2dx_root, 'platform', 'third_party', 'emscripten', 'libtiff', 'include'),
-          os.path.join(cocos2dx_root, 'platform', 'third_party', 'emscripten', 'libjpeg'),
-          os.path.join(cocos2dx_root, 'platform', 'third_party', 'emscripten', 'libwebp')]
+def make_includes(root):
+  return [os.path.join(root, 'CocosDenshion', 'include'),
+          os.path.join(root, 'extensions'),
+          os.path.join(root, 'extensions', 'AssetsManager'),
+          os.path.join(root, 'extensions', 'CCArmature'),
+          os.path.join(root, 'extensions', 'CCBReader'),
+          os.path.join(root, 'extensions', 'GUI', 'CCControlExtension'),
+          os.path.join(root, 'extensions', 'GUI', 'CCEditBox'),
+          os.path.join(root, 'extensions', 'GUI', 'CCScrollView'),
+          os.path.join(root, 'extensions', 'network'),
+          os.path.join(root, 'extensions', 'Components'),
+          os.path.join(root, 'extensions', 'LocalStorage'),
+          os.path.join(root, 'extensions', 'physics_nodes'),
+          os.path.join(root, 'extensions', 'spine'),
+          os.path.join(root, 'external'),
+          os.path.join(root, 'external', 'chipmunk', 'include', 'chipmunk'),
+          os.path.join(root, 'cocos2dx'),
+          os.path.join(root, 'cocos2dx', 'cocoa'),
+          os.path.join(root, 'cocos2dx', 'include'),
+          os.path.join(root, 'cocos2dx', 'kazmath', 'include'),
+          os.path.join(root, 'cocos2dx', 'platform'),
+          os.path.join(root, 'cocos2dx', 'platform', 'emscripten'),
+          os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'linux', 'libfreetype2'),
+          os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'common', 'etc'),
+          os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libtiff', 'include'),
+          os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libjpeg'),
+          os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libwebp')]

tools/ports/freetype.py

@@ -5,7 +5,6 @@
 
 import os
 import shutil
-from subprocess import Popen
 
 TAG = 'version_1'
 HASH = '0d0b1280ba0501ad0a23cf1daa1f86821c722218b59432734d3087a89acd22aabd5c3e5e1269700dcd41e87073046e906060f167c032eb91a3ac8c5808a02783'
@@ -103,8 +102,10 @@ def create():
     ports.run_commands(commands)
     final = os.path.join(ports.get_build_dir(), 'freetype', 'libfreetype.a')
     shared.try_delete(final)
-    Popen([shared.LLVM_AR, 'rc', final] + o_s).communicate()
-    assert os.path.exists(final)
+    shared.run_process([shared.LLVM_AR, 'rc', final] + o_s)
+
+    ports.install_header_dir(os.path.join(dest_path, 'include'),
+                             target=os.path.join('freetype2', 'freetype'))
     return final
 
   return [shared.Cache.get('libfreetype.a', create, what='port')]
@@ -117,7 +118,8 @@ def clear(ports, shared):
 def process_args(ports, args, settings, shared):
   if settings.USE_FREETYPE == 1:
     get(ports, settings, shared)
-    args += ['-Xclang', '-isystem' + os.path.join(shared.Cache.get_path('ports-builds'), 'freetype/include')]
+    args += ['-I' + os.path.join(ports.get_include_dir(), 'freetype2', 'freetype')]
+
   return args