Make symlinks so that RBE will work for Xcode toolchains (#417)

Author: dnfield

build/config/ios/ios_sdk.gni

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("//build/toolchain/goma.gni")
+
 declare_args() {
   # SDK path to use. When empty this will use the default SDK based on the
   # value of use_ios_simulator.
@@ -27,15 +29,29 @@ declare_args() {
 }
 
 if (ios_sdk_path == "") {
+  ios_sdk_args = []
+  if (use_goma) {
+    ios_sdk_args += [
+      "--symlink",
+      rebase_path(root_gen_dir),
+    ]
+  }
   if (!use_ios_simulator && ios_device_sdk_path == "") {
+    ios_sdk_args += [
+      "--sdk",
+      "iphoneos",
+    ]
     _ios_device_sdk_result =
-        exec_script("ios_sdk.py", [ "iphoneos" ], "list lines")
+        exec_script("ios_sdk.py", ios_sdk_args, "list lines")
     ios_device_sdk_path = _ios_device_sdk_result[0]
   }
 
   if (use_ios_simulator && ios_simulator_sdk_path == "") {
-    _ios_sim_sdk_result =
-        exec_script("ios_sdk.py", [ "iphonesimulator" ], "list lines")
+    ios_sdk_args += [
+      "--sdk",
+      "iphonesimulator",
+    ]
+    _ios_sim_sdk_result = exec_script("ios_sdk.py", ios_sdk_args, "list lines")
     ios_simulator_sdk_path = _ios_sim_sdk_result[0]
   }
 

build/config/ios/ios_sdk.py

@@ -2,22 +2,45 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import argparse
+import errno
+import os
 import subprocess
 import sys
 
+sys.path.insert(1, '../../build')
+from pyutil.file_util import symlink
+
 # This script returns the path to the SDK of the given type. Pass the type of
-# SDK you want, which is typically "iphone" or "iphonesimulator".
+# SDK you want, which is typically 'iphone' or 'iphonesimulator'.
+
+def main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--symlink',
+                      help='Whether to create a symlink in the buildroot to the SDK.')
+  parser.add_argument('--sdk',
+                      choices=['iphone', 'iphonesimulator'],
+                      help='Which SDK to find.')
+  args = parser.parse_args()
+
+  command =  [
+    'xcodebuild',
+    '-version',
+    '-sdk',
+    args.sdk,
+    'Path'
+  ]
 
-if len(sys.argv) != 2:
-  print "Takes one arg (SDK to find)"
-  sys.exit(1)
+  sdk_output = subprocess.check_output(command).strip()
+  if args.symlink:
+    symlink_target = os.path.join(args.symlink, os.path.basename(sdk_output))
+    symlink(sdk_output, symlink_target)
+    sdk_output = symlink_target
 
-command =  [
-  'xcodebuild',
-  '-version',
-  '-sdk',
-  sys.argv[1],
-  'Path'
-]
+  print(sdk_output)
+  return 0
 
-print subprocess.check_output(command).strip()
+if __name__ == '__main__':
+  if sys.platform != 'darwin':
+    raise Exception('This script only runs on Mac')
+  sys.exit(main(sys.argv))

build/config/mac/mac_sdk.gni

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("//build/toolchain/goma.gni")
+
 declare_args() {
   # Minimum supported version of the Mac SDK.
   mac_sdk_min = "10.12"
@@ -16,24 +18,24 @@ declare_args() {
   mac_sdk_path = ""
 }
 
-find_sdk_args = [
-  "--print_sdk_path",
-  mac_sdk_min,
-]
-
-# The tool will print the SDK path on the first line, and the version on the
-# second line.
-find_sdk_lines =
-    exec_script("//build/mac/find_sdk.py", find_sdk_args, "list lines")
+if (mac_sdk_path == "") {
+  find_sdk_args = []
+  if (use_goma) {
+    # RBE has a restriction that paths cannot come from outside the build root.
+    find_sdk_args += [
+      "--symlink",
+      rebase_path(root_gen_dir),
+    ]
+  }
+  find_sdk_args += [
+    "--print_sdk_path",
+    mac_sdk_min,
+  ]
 
-mac_sdk_version = find_sdk_lines[1]
+  # The tool will print the SDK path on the first line, and the version on the
+  # second line.
+  find_sdk_lines =
+      exec_script("//build/mac/find_sdk.py", find_sdk_args, "list lines")
 
-if (mac_sdk_path == "") {
-  # TODO(brettw) http://crbug.com/335325  when everybody moves to XCode 5 we
-  # can remove the --print_sdk_path argument to find_sdk and instead just use
-  # the following two lines to get the path. Although it looks longer here, it
-  # saves forking a process in find_sdk.py so will be faster.
-  #mac_sdk_root = "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX"
-  #mac_sdk_path = mac_sdk_root + mac_sdk_version + ".sdk"
   mac_sdk_path = find_sdk_lines[0]
 }

build/config/mac/xcode_toolchain.py

@@ -3,15 +3,31 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import argparse
 import sys
 import os
 import subprocess
 
-def Main():
-  path = subprocess.check_output(['/usr/bin/env', 'xcode-select', '-p']).strip();
+sys.path.insert(1, '../../build')
+from pyutil.file_util import symlink
+
+
+def main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--symlink',
+                      help='Whether to create a symlink in the buildroot to the SDK.')
+  args = parser.parse_args()
+
+  path = subprocess.check_output(['/usr/bin/env', 'xcode-select', '-p']).strip()
   path = os.path.join(path, "Toolchains", "XcodeDefault.xctoolchain")
   assert os.path.exists(path)
-  print path
+
+  if args.symlink:
+    symlink_target = os.path.join(args.symlink, os.path.basename(path))
+    symlink(path, symlink_target)
+    path = symlink_target
+
+  print(path)
 
 if __name__ == '__main__':
-  sys.exit(Main())
+  sys.exit(main(sys.argv))

build/config/sysroot.gni

@@ -5,6 +5,8 @@
 # This header file defines the "sysroot" variable which is the absolute path
 # of the sysroot. If no sysroot applies, the variable will be an empty string.
 
+import("//build/toolchain/goma.gni")
+
 declare_args() {
   # The absolute path of the sysroot that is applied when compiling using
   # the target toolchain.
@@ -50,8 +52,15 @@ if (current_toolchain == default_toolchain && target_sysroot != "") {
 }
 
 if ((is_mac || is_ios) && xcode_toolchain == "") {
+  xcode_toolchain_args = []
+  if (use_goma) {
+    xcode_toolchain_args += [
+      "--symlink",
+      rebase_path(root_gen_dir),
+    ]
+  }
   xcode_toolchain = exec_script("//build/config/mac/xcode_toolchain.py",
-                                [],
+                                xcode_toolchain_args,
                                 "trim string",
                                 [])
 }

build/mac/find_sdk.py

@@ -15,9 +15,11 @@
 import subprocess
 import sys
 
-
 from optparse import OptionParser
 
+sys.path.insert(1, '../../build')
+from pyutil.file_util import symlink
+
 
 def parse_version(version_str):
   """'10.6' => [10, 6]"""
@@ -35,6 +37,9 @@ def main():
   parser.add_option("--print_sdk_path",
                     action="store_true", dest="print_sdk_path", default=False,
                     help="Additionaly print the path the SDK (appears first).")
+  parser.add_option("--symlink",
+                    action="store", type="string", dest="symlink", default="",
+                    help="Whether to create a symlink in the buildroot to the SDK.")
   (options, args) = parser.parse_args()
   min_sdk_version = args[0]
 
@@ -43,8 +48,7 @@ def main():
                          stderr=subprocess.STDOUT)
   out, err = job.communicate()
   if job.returncode != 0:
-    print >> sys.stderr, out
-    print >> sys.stderr, err
+    sys.stderr.writelines([out, err])
     raise Exception(('Error %d running xcode-select, you might have to run '
       '|sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer| '
       'if you are using Xcode 4.') % job.returncode)
@@ -64,27 +68,33 @@ def main():
   best_sdk = sorted(sdks, key=parse_version)[0]
 
   if options.verify and best_sdk != min_sdk_version and not options.sdk_path:
-    print >> sys.stderr, ''
-    print >> sys.stderr, '                                           vvvvvvv'
-    print >> sys.stderr, ''
-    print >> sys.stderr, \
-        'This build requires the %s SDK, but it was not found on your system.' \
-        % min_sdk_version
-    print >> sys.stderr, \
-        'Either install it, or explicitly set mac_sdk in your GYP_DEFINES.'
-    print >> sys.stderr, ''
-    print >> sys.stderr, '                                           ^^^^^^^'
-    print >> sys.stderr, ''
+    sys.stderr.writelines([
+      '',
+      '                                           vvvvvvv',
+      '',
+      'This build requires the %s SDK, but it was not found on your system.' \
+        % min_sdk_version,
+      'Either install it, or explicitly set mac_sdk in your gn args.',
+      '',
+      '                                           ^^^^^^^',
+      ''])
     return min_sdk_version
 
-  if options.print_sdk_path:
-    print subprocess.check_output(['xcodebuild', '-version', '-sdk',
-                                   'macosx' + best_sdk, 'Path']).strip()
+  if options.symlink or options.print_sdk_path:
+    sdk_output = subprocess.check_output(['xcodebuild', '-version', '-sdk',
+                                          'macosx' + best_sdk, 'Path']).strip()
+    if options.symlink:
+      symlink_target = os.path.join(options.symlink, os.path.basename(sdk_output))
+      symlink(sdk_output, symlink_target)
+      sdk_output = symlink_target
+
+    if options.print_sdk_path:
+      print(sdk_output)
 
   return best_sdk
 
 
 if __name__ == '__main__':
   if sys.platform != 'darwin':
     raise Exception("This script only runs on Mac")
-  print main()
+  print(main())

build/pyutil/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.

build/pyutil/file_util.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import errno
+import os
+
+"""Creates a directory and its parents (i.e. `mkdir -p`).
+
+If the directory already exists, does nothing."""
+def mkdir_p(path):
+  try:
+    os.makedirs(path)
+  except OSError as exc:
+    if exc.errno == errno.EEXIST and os.path.isdir(path):
+      pass
+    else:
+      raise
+
+
+"""Creates or ovewrites a symlink from `link` to `target`."""
+def symlink(target, link):
+  mkdir_p(os.path.dirname(link))
+  tmp_link = link + '.tmp'
+  try:
+    os.remove(tmp_link)
+  except OSError:
+    pass
+  os.symlink(target, tmp_link)
+  os.rename(tmp_link, link)