From b40371d50104d5a9d397c017e922fe6d43247434 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 16 Aug 2024 10:36:14 -0700 Subject: [PATCH 1/6] macOS: Extract framework creation to sky_utils.py This refactors create_mac_framework.py to extract framework creation to sky_utils.py. The only other changes are minor variable renaming and extraction of functions to make the code more readable/easier to work with. The resulting zip archives have been verified to be identical before and after. This is refactoring to support embedding dSYM debug information in a follow-up patch. --- sky/tools/create_macos_framework.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sky/tools/create_macos_framework.py b/sky/tools/create_macos_framework.py index 6be5780bfd2d3..b41bafdfecee0 100755 --- a/sky/tools/create_macos_framework.py +++ b/sky/tools/create_macos_framework.py @@ -38,31 +38,28 @@ def main(): if os.path.isabs(args.x64_out_dir) else sky_utils.buildroot_relative_path(args.x64_out_dir) ) - fat_framework_bundle = os.path.join(dst, 'FlutterMacOS.framework') arm64_framework = os.path.join(arm64_out_dir, 'FlutterMacOS.framework') - x64_framework = os.path.join(x64_out_dir, 'FlutterMacOS.framework') - - arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS') - x64_dylib = os.path.join(x64_framework, 'FlutterMacOS') - if not os.path.isdir(arm64_framework): print('Cannot find macOS arm64 Framework at %s' % arm64_framework) return 1 + x64_framework = os.path.join(x64_out_dir, 'FlutterMacOS.framework') if not os.path.isdir(x64_framework): print('Cannot find macOS x64 Framework at %s' % x64_framework) return 1 + arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS') if not os.path.isfile(arm64_dylib): print('Cannot find macOS arm64 dylib at %s' % arm64_dylib) return 1 + x64_dylib = os.path.join(x64_framework, 'FlutterMacOS') if not os.path.isfile(x64_dylib): print('Cannot find macOS x64 dylib at %s' % x64_dylib) return 1 + fat_framework_bundle = os.path.join(dst, 'FlutterMacOS.framework') sky_utils.copy_tree(arm64_framework, fat_framework_bundle, symlinks=True) - regenerate_symlinks(fat_framework_bundle) fat_framework_binary = os.path.join(fat_framework_bundle, 'Versions', 'A', 'FlutterMacOS') From e28e232e9d5948de8b6e7fd850ee00149970776e Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 16 Aug 2024 11:35:52 -0700 Subject: [PATCH 2/6] Rename fat_framework_bundle to fat_framework --- sky/tools/create_macos_framework.py | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sky/tools/create_macos_framework.py b/sky/tools/create_macos_framework.py index b41bafdfecee0..a82278874b62a 100755 --- a/sky/tools/create_macos_framework.py +++ b/sky/tools/create_macos_framework.py @@ -58,20 +58,20 @@ def main(): print('Cannot find macOS x64 dylib at %s' % x64_dylib) return 1 - fat_framework_bundle = os.path.join(dst, 'FlutterMacOS.framework') - sky_utils.copy_tree(arm64_framework, fat_framework_bundle, symlinks=True) - regenerate_symlinks(fat_framework_bundle) + fat_framework = os.path.join(dst, 'FlutterMacOS.framework') + sky_utils.copy_tree(arm64_framework, fat_framework, symlinks=True) + regenerate_symlinks(fat_framework) - fat_framework_binary = os.path.join(fat_framework_bundle, 'Versions', 'A', 'FlutterMacOS') + fat_framework_binary = os.path.join(fat_framework, 'Versions', 'A', 'FlutterMacOS') # Create the arm64/x64 fat framework. sky_utils.lipo([arm64_dylib, x64_dylib], fat_framework_binary) # Make the framework readable and executable: u=rwx,go=rx. - subprocess.check_call(['chmod', '755', fat_framework_bundle]) + subprocess.check_call(['chmod', '755', fat_framework]) # Add group and other readability to all files. - versions_path = os.path.join(fat_framework_bundle, 'Versions') + versions_path = os.path.join(fat_framework, 'Versions') subprocess.check_call(['chmod', '-R', 'og+r', versions_path]) # Find all the files below the target dir with owner execute permission find_subprocess = subprocess.Popen(['find', versions_path, '-perm', '-100', '-print0'], @@ -82,10 +82,10 @@ def main(): find_subprocess.wait() xargs_subprocess.wait() - process_framework(dst, args, fat_framework_bundle, fat_framework_binary) + process_framework(dst, args, fat_framework, fat_framework_binary) # Create XCFramework from the arm64 and x64 fat framework. - xcframeworks = [fat_framework_bundle] + xcframeworks = [fat_framework] create_xcframework(location=dst, name='FlutterMacOS', frameworks=xcframeworks) if args.zip: @@ -94,40 +94,40 @@ def main(): return 0 -def regenerate_symlinks(fat_framework_bundle): +def regenerate_symlinks(fat_framework): """Regenerates the symlinks structure. Recipes V2 upload artifacts in CAS before integration and CAS follows symlinks. This logic regenerates the symlinks in the expected structure. """ - if os.path.islink(os.path.join(fat_framework_bundle, 'FlutterMacOS')): + if os.path.islink(os.path.join(fat_framework, 'FlutterMacOS')): return - os.remove(os.path.join(fat_framework_bundle, 'FlutterMacOS')) - shutil.rmtree(os.path.join(fat_framework_bundle, 'Headers'), True) - shutil.rmtree(os.path.join(fat_framework_bundle, 'Modules'), True) - shutil.rmtree(os.path.join(fat_framework_bundle, 'Resources'), True) - current_version_path = os.path.join(fat_framework_bundle, 'Versions', 'Current') + os.remove(os.path.join(fat_framework, 'FlutterMacOS')) + shutil.rmtree(os.path.join(fat_framework, 'Headers'), True) + shutil.rmtree(os.path.join(fat_framework, 'Modules'), True) + shutil.rmtree(os.path.join(fat_framework, 'Resources'), True) + current_version_path = os.path.join(fat_framework, 'Versions', 'Current') shutil.rmtree(current_version_path, True) os.symlink('A', current_version_path) os.symlink( os.path.join('Versions', 'Current', 'FlutterMacOS'), - os.path.join(fat_framework_bundle, 'FlutterMacOS') + os.path.join(fat_framework, 'FlutterMacOS') ) os.symlink( - os.path.join('Versions', 'Current', 'Headers'), os.path.join(fat_framework_bundle, 'Headers') + os.path.join('Versions', 'Current', 'Headers'), os.path.join(fat_framework, 'Headers') ) os.symlink( - os.path.join('Versions', 'Current', 'Modules'), os.path.join(fat_framework_bundle, 'Modules') + os.path.join('Versions', 'Current', 'Modules'), os.path.join(fat_framework, 'Modules') ) os.symlink( os.path.join('Versions', 'Current', 'Resources'), - os.path.join(fat_framework_bundle, 'Resources') + os.path.join(fat_framework, 'Resources') ) -def process_framework(dst, args, fat_framework_bundle, fat_framework_binary): +def process_framework(dst, args, fat_framework, fat_framework_binary): if args.dsym: - dsym_out = os.path.splitext(fat_framework_bundle)[0] + '.dSYM' + dsym_out = os.path.splitext(fat_framework)[0] + '.dSYM' sky_utils.extract_dsym(fat_framework_binary, dsym_out) if args.zip: dsym_dst = os.path.join(dst, 'FlutterMacOS.dSYM') From 04ce39942a60b6762d540adf45609c87095af310 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 16 Aug 2024 12:04:56 -0700 Subject: [PATCH 3/6] Extract create_fat_macos_framework --- sky/tools/create_macos_framework.py | 59 +++++++++++++++++------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/sky/tools/create_macos_framework.py b/sky/tools/create_macos_framework.py index a82278874b62a..0ca4fca37bd1b 100755 --- a/sky/tools/create_macos_framework.py +++ b/sky/tools/create_macos_framework.py @@ -59,19 +59,38 @@ def main(): return 1 fat_framework = os.path.join(dst, 'FlutterMacOS.framework') + create_fat_macos_framework(fat_framework, arm64_framework, x64_framework) + process_framework(dst, args, fat_framework) + + # Create XCFramework from the arm64 and x64 fat framework. + xcframeworks = [fat_framework] + create_xcframework(location=dst, name='FlutterMacOS', frameworks=xcframeworks) + + if args.zip: + zip_framework(dst) + + return 0 + + +def create_fat_macos_framework(fat_framework, arm64_framework, x64_framework): sky_utils.copy_tree(arm64_framework, fat_framework, symlinks=True) regenerate_symlinks(fat_framework) fat_framework_binary = os.path.join(fat_framework, 'Versions', 'A', 'FlutterMacOS') # Create the arm64/x64 fat framework. + arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS') + x64_dylib = os.path.join(x64_framework, 'FlutterMacOS') sky_utils.lipo([arm64_dylib, x64_dylib], fat_framework_binary) + set_framework_permissions(fat_framework) + +def set_framework_permissions(framework_dir): # Make the framework readable and executable: u=rwx,go=rx. - subprocess.check_call(['chmod', '755', fat_framework]) + subprocess.check_call(['chmod', '755', framework_dir]) # Add group and other readability to all files. - versions_path = os.path.join(fat_framework, 'Versions') + versions_path = os.path.join(framework_dir, 'Versions') subprocess.check_call(['chmod', '-R', 'og+r', versions_path]) # Find all the files below the target dir with owner execute permission find_subprocess = subprocess.Popen(['find', versions_path, '-perm', '-100', '-print0'], @@ -82,50 +101,40 @@ def main(): find_subprocess.wait() xargs_subprocess.wait() - process_framework(dst, args, fat_framework, fat_framework_binary) - - # Create XCFramework from the arm64 and x64 fat framework. - xcframeworks = [fat_framework] - create_xcframework(location=dst, name='FlutterMacOS', frameworks=xcframeworks) - - if args.zip: - zip_framework(dst) - - return 0 - -def regenerate_symlinks(fat_framework): +def regenerate_symlinks(framework_dir): """Regenerates the symlinks structure. Recipes V2 upload artifacts in CAS before integration and CAS follows symlinks. This logic regenerates the symlinks in the expected structure. """ - if os.path.islink(os.path.join(fat_framework, 'FlutterMacOS')): + if os.path.islink(os.path.join(framework_dir, 'FlutterMacOS')): return - os.remove(os.path.join(fat_framework, 'FlutterMacOS')) - shutil.rmtree(os.path.join(fat_framework, 'Headers'), True) - shutil.rmtree(os.path.join(fat_framework, 'Modules'), True) - shutil.rmtree(os.path.join(fat_framework, 'Resources'), True) - current_version_path = os.path.join(fat_framework, 'Versions', 'Current') + os.remove(os.path.join(framework_dir, 'FlutterMacOS')) + shutil.rmtree(os.path.join(framework_dir, 'Headers'), True) + shutil.rmtree(os.path.join(framework_dir, 'Modules'), True) + shutil.rmtree(os.path.join(framework_dir, 'Resources'), True) + current_version_path = os.path.join(framework_dir, 'Versions', 'Current') shutil.rmtree(current_version_path, True) os.symlink('A', current_version_path) os.symlink( os.path.join('Versions', 'Current', 'FlutterMacOS'), - os.path.join(fat_framework, 'FlutterMacOS') + os.path.join(framework_dir, 'FlutterMacOS') ) os.symlink( - os.path.join('Versions', 'Current', 'Headers'), os.path.join(fat_framework, 'Headers') + os.path.join('Versions', 'Current', 'Headers'), os.path.join(framework_dir, 'Headers') ) os.symlink( - os.path.join('Versions', 'Current', 'Modules'), os.path.join(fat_framework, 'Modules') + os.path.join('Versions', 'Current', 'Modules'), os.path.join(framework_dir, 'Modules') ) os.symlink( os.path.join('Versions', 'Current', 'Resources'), - os.path.join(fat_framework, 'Resources') + os.path.join(framework_dir, 'Resources') ) -def process_framework(dst, args, fat_framework, fat_framework_binary): +def process_framework(dst, args, fat_framework): + fat_framework_binary = os.path.join(fat_framework, 'Versions', 'A', 'FlutterMacOS') if args.dsym: dsym_out = os.path.splitext(fat_framework)[0] + '.dSYM' sky_utils.extract_dsym(fat_framework_binary, dsym_out) From ee876db02944d64905e616a09c680d31e067940f Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 16 Aug 2024 12:46:10 -0700 Subject: [PATCH 4/6] Extract framework creation funcs to sky_utils --- sky/tools/create_macos_framework.py | 63 +---------------------------- sky/tools/sky_utils.py | 61 ++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/sky/tools/create_macos_framework.py b/sky/tools/create_macos_framework.py index 0ca4fca37bd1b..bbfe65c9dc158 100755 --- a/sky/tools/create_macos_framework.py +++ b/sky/tools/create_macos_framework.py @@ -59,7 +59,7 @@ def main(): return 1 fat_framework = os.path.join(dst, 'FlutterMacOS.framework') - create_fat_macos_framework(fat_framework, arm64_framework, x64_framework) + sky_utils.create_fat_macos_framework(fat_framework, arm64_framework, x64_framework) process_framework(dst, args, fat_framework) # Create XCFramework from the arm64 and x64 fat framework. @@ -72,67 +72,6 @@ def main(): return 0 -def create_fat_macos_framework(fat_framework, arm64_framework, x64_framework): - sky_utils.copy_tree(arm64_framework, fat_framework, symlinks=True) - regenerate_symlinks(fat_framework) - - fat_framework_binary = os.path.join(fat_framework, 'Versions', 'A', 'FlutterMacOS') - - # Create the arm64/x64 fat framework. - arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS') - x64_dylib = os.path.join(x64_framework, 'FlutterMacOS') - sky_utils.lipo([arm64_dylib, x64_dylib], fat_framework_binary) - set_framework_permissions(fat_framework) - - -def set_framework_permissions(framework_dir): - # Make the framework readable and executable: u=rwx,go=rx. - subprocess.check_call(['chmod', '755', framework_dir]) - - # Add group and other readability to all files. - versions_path = os.path.join(framework_dir, 'Versions') - subprocess.check_call(['chmod', '-R', 'og+r', versions_path]) - # Find all the files below the target dir with owner execute permission - find_subprocess = subprocess.Popen(['find', versions_path, '-perm', '-100', '-print0'], - stdout=subprocess.PIPE) - # Add execute permission for other and group for all files that had it for owner. - xargs_subprocess = subprocess.Popen(['xargs', '-0', 'chmod', 'og+x'], - stdin=find_subprocess.stdout) - find_subprocess.wait() - xargs_subprocess.wait() - - -def regenerate_symlinks(framework_dir): - """Regenerates the symlinks structure. - - Recipes V2 upload artifacts in CAS before integration and CAS follows symlinks. - This logic regenerates the symlinks in the expected structure. - """ - if os.path.islink(os.path.join(framework_dir, 'FlutterMacOS')): - return - os.remove(os.path.join(framework_dir, 'FlutterMacOS')) - shutil.rmtree(os.path.join(framework_dir, 'Headers'), True) - shutil.rmtree(os.path.join(framework_dir, 'Modules'), True) - shutil.rmtree(os.path.join(framework_dir, 'Resources'), True) - current_version_path = os.path.join(framework_dir, 'Versions', 'Current') - shutil.rmtree(current_version_path, True) - os.symlink('A', current_version_path) - os.symlink( - os.path.join('Versions', 'Current', 'FlutterMacOS'), - os.path.join(framework_dir, 'FlutterMacOS') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Headers'), os.path.join(framework_dir, 'Headers') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Modules'), os.path.join(framework_dir, 'Modules') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Resources'), - os.path.join(framework_dir, 'Resources') - ) - - def process_framework(dst, args, fat_framework): fat_framework_binary = os.path.join(fat_framework, 'Versions', 'A', 'FlutterMacOS') if args.dsym: diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index 30c3eb429a3a3..c99d24543f45b 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -118,6 +118,67 @@ def copy_tree(source_path, destination_path, symlinks=False): shutil.copytree(source_path, destination_path, symlinks=symlinks) +def create_fat_macos_framework(fat_framework, arm64_framework, x64_framework): + copy_tree(arm64_framework, fat_framework, symlinks=True) + _regenerate_symlinks(fat_framework) + + fat_framework_binary = os.path.join(fat_framework, 'Versions', 'A', 'FlutterMacOS') + + # Create the arm64/x64 fat framework. + arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS') + x64_dylib = os.path.join(x64_framework, 'FlutterMacOS') + lipo([arm64_dylib, x64_dylib], fat_framework_binary) + _set_framework_permissions(fat_framework) + + +def _regenerate_symlinks(framework_dir): + """Regenerates the symlinks structure. + + Recipes V2 upload artifacts in CAS before integration and CAS follows symlinks. + This logic regenerates the symlinks in the expected structure. + """ + if os.path.islink(os.path.join(framework_dir, 'FlutterMacOS')): + return + os.remove(os.path.join(framework_dir, 'FlutterMacOS')) + shutil.rmtree(os.path.join(framework_dir, 'Headers'), True) + shutil.rmtree(os.path.join(framework_dir, 'Modules'), True) + shutil.rmtree(os.path.join(framework_dir, 'Resources'), True) + current_version_path = os.path.join(framework_dir, 'Versions', 'Current') + shutil.rmtree(current_version_path, True) + os.symlink('A', current_version_path) + os.symlink( + os.path.join('Versions', 'Current', 'FlutterMacOS'), + os.path.join(framework_dir, 'FlutterMacOS') + ) + os.symlink( + os.path.join('Versions', 'Current', 'Headers'), os.path.join(framework_dir, 'Headers') + ) + os.symlink( + os.path.join('Versions', 'Current', 'Modules'), os.path.join(framework_dir, 'Modules') + ) + os.symlink( + os.path.join('Versions', 'Current', 'Resources'), + os.path.join(framework_dir, 'Resources') + ) + + +def _set_framework_permissions(framework_dir): + # Make the framework readable and executable: u=rwx,go=rx. + subprocess.check_call(['chmod', '755', framework_dir]) + + # Add group and other readability to all files. + versions_path = os.path.join(framework_dir, 'Versions') + subprocess.check_call(['chmod', '-R', 'og+r', versions_path]) + # Find all the files below the target dir with owner execute permission + find_subprocess = subprocess.Popen(['find', versions_path, '-perm', '-100', '-print0'], + stdout=subprocess.PIPE) + # Add execute permission for other and group for all files that had it for owner. + xargs_subprocess = subprocess.Popen(['xargs', '-0', 'chmod', 'og+x'], + stdin=find_subprocess.stdout) + find_subprocess.wait() + xargs_subprocess.wait() + + def create_zip(cwd, zip_filename, paths): """Creates a zip archive in cwd, containing a set of cwd-relative files. From f10ffdd6e905c24ec2baa9ad20475d3e4258d7a5 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 16 Aug 2024 12:53:16 -0700 Subject: [PATCH 5/6] Formatting --- sky/tools/sky_utils.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index c99d24543f45b..f9441eb226c16 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -150,15 +150,10 @@ def _regenerate_symlinks(framework_dir): os.path.join('Versions', 'Current', 'FlutterMacOS'), os.path.join(framework_dir, 'FlutterMacOS') ) + os.symlink(os.path.join('Versions', 'Current', 'Headers'), os.path.join(framework_dir, 'Headers')) + os.symlink(os.path.join('Versions', 'Current', 'Modules'), os.path.join(framework_dir, 'Modules')) os.symlink( - os.path.join('Versions', 'Current', 'Headers'), os.path.join(framework_dir, 'Headers') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Modules'), os.path.join(framework_dir, 'Modules') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Resources'), - os.path.join(framework_dir, 'Resources') + os.path.join('Versions', 'Current', 'Resources'), os.path.join(framework_dir, 'Resources') ) From e25ac32077d7f74645f014473acb47892ce9ecb6 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 16 Aug 2024 13:39:18 -0700 Subject: [PATCH 6/6] Remove unused import --- sky/tools/create_macos_framework.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sky/tools/create_macos_framework.py b/sky/tools/create_macos_framework.py index bbfe65c9dc158..8f9cd71f09d7d 100755 --- a/sky/tools/create_macos_framework.py +++ b/sky/tools/create_macos_framework.py @@ -5,7 +5,6 @@ # found in the LICENSE file. import argparse -import subprocess import shutil import sys import os