[vm] Build dart2native dependencies with the normal "create_sdk" target

This removes special logic for creating the `dart-sdk` we distribute
which used to build release and product mode and copied some binaries
from the latter into the former, before the SDK was actuallly ready to
test and distribute.

This changes the GN build rules to build the necessary
gen_snapshot/dart_precompiled_runtime product binaries during the normal
release build.

Normally during --mode=product builds the global build config in
//build/config/BUILDCONFIG.gn will set `-fvisibility=false`.

=> Doing so results in much smaller binaries - because only explicitly
   exported symbols are visible, the rest can be tree shaken by the linker.

Since we are building --mode=release, the `-fvisibility=false` will not
be set. In order to set the flag for the 2 special product-mode binaries
we need to add -fvisibility=hidden manually, in:

  * dart_product_config: Which is used for compiling VM sources.
  * 3rd party double-conversion library
  * 3rd party boringssl library
  * 3rd party icu library

The upstream CLs are:

  * BoringSSL: https://dart-review.googlesource.com/c/boringssl_gen/+/150482
  * ICU: https://chromium-review.googlesource.com/c/chromium/deps/icu/+/2236407

Issue #42230

Change-Id: I3e47664d9fadb9ed1ad033bb17d46e769442f741
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/150524
Commit-Queue: Martin Kustermann <[email protected]>
Reviewed-by: Alexander Thomas <[email protected]>
Reviewed-by: Zach Anderson <[email protected]>

Author: mkustermann

pkg/dart2native/bin/dart2native.dart

@@ -19,8 +19,6 @@ final String dartaotruntime =
     path.join(binDir, 'dartaotruntime${executableSuffix}');
 final String genSnapshot =
     path.join(binDir, 'utils', 'gen_snapshot${executableSuffix}');
-final String platformDill =
-    path.join(sdkDir, 'lib', '_internal', 'vm_platform_strong.dill');
 final String productPlatformDill =
     path.join(sdkDir, 'lib', '_internal', 'vm_platform_strong_product.dill');
 
@@ -45,20 +43,8 @@ Future<void> generateNative(
       print('Generating AOT kernel dill.');
     }
 
-    // Prefer to use the product platform file, if available. Fall back to the
-    // normal one (this happens if `out/<build-dir>/dart-sdk` is used).
-    //
-    // Background information: For the `dart-sdk` we distribute we build release
-    // and product mode configurations. Then we have an extra bundling step
-    // which will add product-mode
-    // gen_snapshot/dartaotruntime/vm_platform_strong_product.dill to the
-    // release SDK (see tools/bots/dart_sdk.py:CopyAotBinaries)
-    final String platformFileToUse = File(productPlatformDill).existsSync()
-        ? productPlatformDill
-        : platformDill;
-
     final kernelResult = await generateAotKernel(dart, genKernel,
-        platformFileToUse, sourceFile, kernelFile, packages, defines);
+        productPlatformDill, sourceFile, kernelFile, packages, defines);
     if (kernelResult.exitCode != 0) {
       stderr.writeln(kernelResult.stdout);
       stderr.writeln(kernelResult.stderr);

runtime/BUILD.gn

@@ -40,8 +40,16 @@ config("dart_maybe_product_config") {
 # If a DEBUG build has been specified it will be ignored.
 config("dart_product_config") {
   defines = []
+  cflags = []
   if (!dart_debug) {
     defines += [ "PRODUCT" ]
+    if (is_posix) {
+      cflags = [
+        # This is the equivalent from `build/config/BUILDCONFIG.gn` which includes
+        # `build/config/gcc:symbol_visibility_hidden` in product mode.
+        "-fvisibility=hidden",
+      ]
+    }
   }
 }
 

runtime/bin/BUILD.gn

@@ -69,6 +69,7 @@ build_libdart_builtin("libdart_builtin_product") {
   extra_configs = [
     "..:dart_product_config",
     "..:dart_os_config",
+    "..:dart_arch_config",
   ]
 }
 
@@ -136,7 +137,11 @@ template("build_elf_loader") {
 }
 
 build_elf_loader("elf_loader") {
-  deps = [ ":libdart_builtin" ]
+  if (dart_runtime_mode == "release") {
+    deps = [ ":libdart_builtin_product" ]
+  } else {
+    deps = [ ":libdart_builtin" ]
+  }
 }
 
 build_elf_loader("elf_loader_product") {
@@ -512,6 +517,12 @@ dart_io("standalone_dart_io") {
   extra_deps = [ ":libdart_builtin" ]
 }
 
+dart_io("standalone_dart_io_product") {
+  extra_configs = [ "..:dart_product_config" ]
+  extra_sources = []
+  extra_deps = [ ":libdart_builtin_product" ]
+}
+
 gen_snapshot_action("generate_snapshot_bin") {
   deps = [ "../vm:vm_platform_stripped" ]
   vm_snapshot_data = "$target_gen_dir/vm_snapshot_data.bin"
@@ -732,6 +743,11 @@ action("generate_abi_version_cc_file") {
 }
 
 template("dart_executable") {
+  use_product_mode = dart_runtime_mode == "release"
+  if (defined(invoker.use_product_mode)) {
+    use_product_mode = invoker.use_product_mode
+  }
+
   extra_configs = []
   if (defined(invoker.extra_configs)) {
     extra_configs += invoker.extra_configs
@@ -761,8 +777,12 @@ template("dart_executable") {
                  "..:dart_arch_config",
                  "..:dart_config",
                  "..:dart_os_config",
-                 "..:dart_maybe_product_config",
                ] + extra_configs
+    if (use_product_mode) {
+      configs += [ "..:dart_product_config" ]
+    } else {
+      configs += [ "..:dart_maybe_product_config" ]
+    }
     if (target_os != current_os && target_os == "fuchsia") {
       # We already have these in the standalone build, but Fuchsia doesn't
       # have them. They are needed for running Fuchsia binaries built for the
@@ -775,12 +795,17 @@ template("dart_executable") {
     }
 
     deps = [
-             ":standalone_dart_io",
-             "//third_party/boringssl",
-             "//third_party/zlib",
-             ":crashpad",
-             ":generate_abi_version_cc_file",
-           ] + extra_deps
+      ":crashpad",
+      ":generate_abi_version_cc_file",
+      "//third_party/boringssl",
+      "//third_party/zlib",
+    ]
+    if (use_product_mode) {
+      deps += [ ":standalone_dart_io_product" ]
+    } else {
+      deps += [ ":standalone_dart_io" ]
+    }
+    deps += extra_deps
 
     defines = extra_defines
     if (exclude_kernel_service) {
@@ -888,6 +913,27 @@ dart_executable("dart_precompiled_runtime") {
   }
 }
 
+dart_executable("dart_precompiled_runtime_product") {
+  use_product_mode = true
+  extra_configs = [ "..:dart_precompiled_runtime_config" ]
+  extra_deps = [
+    "..:libdart_precompiled_runtime_product",
+    "../platform:libdart_platform_precompiled_runtime_product",
+  ]
+  extra_sources = [
+    "builtin.cc",
+    "gzip.cc",
+    "gzip.h",
+    "loader.cc",
+    "loader.h",
+    "main.cc",
+    "observatory_assets_empty.cc",
+    "snapshot_empty.cc",
+  ]
+
+  extra_deps += [ ":elf_loader_product" ]
+}
+
 executable("process_test") {
   sources = [ "process_test.cc" ]
 }

runtime/configs.gni

@@ -66,66 +66,77 @@ _all_configs = [
     configs = _jit_config
     snapshot = true
     compiler = true
+    is_product = false
   },
   {
     suffix = "_jit_product"
     configs = _jit_product_config
     snapshot = true
     compiler = true
+    is_product = true
   },
   {
     suffix = "_precompiled_runtime"
     configs = _precompiled_runtime_config
     snapshot = true
     compiler = false
+    is_product = false
   },
   {
     suffix = "_precompiled_runtime_product"
     configs = _precompiled_runtime_product_config
     snapshot = true
     compiler = false
+    is_product = true
   },
   {
     suffix = "_precompiler"
     configs = _precompiler_config
     snapshot = false
     compiler = true
+    is_product = false
   },
   {
     suffix = "_precompiler_product"
     configs = _precompiler_product_config
     snapshot = false
     compiler = true
+    is_product = true
   },
   {
     suffix = "_precompiler_fuchsia"
     configs = _precompiler_fuchsia_config
     snapshot = false
     compiler = true
+    is_product = false
   },
   {
     suffix = "_precompiler_product_fuchsia"
     configs = _precompiler_product_fuchsia_config
     snapshot = false
     compiler = true
+    is_product = true
   },
   {
     suffix = "_precompiler_host_targeting_host"
     configs = _precompiler_host_targeting_host_config
     snapshot = false
     compiler = true
+    is_product = false
   },
   {
     suffix = "_precompiler_product_host_targeting_host"
     configs = _precompiler_product_host_targeting_host_config
     snapshot = false
     compiler = true
+    is_product = true
   },
   {
     suffix = "_libfuzzer"
     configs = _libfuzzer_config
     snapshot = true
     compiler = true
+    is_product = false
   },
 ]
 
@@ -174,6 +185,14 @@ template("library_for_all_configs") {
   if (defined(invoker.extra_deps)) {
     extra_deps += invoker.extra_deps
   }
+  extra_product_deps = []
+  if (defined(invoker.extra_product_deps)) {
+    extra_product_deps += invoker.extra_product_deps
+  }
+  extra_nonproduct_deps = []
+  if (defined(invoker.extra_nonproduct_deps)) {
+    extra_nonproduct_deps += invoker.extra_nonproduct_deps
+  }
   foreach(conf, _all_configs) {
     target(invoker.target_type, "${target_name}${conf.suffix}") {
       forward_variables_from(invoker,
@@ -196,6 +215,11 @@ template("library_for_all_configs") {
         }
       }
       deps = configured_deps + extra_deps
+      if (conf.is_product) {
+        deps += extra_product_deps
+      } else {
+        deps += extra_nonproduct_deps
+      }
       if (conf.snapshot) {
         if (defined(snapshot_sources)) {
           sources += snapshot_sources

runtime/third_party/double-conversion/src/BUILD.gn

@@ -3,6 +3,12 @@
 # BSD-style license that can be found in the LICENSE file.
 
 source_set("libdouble_conversion") {
+  # We are only interested in exposing the exported symbols (for size reasons).
+  cflags = []
+  if (is_posix) {
+    cflags += [ "-fvisibility=hidden" ]
+  }
+
   configs += [
     "../../..:dart_arch_config",
     "../../..:dart_config",

runtime/vm/BUILD.gn

@@ -63,7 +63,14 @@ config("libdart_vm_config") {
 
 library_for_all_configs("libdart_vm") {
   target_type = "source_set"
-  extra_deps = [ "//third_party/icu" ]
+  extra_product_deps = [
+    "//third_party/icu:icui18n_hidden_visibility",
+    "//third_party/icu:icuuc_hidden_visibility",
+  ]
+  extra_nonproduct_deps = [
+    "//third_party/icu:icui18n",
+    "//third_party/icu:icuuc",
+  ]
   if (is_fuchsia) {
     if (using_fuchsia_sdk) {
       extra_deps += [
@@ -147,6 +154,10 @@ library_for_all_configs("libdart_lib") {
 template("gen_vm_platform") {
   assert(defined(invoker.output_postfix),
          "Must define output postfix (e.g., '_strong'")
+  is_product_flag = dart_runtime_mode == "release"
+  if (defined(invoker.product_mode)) {
+    is_product_flag = invoker.product_mode
+  }
   compile_platform(target_name) {
     output_postfix = invoker.output_postfix
     if (defined(invoker.add_implicit_vm_platform_dependency)) {
@@ -166,7 +177,6 @@ template("gen_vm_platform") {
       "$root_out_dir/vm_outline" + output_postfix + ".dill",
     ]
     args = [ "dart:core" ]
-    is_product_flag = dart_runtime_mode == "release"
     allow_causal_async_stacks = !is_product_flag
     args += [
       "-Ddart.vm.product=$is_product_flag",
@@ -193,6 +203,16 @@ gen_vm_platform("vm_platform") {
   output_postfix = "_strong"
 }
 
+gen_vm_platform("vm_platform_product") {
+  add_implicit_vm_platform_dependency = false
+  exclude_source = false
+  output_postfix = "_strong_product"
+
+  # In Debug mode we use debug binaries for dart2native.
+  # (see also the "dart_product_config" config)
+  product_mode = !is_debug
+}
+
 gen_vm_platform("vm_platform_stripped") {
   add_implicit_vm_platform_dependency = false
   exclude_source = true