Add new builder for experiment with dynamic modules. (#162855)

sigmundch · Sigmund Cherem · web-flow · commit 2376fa02f1be · 2025-02-11T20:16:43.000Z
This adds a new flavor of android_release_arm64, which enables a build flag. This is not part of the merge-queue since it's intended for experimental and internal use at the moment. Tested locally via `et build --config ci/android_release_arm64_ddm` This is a first step to get a better gauge of the cost on the build. The .json is identical to the android_release_arm64 configuration except for 2 new flags added (easier to diff locally). The .ci.yaml has additional changes to make set this up as no backfill and no release. cc @jtmcdole @jiahaog @andrewkolos --------- Co-authored-by: Sigmund Cherem < 2049220+sigmundch@users.noreply.github.com>
diff --git a/engine/src/flutter/.ci.yaml b/engine/src/flutter/.ci.yaml
@@ -247,6 +247,30 @@ targets:
       # at https://github.com/flutter/flutter/issues/152186.
       cores: "8"
 
+  - name: Linux linux_host_engine_ddm
+    recipe: engine_v2/engine_v2
+    bringup: true
+    timeout: 120
+    properties:
+      release_build: "false"
+      backfill: "false"
+      config_name: linux_host_engine_ddm
+      dependencies: >-
+        [
+          {"dependency": "goldctl", "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"}
+        ]
+    # Do not remove(https://github.com/flutter/flutter/issues/144644)
+    # Scheduler will fail to get the platform
+    drone_dimensions:
+      - os=Linux
+    dimensions:
+      # This is needed so that orchestrators that only spawn subbuilds are not
+      # assigned to the large 32 core workers when doing release builds.
+      # For more details see the issue
+      # at https://github.com/flutter/flutter/issues/152186.
+      cores: "8"
+
+
   - name: Linux linux_android_aot_engine
     recipe: engine_v2/engine_v2
     timeout: 120
@@ -265,6 +289,26 @@ targets:
       # at https://github.com/flutter/flutter/issues/152186.
       cores: "8"
 
+  - name: Linux linux_android_aot_engine_ddm
+    recipe: engine_v2/engine_v2
+    bringup: true
+    timeout: 120
+    properties:
+      config_name: linux_android_aot_engine_ddm
+      release_build: "false"
+      backfill: "false"
+    # Do not remove(https://github.com/flutter/flutter/issues/144644)
+    # Scheduler will fail to get the platform
+    drone_dimensions:
+      - os=Linux
+    dimensions:
+      # This is needed so that orchestrators that only spawn subbuilds are not
+      # assigned to the large 32 core workers when doing release builds.
+      # For more details see the issue
+      # at https://github.com/flutter/flutter/issues/152186.
+      cores: "8"
+
+
   - name: Linux linux_android_debug_engine
     recipe: engine_v2/engine_v2
     timeout: 120
@@ -283,6 +327,25 @@ targets:
       # at https://github.com/flutter/flutter/issues/152186.
       cores: "8"
 
+  - name: Linux linux_android_debug_engine_ddm
+    recipe: engine_v2/engine_v2
+    bringup: true
+    timeout: 120
+    properties:
+      release_build: "false"
+      backfill: "false"
+      config_name: linux_android_debug_engine_ddm
+    # Do not remove(https://github.com/flutter/flutter/issues/144644)
+    # Scheduler will fail to get the platform
+    drone_dimensions:
+      - os=Linux
+    dimensions:
+      # This is needed so that orchestrators that only spawn subbuilds are not
+      # assigned to the large 32 core workers when doing release builds.
+      # For more details see the issue
+      # at https://github.com/flutter/flutter/issues/152186.
+      cores: "8"
+
   - name: Linux linux_license
     recipe: engine_v2/builder
     timeout: 120
diff --git a/engine/src/flutter/build/archives/BUILD.gn b/engine/src/flutter/build/archives/BUILD.gn
@@ -58,14 +58,17 @@ if (build_engine_artifacts) {
       # Mac artifacts sometimes use mac and sometimes darwin. Standardizing the
       # names will require changes in the list of artifacts the tool is downloading.
       if (flutter_runtime_mode == "debug") {
-        prefix = "darwin-$target_cpu/"
+        prefix = "darwin-$target_cpu"
       } else {
-        prefix = "darwin-$target_cpu-$flutter_runtime_mode/"
+        prefix = "darwin-$target_cpu-$flutter_runtime_mode"
       }
     } else if (flutter_runtime_mode != "debug") {
-      prefix = "$full_target_platform_name-$flutter_runtime_mode/"
+      prefix = "$full_target_platform_name-$flutter_runtime_mode"
     } else {
-      prefix = "$full_target_platform_name/"
+      prefix = "$full_target_platform_name"
+    }
+    if (dart_dynamic_modules) {
+      prefix = "$prefix-ddm"
     }
     output = "$prefix/artifacts.zip"
 
@@ -199,7 +202,11 @@ generated_file("dart_sdk_entitlement_config") {
 if (build_engine_artifacts && flutter_prebuilt_dart_sdk) {
   zip_bundle("dart_sdk_archive") {
     deps = [ ":dart_sdk_entitlement_config" ]
-    output = "dart-sdk-$full_target_platform_name.zip"
+    if (dart_dynamic_modules) {
+      output = "dart-sdk-$full_target_platform_name-ddm.zip"
+    } else {
+      output = "dart-sdk-$full_target_platform_name.zip"
+    }
     files = [
       {
         source = prebuilt_dart_sdk
@@ -224,7 +231,11 @@ if (build_engine_artifacts && flutter_prebuilt_dart_sdk) {
 if (build_engine_artifacts && !flutter_prebuilt_dart_sdk) {
   zip_bundle_from_file("dart_sdk_archive") {
     deps = [ "$dart_src:create_sdk" ]
-    output = "dart-sdk-$full_target_platform_name.zip"
+    if (dart_dynamic_modules) {
+      output = "dart-sdk-$full_target_platform_name-ddm.zip"
+    } else {
+      output = "dart-sdk-$full_target_platform_name.zip"
+    }
     files = [
       {
         source = rebase_path("$root_build_dir/dart-sdk")
diff --git a/engine/src/flutter/build/zip_bundle.gni b/engine/src/flutter/build/zip_bundle.gni
@@ -4,6 +4,7 @@
 
 import("//flutter/common/config.gni")
 import("//flutter/shell/version/version.gni")
+import("$dart_src/runtime/runtime_args.gni")
 
 if (flutter_runtime_mode == "jit_release") {
   android_zip_archive_dir = "android-$target_cpu-jit-release"
@@ -12,6 +13,9 @@ if (flutter_runtime_mode == "jit_release") {
   if (flutter_runtime_mode != "debug") {
     android_zip_archive_dir += "-$flutter_runtime_mode"
   }
+  if (dart_dynamic_modules) {
+    android_zip_archive_dir += "-ddm"
+  }
 }
 
 # Creates a zip file in the $root_build_dir/zip_archives folder.
diff --git a/engine/src/flutter/ci/builders/linux_android_aot_engine_ddm.json b/engine/src/flutter/ci/builders/linux_android_aot_engine_ddm.json
@@ -0,0 +1,82 @@
+{
+    "_comment": [
+        "The builds defined in this file should not contain tests, ",
+        "and the file should not contain builds that are essentially tests. ",
+        "The only builds in this file should be the builds necessary to produce ",
+        "release artifacts. ",
+        "Tests to run on linux hosts should go in one of the other linux_ build ",
+        "definition files."
+    ],
+    "builds": [
+        {
+            "archives": [
+                {
+                    "name": "ci/android_release_arm64_ddm",
+                    "type": "gcs",
+                    "base_path": "out/ci/android_release_arm64_ddm/zip_archives/",
+                    "include_paths": [
+                        "out/ci/android_release_arm64_ddm/zip_archives/android-arm64-release-ddm/artifacts.zip",
+                        "out/ci/android_release_arm64_ddm/zip_archives/android-arm64-release-ddm/linux-x64.zip",
+                        "out/ci/android_release_arm64_ddm/zip_archives/android-arm64-release-ddm/symbols.zip",
+                        "out/ci/android_release_arm64_ddm/zip_archives/android-arm64-release-ddm/analyze-snapshot-linux-x64.zip"
+                    ],
+                    "realm": "production"
+                }
+            ],
+            "drone_dimensions": [
+                "device_type=none",
+                "os=Linux"
+            ],
+            "gclient_variables": {
+                "use_rbe": true
+            },
+            "gn": [
+                "--target-dir",
+                "ci/android_release_arm64_ddm",
+                "--runtime-mode",
+                "release",
+                "--android",
+                "--android-cpu",
+                "arm64",
+                "--no-prebuilt-dart-sdk",
+                "--gn-args=dart_dynamic_modules=true",
+                "--rbe",
+                "--no-goma"
+            ],
+            "name": "ci/android_release_arm64_ddm",
+            "description": "Produces experimental release mode artifacts to target 64-bit arm Android from a Linux host with dynamic modules enabled.",
+            "ninja": {
+                "config": "ci/android_release_arm64_ddm",
+                "targets": [
+                    "default",
+                    "clang_x64/gen_snapshot",
+                    "flutter/shell/platform/android:analyze_snapshot"
+                ]
+            },
+            "tests": [
+                {
+                    "name": "Generate treemap for android_release_arm64_ddm",
+                    "language": "bash",
+                    "script": "flutter/ci/binary_size_treemap.sh",
+                    "parameters": [
+                        "../../src/out/ci/android_release_arm64_ddm/libflutter.so",
+                        "${FLUTTER_LOGS_DIR}"
+                    ]
+                }
+            ]
+        }
+    ],
+    "generators": {
+        "tasks": [
+            {
+                "name": "Verify-export-symbols-release-binaries",
+                "parameters": [
+                    "src/out/ci",
+                    "src/flutter/buildtools"
+                ],
+                "script": "flutter/testing/symbols/verify_exported.dart",
+                "language": "dart"
+            }
+        ]
+    }
+}
diff --git a/engine/src/flutter/ci/builders/linux_android_debug_engine_ddm.json b/engine/src/flutter/ci/builders/linux_android_debug_engine_ddm.json
@@ -0,0 +1,65 @@
+{
+    "_comment": [
+        "The builds defined in this file should not contain tests, ",
+        "and the file should not contain builds that are essentially tests. ",
+        "The only builds in this file should be the builds necessary to produce ",
+        "release artifacts. ",
+        "Tests to run on linux hosts should go in one of the other linux_ build ",
+        "definition files."
+    ],
+    "builds": [
+        {
+            "archives": [
+                {
+                    "name": "ci/android_debug_arm64_ddm",
+                    "type": "gcs",
+                    "base_path": "out/ci/android_debug_arm64_ddm/zip_archives/",
+                    "include_paths": [
+                        "out/ci/android_debug_arm64_ddm/zip_archives/android-arm64-ddm/artifacts.zip",
+                        "out/ci/android_debug_arm64_ddm/zip_archives/android-arm64-ddm/symbols.zip"
+                    ],
+                    "realm": "production"
+                }
+            ],
+            "drone_dimensions": [
+                "device_type=none",
+                "os=Linux"
+            ],
+            "gclient_variables": {
+                "use_rbe": true
+            },
+            "gn": [
+                "--target-dir",
+                "ci/android_debug_arm64_ddm",
+                "--android",
+                "--android-cpu=arm64",
+                "--no-lto",
+                "--no-prebuilt-dart-sdk",
+                "--gn-args=dart_dynamic_modules=true",
+                "--rbe",
+                "--no-goma"
+            ],
+            "name": "ci/android_debug_arm64_ddm",
+            "description": "Produces experimental debug mode artifacts to target 64-bit arm Android from a Linux host with dynamic modules enabled.",
+            "ninja": {
+                "config": "ci/android_debug_arm64_ddm",
+                "targets": [
+                    "flutter"
+                ]
+            }
+        }
+    ],
+    "generators": {
+        "tasks": [
+            {
+                "name": "Verify-export-symbols-release-binaries",
+                "parameters": [
+                    "src/out/ci",
+                    "src/flutter/buildtools"
+                ],
+                "script": "flutter/testing/symbols/verify_exported.dart",
+                "language": "dart"
+            }
+        ]
+    }
+}
diff --git a/engine/src/flutter/ci/builders/linux_host_engine_ddm.json b/engine/src/flutter/ci/builders/linux_host_engine_ddm.json
@@ -0,0 +1,65 @@
+{
+    "_comment": [
+        "The builds defined in this file should not contain tests, ",
+        "and the file should not contain builds that are essentially tests. ",
+        "The only builds in this file should be the builds necessary to produce ",
+        "release artifacts. ",
+        "Tests to run on linux hosts should go in one of the other linux_ build ",
+        "definition files."
+    ],
+    "builds": [
+        {
+            "archives": [
+                {
+                    "name": "ci/host_debug_ddm",
+                    "base_path": "out/ci/host_debug_ddm/zip_archives/",
+                    "type": "gcs",
+                    "include_paths": [
+                        "out/ci/host_debug_ddm/zip_archives/dart-sdk-linux-x64-ddm.zip"
+                    ],
+                    "realm": "production"
+                }
+            ],
+            "drone_dimensions": [
+                "device_type=none",
+                "os=Linux"
+            ],
+            "gclient_variables": {
+                "download_android_deps": false,
+                "download_jdk": false,
+                "use_rbe": true
+            },
+            "gn": [
+                "--target-dir",
+                "ci/host_debug_ddm",
+                "--runtime-mode",
+                "debug",
+                "--no-prebuilt-dart-sdk",
+                "--gn-args=dart_dynamic_modules=true",
+                "--rbe",
+                "--no-goma"
+            ],
+            "name": "ci/host_debug_ddm",
+            "description": "Produces experimental debug mode Linux host-side tooling with dynamic modules enabled.",
+            "ninja": {
+                "config": "ci/host_debug_ddm",
+                "targets": [
+                    "flutter/build/archives:dart_sdk_archive"
+                ]
+            }
+        }
+    ],
+    "generators": {
+        "tasks": [
+            {
+                "name": "Verify-export-symbols-release-binaries",
+                "parameters": [
+                    "src/out/ci",
+                    "src/flutter/buildtools"
+                ],
+                "script": "flutter/testing/symbols/verify_exported.dart",
+                "language": "dart"
+            }
+        ]
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@`
`4`	`4`
`5`	`5`	`import("//flutter/common/config.gni")`
`6`	`6`	`import("//flutter/shell/version/version.gni")`
	`7`	`+import("$dart_src/runtime/runtime_args.gni")`
`7`	`8`
`8`	`9`	`if (flutter_runtime_mode == "jit_release") {`
`9`	`10`	`android_zip_archive_dir = "android-$target_cpu-jit-release"`
`@@ -12,6 +13,9 @@ if (flutter_runtime_mode == "jit_release") {`
`12`	`13`	`if (flutter_runtime_mode != "debug") {`
`13`	`14`	`android_zip_archive_dir += "-$flutter_runtime_mode"`
`14`	`15`	`}`
	`16`	`+ if (dart_dynamic_modules) {`
	`17`	`+ android_zip_archive_dir += "-ddm"`
	`18`	`+ }`
`15`	`19`	`}`
`16`	`20`
`17`	`21`	`# Creates a zip file in the $root_build_dir/zip_archives folder.`