Upload xcresults to LUCI cloud storage (#41647)

Taking over from #41644

fixes: flutter/flutter#125823

Steps to verify

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style

Author: Chris Yang

testing/run_tests.py

@@ -17,6 +17,7 @@
 import multiprocessing
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -753,21 +754,43 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None):
     ios_unit_test_dir = os.path.join(
         BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests'
     )
-    # Avoid using xcpretty unless the following can be addressed:
-    # - Make sure all relevant failure output is printed on a failure.
-    # - Make sure that a failing exit code is set for CI.
-    # See https://github.com/flutter/flutter/issues/63742
-    test_command = [
-        'xcodebuild '
-        '-sdk iphonesimulator '
-        '-scheme IosUnitTests '
-        "-destination name='" + new_simulator_name + "' "
-        'test '
-        'FLUTTER_ENGINE=' + ios_variant
-    ]
-    if test_filter is not None:
-      test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter
-    run_cmd(test_command, cwd=ios_unit_test_dir, shell=True)
+
+    with tempfile.TemporaryDirectory(suffix='ios_embedding_xcresult'
+                                    ) as result_bundle_temp:
+      result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding')
+
+      # Avoid using xcpretty unless the following can be addressed:
+      # - Make sure all relevant failure output is printed on a failure.
+      # - Make sure that a failing exit code is set for CI.
+      # See https://github.com/flutter/flutter/issues/63742
+      test_command = [
+          'xcodebuild '
+          '-sdk iphonesimulator '
+          '-scheme IosUnitTests '
+          '-resultBundlePath ' + result_bundle_path + ' '
+          '-destination name=' + new_simulator_name + ' '
+          'test '
+          'FLUTTER_ENGINE=' + ios_variant
+      ]
+      if test_filter is not None:
+        test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter
+      run_cmd(test_command, cwd=ios_unit_test_dir, shell=True)
+
+      # except:
+      # The LUCI environment may provide a variable containing a directory path
+      # for additional output files that will be uploaded to cloud storage.
+      # Upload the xcresult when the tests fail.
+      luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR')
+      xcresult_bundle = os.path.join(
+          result_bundle_temp, 'ios_embedding.xcresult'
+      )
+      if luci_test_outputs_path and os.path.exists(xcresult_bundle):
+        dump_path = os.path.join(
+            luci_test_outputs_path, 'ios_embedding.xcresult'
+        )
+        # xcresults contain many little files. Archive the bundle before upload.
+        shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle)
+
   finally:
     delete_simulator(new_simulator_name)
 

testing/scenario_app/run_ios_tests.sh

@@ -41,25 +41,58 @@ fi
 # Can also be set via Simulator app Device > Rotate Device Automatically
 defaults write com.apple.iphonesimulator RotateWindowWhenSignaledByGuest -int 1
 
-cd $SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios
+SCENARIO_PATH=$SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios
+pushd .
+cd $SCENARIO_PATH
+
+RESULT_BUNDLE_FOLDER="ios_scenario_xcresult"
+RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}"
+
+# Zip and upload xcresult to luci.
+# First parameter ($1) is the zip output name.
+zip_and_upload_xcresult_to_luci () {
+  # We don't want the zip to contain the abusolute path,
+  # so use relative path (./$RESULT_BUNDLE_FOLDER) instead.
+  zip -q -r $1 "./$RESULT_BUNDLE_FOLDER"
+  mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR
+  exit 1
+}
 
 echo "Running simulator tests with Skia"
 echo ""
 
-set -o pipefail && xcodebuild -sdk iphonesimulator \
+mktemp -d $RESULT_BUNDLE_PATH
+
+if set -o pipefail && xcodebuild -sdk iphonesimulator \
   -scheme Scenarios \
+  -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \
   -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \
   clean test \
-  FLUTTER_ENGINE="$FLUTTER_ENGINE"
+  FLUTTER_ENGINE="$FLUTTER_ENGINE"; then
+  echo "test success."
+else
+  echo "test failed."
+  zip_and_upload_xcresult_to_luci "ios_scenario_xcresult.zip"
+fi
+rm -rf $RESULT_BUNDLE_PATH
 
 echo "Running simulator tests with Impeller"
 echo ""
 
 # Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250
-set -o pipefail && xcodebuild -sdk iphonesimulator \
+if set -o pipefail && xcodebuild -sdk iphonesimulator \
   -scheme Scenarios \
+  -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \
   -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \
   clean test \
   FLUTTER_ENGINE="$FLUTTER_ENGINE" \
   -skip-testing "ScenariosUITests/BogusFontTextTest/testFontRenderingWhenSuppliedWithBogusFont" \
-  INFOPLIST_FILE="Scenarios/Info_Impeller.plist" # Plist with FLTEnableImpeller=YES
+  INFOPLIST_FILE="Scenarios/Info_Impeller.plist"; then # Plist with FLTEnableImpeller=YES
+  echo "test success."
+else
+  echo "test failed."
+  zip_and_upload_xcresult_to_luci "ios_scenario_impeller_xcresult.zip"
+fi
+rm -rf $RESULT_BUNDLE_PATH
+
+popd