Upload xcresults to LUCI cloud storage

upload scenario app test result

fix script

Author: jmagman

testing/run_tests.py

@@ -17,6 +17,7 @@
 import multiprocessing
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -752,6 +753,12 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None):
     ios_unit_test_dir = os.path.join(
         BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests'
     )
+
+    result_bundle_temp = tempfile.TemporaryDirectory(
+        suffix='ios_embedding_xcresult'
+    ).name
+    result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding')
+
     # Avoid using xcpretty unless the following can be addressed:
     # - Make sure all relevant failure output is printed on a failure.
     # - Make sure that a failing exit code is set for CI.
@@ -760,13 +767,27 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None):
         'xcodebuild '
         '-sdk iphonesimulator '
         '-scheme IosUnitTests '
-        "-destination name='" + new_simulator_name + "' "
+        '-resultBundlePath ' + result_bundle_path + " -destination name='" +
+        new_simulator_name + "' "
         'test '
         'FLUTTER_ENGINE=' + ios_variant
     ]
     if test_filter is not None:
       test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter
     run_cmd(test_command, cwd=ios_unit_test_dir, shell=True)
+
+    # except:
+    # The LUCI environment may provide a variable containing a directory path
+    # for additional output files that will be uploaded to cloud storage.
+    # Upload the xcresult when the tests fail.
+    luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR')
+    xcresult_bundle = os.path.join(result_bundle_temp, 'ios_embedding.xcresult')
+    print(xcresult_bundle)
+    if luci_test_outputs_path and os.path.exists(xcresult_bundle):
+      dump_path = os.path.join(luci_test_outputs_path, 'ios_embedding.xcresult')
+      # xcresults contain many little files. Archive the bundle before upload.
+      shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle)
+
   finally:
     delete_simulator(new_simulator_name)
 

testing/scenario_app/run_ios_tests.sh

@@ -41,25 +41,57 @@ fi
 # Can also be set via Simulator app Device > Rotate Device Automatically
 defaults write com.apple.iphonesimulator RotateWindowWhenSignaledByGuest -int 1
 
-cd $SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios
+SCENARIO_PATH=$SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios
+cd $SCENARIO_PATH
+
+RESULT_BUNDLE_FOLDER="ios_scenario_xcresult"
+RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}"
+
+# Zip and upload xcresult to luci.
+# First parameter ($1) is the zip output name.
+ZIP_AND_UPLOAD_XCRESULT_TO_LUCI () {
+  # Using RESULT_BUNDLE_PATH causes the zip containing all the sub directories.
+  # So use relative directory instead.
+  echo $1
+  zip -q -r $1 "./$RESULT_BUNDLE_FOLDER"
+  if ( -z "$FLUTTER_TEST_OUTPUTS_DIR") then
+    mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR
+  fi
+  exit 1
+}
 
 echo "Running simulator tests with Skia"
 echo ""
 
-set -o pipefail && xcodebuild -sdk iphonesimulator \
+mktemp -d $RESULT_BUNDLE_PATH
+trap 'rm -rf $RESULT_BUNDLE_PATH' EXIT
+
+if set -o pipefail && xcodebuild -sdk iphonesimulator \
   -scheme Scenarios \
+  -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \
   -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \
   clean test \
-  FLUTTER_ENGINE="$FLUTTER_ENGINE"
+  FLUTTER_ENGINE="$FLUTTER_ENGINE"; then
+  echo "test success."
+else
+  echo "test failed."
+  ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_xcresult.zip"
+fi
 
 echo "Running simulator tests with Impeller"
 echo ""
 
 # Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250
-set -o pipefail && xcodebuild -sdk iphonesimulator \
+if set -o pipefail && xcodebuild -sdk iphonesimulator \
   -scheme Scenarios \
+  -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \
   -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \
   clean test \
   FLUTTER_ENGINE="$FLUTTER_ENGINE" \
   -skip-testing "ScenariosUITests/BogusFontTextTest/testFontRenderingWhenSuppliedWithBogusFont" \
-  INFOPLIST_FILE="Scenarios/Info_Impeller.plist" # Plist with FLTEnableImpeller=YES
+  INFOPLIST_FILE="Scenarios/Info_Impeller.plist"; then # Plist with FLTEnableImpeller=YES
+  echo "test success."
+else
+  echo "test failed."
+  ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_impeller_xcresult.zip"
+fi