diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8878d1a3499..3cd380c970f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -333,6 +333,21 @@ jobs: projectFile: maui_scenarios_ios.proj dotnetVersionsLinks: main: https://aka.ms/dotnet/sdk/maui/net8.0.json + runtimeFlavor: mono + + # Maui iOS scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_ios + projectFile: maui_scenarios_ios.proj + dotnetVersionsLinks: + main: https://aka.ms/dotnet/sdk/maui/net8.0.json + runtimeFlavor: coreclr ## Maui scenario benchmarks #- template: /eng/performance/build_machine_matrix.yml diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 393529f1020..b914dca8b93 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -10,6 +10,7 @@ scenarios_out $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + --nativeaot true @@ -24,12 +25,24 @@ - + + xamarinios + $(ScenariosDir)%(ScenarioDirectoryName) + XamariniOSDefault + com.companyname.XamariniOSDefault + + mauiios $(ScenariosDir)%(ScenarioDirectoryName) MauiiOSDefault net.dot.mauitesting + + mauiblazorios + $(ScenariosDir)%(ScenarioDirectoryName) + MauiBlazoriOSDefault + net.dot.mauiblazortesting + - - mauiblazorios - $(ScenariosDir)%(ScenarioDirectoryName) - MauiBlazoriOSDefault - net.dot.mauiblazortesting - - $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(OptionalCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) @@ -61,7 +68,35 @@ cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip $(Python) test.py sod --scenario-name "%(Identity)" - + + $(ScenariosDir)xamarinios.zip + 00:15:00 + ios-device + + + + + $(ScenariosDir)mauiios.zip 00:15:00 ios-device @@ -89,7 +124,7 @@ ]]> - + $(ScenariosDir)mauiblazorios.zip 00:15:00 ios-device diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 4c32a1a172f..f56a1460150 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -13,6 +13,7 @@ parameters: affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc) runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces) + runtimeFlavor: '' # optional -- Runtime flavor used for scenarios jobs: - template: ../common/templates/jobs/jobs.yml @@ -21,8 +22,8 @@ jobs: enablePublishBuildArtifacts: true helixRepo: dotnet/performance jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}' + - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.runtimeFlavor }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}' + displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.runtimeFlavor }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}' timeoutInMinutes: 320 variables: - name: AffinityValue # Make the affinity value available to the csproj generation @@ -158,7 +159,7 @@ jobs: $(Python) -m pip install urllib3==1.26.15 $(Python) -m pip install requests .\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet - dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog + dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -193,7 +194,7 @@ jobs: $(Python) -m pip install --user urllib3==1.26.15 $(Python) -m pip install --user requests . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet - dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -221,7 +222,7 @@ jobs: $(Python) -m pip install --user urllib3==1.26.15 $(Python) -m pip install --user requests . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet - dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -243,4 +244,5 @@ jobs: Creator: $(Creator) Architecture: ${{ parameters.architecture }} WorkItemTimeout: 4:00 # 4 hours - Python: $(Python) \ No newline at end of file + Python: $(Python) + RuntimeFlavor: ${{ parameters.runtimeFlavor }} \ No newline at end of file diff --git a/eng/performance/send-to-helix.yml b/eng/performance/send-to-helix.yml index 69e1b88b074..a37804673c3 100644 --- a/eng/performance/send-to-helix.yml +++ b/eng/performance/send-to-helix.yml @@ -23,9 +23,10 @@ parameters: condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false ProjectFile: helix.proj # optional -- override default project used to send benchmarks + RuntimeFlavor: '' # optional -- runtime flavor used for scenarios steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: Python: ${{ parameters.Python }} @@ -51,7 +52,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: Python: ${{ parameters.Python }} diff --git a/src/scenarios/mauiblazorios/pre.py b/src/scenarios/mauiblazorios/pre.py index 38ba650897a..ab781a42ade 100644 --- a/src/scenarios/mauiblazorios/pre.py +++ b/src/scenarios/mauiblazorios/pre.py @@ -72,8 +72,8 @@ remove_aab_files(output_dir) # Copy the MauiVersion to a file so we have it on the machine -maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versions_write_json(version_dict, rf"{output_dir}/versions.json") -print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") diff --git a/src/scenarios/mauiios/pre.py b/src/scenarios/mauiios/pre.py index fce66fe9bb1..7e85e2aa116 100644 --- a/src/scenarios/mauiios/pre.py +++ b/src/scenarios/mauiios/pre.py @@ -35,7 +35,7 @@ remove_aab_files(output_dir) # Copy the MauiVersion to a file so we have it on the machine -maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versions_write_json(version_dict, rf"{output_dir}/versions.json") -print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index 7fe274a490b..9bb21eb647f 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -91,6 +91,7 @@ def __init__(self): self.operation = args.operation self.framework = args.framework self.runtime_identifier = args.runtime + self.nativeaot = args.nativeaot self.msbuild = args.msbuild print(self.msbuild) self.msbuildstatic = args.msbuildstatic @@ -149,6 +150,10 @@ def add_common_arguments(self, parser: ArgumentParser): dest='runtime', metavar='runtime', help='runtime for build or publish - ex: win-x64') + parser.add_argument('-n', '--nativeaot', + dest='nativeaot', + metavar='nativeaot', + help='use Native AOT runtime for build or publish') parser.add_argument('--msbuild', dest='msbuild', metavar='msbuild', @@ -202,6 +207,9 @@ def execute(self, build_args: list = []): build_args.append('--self-contained') elif self.no_self_contained: build_args.append('--no-self-contained') + if self.nativeaot: + build_args.append('/p:PublishAot=true') + build_args.append('/p:PublishAotUsingRuntimePack=true') self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args) if self.operation == CROSSGEN: startup_args = [ diff --git a/src/scenarios/xamarinios/post.py b/src/scenarios/xamarinios/post.py new file mode 100644 index 00000000000..c87a49b4b7d --- /dev/null +++ b/src/scenarios/xamarinios/post.py @@ -0,0 +1,7 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories + +clean_directories() diff --git a/src/scenarios/xamarinios/pre.py b/src/scenarios/xamarinios/pre.py new file mode 100644 index 00000000000..4efc24aee4d --- /dev/null +++ b/src/scenarios/xamarinios/pre.py @@ -0,0 +1,41 @@ +''' +pre-command +''' +import shutil +import sys +import subprocess +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios +from test import EXENAME + +setup_loggers(True) + +precommands = PreCommands() +install_versioned_maui(precommands) + +# Setup the Xamarin folder +precommands.new(template='ios', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Build the APK +shutil.copy('./MauiNuGet.config', './app/Nuget.config') +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.xamarintesting']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the XamarinVersion to a file so we have it on the machine +xamarin_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll") +version_dict = { "xamarinVersion": xamarin_version } +versions_write_json(version_dict, rf"{output_dir}/versions.json") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll") diff --git a/src/scenarios/xamarinios/test.py b/src/scenarios/xamarinios/test.py new file mode 100644 index 00000000000..3be3df35083 --- /dev/null +++ b/src/scenarios/xamarinios/test.py @@ -0,0 +1,16 @@ +''' +Mobile Xamarin App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'XamariniOSDefault' + +if __name__ == "__main__": + versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run()