diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 791fb4fef6e..98467581667 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -303,7 +303,7 @@ jobs: channels: - release/6.0 - # Maui Android scenario benchmarks + # Maui Android scenario benchmarks - template: /eng/performance/scenarios.yml parameters: osName: windows @@ -318,6 +318,21 @@ jobs: channels: - release/6.0 + # Maui iOS scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: osx + architecture: x64 + osVersion: 12 + pool: + vmImage: 'macos-12' + queue: OSX.1015.Amd64.Iphone.Perf + machinePool: iPhoneMini12 + kind: maui_scenarios_ios + projectFile: maui_scenarios_ios.proj + channels: + - release/6.0 + # # Windows x64 micro benchmarks # - template: /eng/performance/benchmark_jobs.yml # parameters: diff --git a/eng/Versions.props b/eng/Versions.props index 57740387529..a8a38b0f60e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,6 +8,7 @@ 5.0.104-servicing.21213.7 + 1.0.0-prerelease.23157.1 5.0.4 diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj new file mode 100644 index 00000000000..2fd653ad6c2 --- /dev/null +++ b/eng/performance/maui_scenarios_ios.proj @@ -0,0 +1,161 @@ + + + + + true + + + + $(Python) post.py + scenarios_out + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + + + + + + + + + 00:30 + + + + + + mauiios + $(ScenariosDir)%(ScenarioDirectoryName) + MauiiOSDefault + net.dot.mauitesting + + + + + mauiblazorios + $(ScenariosDir)%(ScenarioDirectoryName) + MauiBlazoriOSDefault + net.dot.mauiblazortesting + + + + + + $(Python) pre.py publish -f $(_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub + $(Python) test.py sod --scenario-name "%(Identity)" + + + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip + $(Python) test.py sod --scenario-name "%(Identity)" + + + $(ScenariosDir)mauiios.zip + 00:15:00 + ios-device + + + + + + $(ScenariosDir)mauiblazorios.zip + 00:15:00 + ios-device + + + + + + + + + + + + export PYTHONPATH=$ORIGPYPATH;$(HelixPostCommands) + + + + + \ No newline at end of file diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 521fa389622..1b202636563 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -117,7 +117,34 @@ jobs: TargetsWindows: 'true' WorkItemDirectory: $(Build.SourcesDirectory) HelixTargetQueues: ${{ parameters.queue }} - - ${{ if ne(parameters.osName, 'windows') }}: + - ${{ if eq(parameters.osName, 'osx') }}: + - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps + displayName: Copy python libraries and NuGet.config + - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)startup -f $(_Framework) -r osx-${{parameters.architecture}} $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj + displayName: Build startup tool + env: + PERFLAB_TARGET_FRAMEWORKS: $(_Framework) + - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)SOD -f $(_Framework) -r osx-${{parameters.architecture}} $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj + displayName: Build SOD tool + env: + PERFLAB_TARGET_FRAMEWORKS: $(_Framework) + - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)FailureReporter -f $(_Framework) -r osx-${{parameters.architecture}} $(Build.SourcesDirectory)/src/tools/Reporting/FailureReporting/FailureReporting.csproj + displayName: Build FailureReporter tool + env: + PERFLAB_TARGET_FRAMEWORKS: $(_Framework) + - script: | + $(Python) -m pip install --user --upgrade pip + $(Python) -m pip install --user requests + . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + displayName: Prepare scenarios + env: + CorrelationPayloadDirectory: $(CorrelationStaging) + Architecture: ${{ parameters.architecture }} + TargetsWindows: 'false' + WorkItemDirectory: $(Build.SourcesDirectory) + HelixTargetQueues: ${{ parameters.queue }} + - ${{ if and(ne(parameters.osName, 'windows'), ne(parameters.osName, 'osx')) }}: - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps displayName: Copy python libraries and NuGet.config - script: $(CorrelationStaging)dotnet/dotnet publish -c Release -o $(CorrelationStaging)startup -f $(_Framework) -r linux-${{parameters.architecture}} $(Build.SourcesDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj diff --git a/scripts/performance/common.py b/scripts/performance/common.py index 7f15dffa627..a33dfdd2fa7 100644 --- a/scripts/performance/common.py +++ b/scripts/performance/common.py @@ -87,6 +87,12 @@ def helixuploadroot(): ''' return environ.get('HELIX_WORKITEM_UPLOAD_ROOT') +def helixworkitemroot(): + ''' + Returns the helix workitem root. Will be None outside of helix. + ''' + return environ.get('HELIX_WORKITEM_ROOT') + def runninginlab(): return environ.get('PERFLAB_INLAB') is not None diff --git a/src/scenarios/mauiblazorios/post.py b/src/scenarios/mauiblazorios/post.py new file mode 100644 index 00000000000..c87a49b4b7d --- /dev/null +++ b/src/scenarios/mauiblazorios/post.py @@ -0,0 +1,7 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories + +clean_directories() diff --git a/src/scenarios/mauiblazorios/pre.py b/src/scenarios/mauiblazorios/pre.py new file mode 100644 index 00000000000..5004182e2b5 --- /dev/null +++ b/src/scenarios/mauiblazorios/pre.py @@ -0,0 +1,78 @@ +''' +pre-command +''' +import shutil +import sys +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios +from test import EXENAME + +setup_loggers(True) +precommands = PreCommands() +install_versioned_maui(precommands) + +# Setup the Maui folder +precommands.new(template='maui-blazor', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Add the index.razor.cs file +with open(f"{const.APPDIR}/Pages/Index.razor.cs", "w") as indexCSFile: + indexCSFile.write(''' + using Microsoft.AspNetCore.Components; + #if ANDROID + using Android.App; + #endif\n\n''' + + f" namespace {EXENAME}.Pages" + +''' + { + public partial class Index + { + protected override void OnAfterRender(bool firstRender) + { + if (firstRender) + { + #if ANDROID + var activity = MainActivity.Context as Activity; + activity.ReportFullyDrawn(); + #else + System.Console.WriteLine(\"__MAUI_Blazor_WebView_OnAfterRender__\"); + #endif + } + } + } + } +''') + +# Replace line in the Android MainActivity.cs file +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "r") as mainActivityFile: + mainActivityFileLines = mainActivityFile.readlines() + +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "w") as mainActivityFile: + for line in mainActivityFileLines: + if line.startswith("{"): + mainActivityFile.write("{\npublic static Android.Content.Context Context { get; private set; }\npublic MainActivity() { Context = this; }") + else: + mainActivityFile.write(line) + +# Build the APK +# NuGet.config file cannot be in the build directory currently due to https://github.com/dotnet/aspnetcore/issues/41397 +# shutil.copy('./MauiNuGet.config', './app/Nuget.config') +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauiblazortesting']) + +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}/versions.json") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") diff --git a/src/scenarios/mauiblazorios/test.py b/src/scenarios/mauiblazorios/test.py new file mode 100644 index 00000000000..5c93ab0f95f --- /dev/null +++ b/src/scenarios/mauiblazorios/test.py @@ -0,0 +1,16 @@ +''' +Mobile Maui App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'MauiBlazoriOSDefault' + +if __name__ == "__main__": + versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/mauiios/pre.py b/src/scenarios/mauiios/pre.py index 23330182c75..5bdd0e11406 100644 --- a/src/scenarios/mauiios/pre.py +++ b/src/scenarios/mauiios/pre.py @@ -1,10 +1,41 @@ ''' pre-command ''' -from performance.logger import setup_loggers -from shutil import copytree -from shared.const import PUBDIR +import shutil +import sys +import subprocess +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios +from test import EXENAME setup_loggers(True) -copytree('app', PUBDIR) +precommands = PreCommands() +install_versioned_maui(precommands) + +# Setup the Maui folder +precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Build the APK +shutil.copy('./MauiNuGet.config', './app/Nuget.config') +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauitesting']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}/versions.json") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") \ No newline at end of file diff --git a/src/scenarios/mauiios/test.py b/src/scenarios/mauiios/test.py index 0aca010a88e..297c47ecef3 100644 --- a/src/scenarios/mauiios/test.py +++ b/src/scenarios/mauiios/test.py @@ -1,12 +1,16 @@ ''' -C# Console app +Mobile Maui App ''' +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiiOSDefault' -if __name__ == "__main__": +if __name__ == "__main__": + versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json") + traits = TestTraits(exename=EXENAME, guiapp='false', ) - Runner(traits).run() + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/mauiiospodcast/post.py b/src/scenarios/mauiiospodcast/post.py new file mode 100644 index 00000000000..0944f25b78d --- /dev/null +++ b/src/scenarios/mauiiospodcast/post.py @@ -0,0 +1,9 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories +from performance.common import remove_directory + +remove_directory("dotnet-podcasts") +clean_directories() \ No newline at end of file diff --git a/src/scenarios/mauiiospodcast/pre.py b/src/scenarios/mauiiospodcast/pre.py new file mode 100644 index 00000000000..401283164c6 --- /dev/null +++ b/src/scenarios/mauiiospodcast/pre.py @@ -0,0 +1,34 @@ +''' +pre-command +''' +import subprocess +from performance.logger import setup_loggers, getLogger +from shared.precommands import PreCommands +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell +from shared import const + +setup_loggers(True) +precommands = PreCommands() +install_versioned_maui(precommands) + +branch = f'{precommands.framework[:6]}' +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1']) +subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately + +precommands.existing(projectdir='./dotnet-podcasts', projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') + +# Build the APK +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.netconf2021.maui']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +# maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\android-arm64\linked\Microsoft.Maui.dll") +# version_dict = { "mauiVersion": maui_version } +# versions_write_json(version_dict, rf"{output_dir}\versions.json") +# print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiiospodcast/test.py b/src/scenarios/mauiiospodcast/test.py new file mode 100644 index 00000000000..d02b07482d5 --- /dev/null +++ b/src/scenarios/mauiiospodcast/test.py @@ -0,0 +1,16 @@ +''' +Mobile Maui App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'MauiiOSPodcast' + +if __name__ == "__main__": + #versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 3226827acaf..b39385e2ac5 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -8,21 +8,22 @@ import glob import re import time +from datetime import datetime +import json from logging import getLogger -from collections import namedtuple from argparse import ArgumentParser from argparse import RawTextHelpFormatter -from io import StringIO -from shutil import move +from shutil import rmtree from shared.crossgen import CrossgenArguments from shared.startup import StartupWrapper from shared.util import publishedexe, pythoncommand, appfolder, xharnesscommand from shared.sod import SODWrapper from shared import const -from performance.common import RunCommand, iswin, extension +from performance.common import RunCommand, iswin, extension, helixworkitemroot from performance.logger import setup_loggers from shared.testtraits import TestTraits, testtypes +from subprocess import CalledProcessError class Runner: diff --git a/src/scenarios/shared/versionmanager.py b/src/scenarios/shared/versionmanager.py index 3eeaf795a5a..170c0230e72 100644 --- a/src/scenarios/shared/versionmanager.py +++ b/src/scenarios/shared/versionmanager.py @@ -28,4 +28,8 @@ def versions_read_json_file_save_env(inputfile = 'versions.json'): def get_version_from_dll_powershell(dll_path: str): result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + return result.stdout.decode('utf-8').strip() + +def get_version_from_dll_powershell_ios(dll_path: str): + result = subprocess.run(['pwsh', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False) return result.stdout.decode('utf-8').strip() \ No newline at end of file