Add Maui Mobile Testing to the perf repo (port from release/7.0 Test)

azure-pipelines.yml

@@ -273,6 +273,19 @@ jobs:
         channels:
           - main
 
+  # Maui Android scenario benchmarks
+  - template: /eng/performance/build_machine_matrix.yml
+    parameters:
+      jobTemplate: /eng/performance/scenarios.yml
+      buildMachines:
+        - win-x64-android-arm64
+      isPublic: false
+      jobParameters:
+        kind: maui_scenarios_android
+        projectFile: maui_scenarios_android.proj
+        dotnetVersionsLinks:
+          main: https://aka.ms/dotnet/sdk/maui/net8.0.json
+
   ## Maui scenario benchmarks
   #- template: /eng/performance/build_machine_matrix.yml
   #  parameters:

eng/performance/build_machine_matrix.yml

@@ -91,4 +91,16 @@ jobs:
         vmImage: windows-2019
       machinePool: Tiger
       queue: Windows.10.Arm64.Perf.Surf
-      ${{ insert }}: ${{ parameters.jobParameters }}
+      ${{ insert }}: ${{ parameters.jobParameters }}
+
+- ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently
+  - template: ${{ parameters.jobTemplate }}
+    parameters:
+      osName: windows
+      architecture: x64
+      osVersion: 19H1
+      pool:
+        vmImage: 'windows-2022'
+      queue: Windows.10.Amd64.Pixel.Perf
+      machinePool: Pixel
+      ${{ insert }}: ${{ parameters.jobParameters }}

eng/performance/maui_scenarios_android.proj

@@ -0,0 +1,82 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <Import Project="Scenarios.Common.props" />
+
+  <PropertyGroup>
+    <IncludeXHarnessCli>true</IncludeXHarnessCli>
+    <MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.21566.2</MicrosoftDotNetXHarnessCLIVersion>
+    <XharnessPath>%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll</XharnessPath>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
+    <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
+  </PropertyGroup>
+
+  <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
+    <Message Text="Removing Dotnet Packs from Correlation Staging" Importance="high" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" />
+  </Target>
+
+  <ItemDefinitionGroup>
+    <HelixWorkItem>
+      <Timeout>00:30</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <MAUIAndroidScenario Include="Maui Android Default">
+      <ScenarioDirectoryName>mauiandroid</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.companyname.mauiandroiddefault-Signed</ApkName>
+      <PackageName>com.companyname.mauiandroiddefault</PackageName>
+    </MAUIAndroidScenario>
+    <MAUIAndroidScenario Include="Maui Android Podcast">
+      <ScenarioDirectoryName>mauiandroidpodcast</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.Microsoft.NetConf2021.Maui-Signed</ApkName>
+      <PackageName>com.Microsoft.NetConf2021.Maui</PackageName>
+    </MAUIAndroidScenario>
+    <MAUIAndroidScenario Include="Maui Blazor Android Default">
+      <ScenarioDirectoryName>mauiblazorandroid</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.companyname.mauiblazorandroiddefault-Signed</ApkName>
+      <PackageName>com.companyname.mauiblazorandroiddefault</PackageName>
+    </MAUIAndroidScenario>
+  </ItemGroup>
+
+
+  <ItemGroup>
+    <PreparePayloadWorkItem Include="@(MAUIAndroidScenario)">
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+
+<!-- We only run the android tests from Windows machines (at least for now) -->
+  <ItemGroup>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) APK Size')">
+      <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) Extracted Size')">
+      <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "&amp; {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity)')">
+      <PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity) NoAnimation')">
+      <PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot; --disable-animations</Command>
+    </HelixWorkItem>
+  </ItemGroup>
+
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+</Project>

eng/performance/scenarios.yml

@@ -6,7 +6,8 @@ parameters:
   pool: ''              # required -- name of the Helix pool
   queue: ''             # required -- name of the Helix queue
   container: ''         # optional -- id of the container
-  channels: []          # required -- list of channels to download .NET from
+  channels: []          # optional (must have dotnetVersionsLinks if not used) -- list of channels to download .NET from
+  dotnetVersionsLinks: [] # optional alternative to channels that uses 'channel: link' values to scrape the link's json for dotnet_version or version
   projectFile: ''       # required -- project file to build (current choices: scenarios.proj/sdk_scenarios.proj )
   machinePool: ''       # required -- Name of perf machine pool (Tiger, Owl, etc)
 
@@ -81,16 +82,37 @@ jobs:
         container: ${{ parameters.container }}
         strategy:
           matrix:
-            ${{ each channel in parameters.channels }}:
-              ${{ channel }}:
-                _Channel: ${{ channel }}
-                _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}"
-                _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net
+            ${{ if ne(length(parameters.channels), 0) }}:
+              ${{ each channel in parameters.channels }}:
+                ${{ channel }}:
+                  _Channel: ${{ channel }}
+                  _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}"
+                  _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net
+            ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}:
+              ${{ each versionPair in parameters.dotnetVersionsLinks }}:
+                ${{ versionPair.key }}_Link:
+                  _Channel: ${{ versionPair.key }}
+                  _DotnetVersionLink: ${{ versionPair.value }}
+                  _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}"
+                  _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_Linked_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net
         steps:
         - checkout: self
           clean: true
-        - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet
-          displayName: Run ci_setup.py
+        - ${{ if ne(length(parameters.channels), 0) }}:
+          - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet
+            displayName: Run ci_setup.py
+        - ${{ elseif ne(length(parameters.dotnetVersionsLinks), 0) }}:
+          - powershell: |
+              $data = Invoke-WebRequest -Uri "$(_DotnetVersionLink)"
+              $values = ConvertFrom-Json $data.content
+              $dotnet_version_check = $values.dotnet_version
+              $version_check = $values.version
+              $dotnet_version = if ($dotnet_version_check) {$dotnet_version_check} Else {$version_check}
+              Write-Host "##vso[task.setvariable variable=DotnetVersion;]$dotnet_version"
+              Write-Host "Found dotnet version $dotnet_version"
+            displayName: Get dotnetVersion to use
+          - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --dotnet-versions $(DotnetVersion) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet
+            displayName: Run ci_setup.py with dotnetVersionsLinks
         - ${{ if eq(parameters.osName, 'windows') }}:
           - script: xcopy .\NuGet.config $(CorrelationStaging) && xcopy .\scripts $(CorrelationStaging)scripts\/e && xcopy .\src\scenarios\shared $(CorrelationStaging)shared\/e && xcopy .\src\scenarios\staticdeps $(CorrelationStaging)staticdeps\/e
             displayName: Copy python libraries and NuGet.config

scripts/dotnet.py

@@ -5,13 +5,14 @@
 """
 
 import ssl
+import datetime
 from argparse import Action, ArgumentParser, ArgumentTypeError, ArgumentError
 from collections import namedtuple
 from glob import iglob
 from json import loads
 from logging import getLogger
 from os import chmod, environ, listdir, makedirs, path, pathsep, system
-from re import search
+from re import search, match, MULTILINE
 from shutil import rmtree
 from stat import S_IRWXU
 from subprocess import CalledProcessError, check_output
@@ -619,27 +620,30 @@ def get_commit_date(
     if not commit_sha:
         raise ValueError('.NET Commit sha was not defined.')
 
+    # Example URL: https://github.com/dotnet/runtime/commit/2d76178d5faa97be86fc8d049c7dbcbdf66dc497.patch
     url = None
-    urlformat = 'https://api.github.com/repos/%s/%s/commits/%s'
     if repository is None:
         # The origin of the repo where the commit belongs to has changed
         # between release. Here we attempt to naively guess the repo.
         core_sdk_frameworks = ChannelMap.get_supported_frameworks()
         repo = 'core-sdk' if framework  in core_sdk_frameworks else 'cli'
-        url = urlformat % ('dotnet', repo, commit_sha)
+        url = f'https://github.com/dotnet/{repo}/commit/{commit_sha}.patch'
     else:
         owner, repo = get_repository(repository)
-        url = urlformat % (owner, repo, commit_sha)
+        url = f'https://github.com/{owner}/{repo}/commit/{commit_sha}.patch'
 
     build_timestamp = None
-    sleep_time = 10 # Start with 10 second sleep timer
+    sleep_time = 10 # Start with 10 second sleep timer        
     for retrycount in range(5):
         try:
             with urlopen(url) as response:
                 getLogger().info("Commit: %s", url)
-                item = loads(response.read().decode('utf-8'))
-                build_timestamp = item['commit']['committer']['date']
-                break
+                patch = response.read().decode('utf-8')
+                dateMatch = search(r'^Date: (.+)$', patch, MULTILINE)
+                if dateMatch:
+                    build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+                    getLogger().info(f"Got UTC timestamp {build_timestamp} from {dateMatch.group(1)}")
+                    break
         except URLError as error:
             getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}")
             sleep(sleep_time)

src/scenarios/mauiandroid/pre.py

@@ -2,48 +2,48 @@
 pre-command
 '''
 import sys
-import os
-from zipfile import ZipFile
+import requests
+from mauishared.mauisharedpython import RemoveAABFiles
 from performance.logger import setup_loggers, getLogger
-from shutil import copyfile
+from shared import const
 from shared.precommands import PreCommands
-from shared.const import PUBDIR
-from argparse import ArgumentParser
+from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell
+from test import EXENAME
 
 setup_loggers(True)
 
-parser = ArgumentParser()
-parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False)
-parser.add_argument(
-        '--apk-name',
-        dest='apk',
-        required=True,
-        type=str,
-        help='Name of the APK to setup')
-args = parser.parse_args()
-
-if not os.path.exists(PUBDIR):
-    os.mkdir(PUBDIR)
-apkname = args.apk
-apknamezip = '%s.zip' % (apkname)
-if not os.path.exists(apkname):
-    getLogger().error('Cannot find %s' % (apkname))
-    exit(-1)
-if args.unzip:
-    if not os.path.exists(apknamezip):
-        copyfile(apkname, apknamezip)
-
-    with ZipFile(apknamezip) as zip:
-        zip.extractall(os.path.join('.', PUBDIR))
-
-    assets_dir = os.path.join(PUBDIR, 'assets')
-    assets_zip = os.path.join(assets_dir, 'assets.zip')
-    with ZipFile(assets_zip) as zip:
-        zip.extractall(assets_dir)
-
-    os.remove(assets_zip)
-else:
-    copyfile(apkname, os.path.join(PUBDIR, apkname))
-
-
-
+precommands = PreCommands()
+target_framework_wo_platform = precommands.framework.split('-')[0]
+
+# Download what we need
+with open ("MauiNuGet.config", "wb") as f:
+    f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content)
+
+workload_install_args = ['--configfile', 'MauiNuGet.config']
+if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7
+    workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json']
+
+precommands.install_workload('maui', workload_install_args) 
+
+# Setup the Maui folder
+precommands.new(template='maui',
+                output_dir=const.APPDIR,
+                bin_dir=const.BINDIR,
+                exename=EXENAME,
+                working_directory=sys.path[0],
+                no_restore=False)
+
+# Build the APK
+precommands.execute(['--no-restore', '--source', 'MauiNuGet.config'])
+
+# Remove the aab files as we don't need them, this saves space
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+RemoveAABFiles(output_dir)
+
+# Copy the MauiVersion to a file so we have it on the machine
+maui_version = GetVersionFromDllPowershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
+version_dict = { "mauiVersion": maui_version }
+versionswritejson(version_dict, rf"{output_dir}\versions.json")
+print(f"Versions: {version_dict}")

src/scenarios/mauiandroid/test.py

@@ -1,11 +1,15 @@
 '''
-C# Console app
+Mobile Maui App
 '''
+from shared.const import PUBDIR
 from shared.runner import TestTraits, Runner
+from shared.versionmanager import versionsreadjsonfilesaveenv
 
 EXENAME = 'MauiAndroidDefault'
 
-if __name__ == "__main__":
+if __name__ == "__main__":    
+    versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json")
+
     traits = TestTraits(exename=EXENAME, 
                         guiapp='false',
                         )

src/scenarios/mauiandroidpodcast/post.py

@@ -0,0 +1,9 @@
+'''
+post cleanup script
+'''
+
+from shared.postcommands import clean_directories
+from performance.common import remove_directory
+
+remove_directory("dotnet-podcasts")
+clean_directories()

src/scenarios/mauiandroidpodcast/pre.py

@@ -0,0 +1,45 @@
+'''
+pre-command
+'''
+import requests
+import subprocess
+from mauishared.mauisharedpython import RemoveAABFiles
+from performance.logger import setup_loggers, getLogger
+from shared.precommands import PreCommands
+from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell
+from shared import const
+
+setup_loggers(True)
+precommands = PreCommands()
+target_framework_wo_platform = precommands.framework.split('-')[0]
+
+# Download what we need
+with open ("MauiNuGet.config", "wb") as f:
+    f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content)
+
+branch = f'{precommands.framework[:6]}'
+subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1'])
+subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately
+
+workload_install_args = ['--configfile', 'MauiNuGet.config']
+if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7
+    workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json']
+
+precommands.install_workload('maui', workload_install_args) 
+precommands.existing(projectdir='./dotnet-podcasts',projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj')
+
+# Build the APK
+precommands._restore()
+precommands.execute(['--no-restore'])
+
+# Remove the aab files as we don't need them, this saves space
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+RemoveAABFiles(output_dir)
+
+# Copy the MauiVersion to a file so we have it on the machine
+maui_version = GetVersionFromDllPowershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
+version_dict = { "mauiVersion": maui_version }
+versionswritejson(version_dict, rf"{output_dir}\versions.json")
+print(f"Versions: {version_dict}")