From 7a9eec046aecdcea206f627e60a0063fc510b93a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 5 Dec 2022 14:59:00 -0800 Subject: [PATCH 01/48] First round of adding Maui Android testing to the performance pipeline. --- eng/performance/maui_scenarios_android.proj | 68 +++++++++++++++ src/scenarios/mauiandroid/pre.py | 69 +++++++-------- src/scenarios/mauiblazorandroid/pre.py | 94 +++++++++++++++------ 3 files changed, 170 insertions(+), 61 deletions(-) create mode 100644 eng/performance/maui_scenarios_android.proj diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj new file mode 100644 index 00000000000..b280c47dfdd --- /dev/null +++ b/eng/performance/maui_scenarios_android.proj @@ -0,0 +1,68 @@ + + + + + true + 1.0.0-prerelease.21566.2 + %HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll + + + + $(Python) post.py + scenarios_out + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + + + + + mauiandroid + $(ScenariosDir)%(ScenarioDirectoryName) + com.companyname.mauiandroiddefault-Signed.apk + com.companyname.mauiandroiddefault + + + mauiblazorandroid + $(ScenariosDir)%(ScenarioDirectoryName) + com.companyname.mauiblazorandroiddefault-Signed.apk + com.companyname.mauiblazorandroiddefault + + + + + + + $(Python) pre.py -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\$(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + $(Python) test.py sod --scenario-name "%(Identity)" + + + + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\$(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + + + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\$(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + + + + + + + diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index aed0d76c750..7b84c1c29ad 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -3,47 +3,48 @@ ''' import sys import os +import requests from zipfile import ZipFile from performance.logger import setup_loggers, getLogger from shutil import copyfile +from shared import const from shared.precommands import PreCommands -from shared.const import PUBDIR from argparse import ArgumentParser +from test import EXENAME setup_loggers(True) -parser = ArgumentParser() -parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False) +parser = ArgumentParser(add_help=False) parser.add_argument( - '--apk-name', - dest='apk', - required=True, + '-o', '--output', + dest='output_dir', + required=False, type=str, - help='Name of the APK to setup') -args = parser.parse_args() - -if not os.path.exists(PUBDIR): - os.mkdir(PUBDIR) -apkname = args.apk -apknamezip = '%s.zip' % (apkname) -if not os.path.exists(apkname): - getLogger().error('Cannot find %s' % (apkname)) - exit(-1) -if args.unzip: - if not os.path.exists(apknamezip): - copyfile(apkname, apknamezip) - - with ZipFile(apknamezip) as zip: - zip.extractall(os.path.join('.', PUBDIR)) - - assets_dir = os.path.join(PUBDIR, 'assets') - assets_zip = os.path.join(assets_dir, 'assets.zip') - with ZipFile(assets_zip) as zip: - zip.extractall(assets_dir) - - os.remove(assets_zip) -else: - copyfile(apkname, os.path.join(PUBDIR, apkname)) - - - + help='capture of the output directory') +args, unknown_args = parser.parse_known_args() + +# Download what we need +with open ("MauiNuGet.config", "wb") as f: + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) + +precommands = PreCommands() +precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) + +# Setup the Maui folder +precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Build the APK +precommands.execute(['--source', 'MauiNuget.config']) + +# Remove the aab files as we don't need them, this saves space +output_file_partial_path = f"com.companyname.{str.lower(EXENAME)}" +if args.output_dir: + output_file_partial_path = os.path.join(args.output_dir, output_file_partial_path) + +os.remove(f"{output_file_partial_path}-Signed.aab") +os.remove(f"{output_file_partial_path}.aab") diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index aed0d76c750..b85e7ac9cad 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -3,47 +3,87 @@ ''' import sys import os +import requests from zipfile import ZipFile from performance.logger import setup_loggers, getLogger from shutil import copyfile +from shared import const from shared.precommands import PreCommands -from shared.const import PUBDIR from argparse import ArgumentParser +from test import EXENAME setup_loggers(True) -parser = ArgumentParser() -parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False) +parser = ArgumentParser(add_help=False) parser.add_argument( - '--apk-name', - dest='apk', - required=True, + '-o', '--output', + dest='output_dir', + required=False, type=str, - help='Name of the APK to setup') -args = parser.parse_args() + help='capture of the output directory') +args, unknown_args = parser.parse_known_args() -if not os.path.exists(PUBDIR): - os.mkdir(PUBDIR) -apkname = args.apk -apknamezip = '%s.zip' % (apkname) -if not os.path.exists(apkname): - getLogger().error('Cannot find %s' % (apkname)) - exit(-1) -if args.unzip: - if not os.path.exists(apknamezip): - copyfile(apkname, apknamezip) +# Download what we need +with open ("MauiNuGet.config", "wb") as f: + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) - with ZipFile(apknamezip) as zip: - zip.extractall(os.path.join('.', PUBDIR)) +precommands = PreCommands() +precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) - assets_dir = os.path.join(PUBDIR, 'assets') - assets_zip = os.path.join(assets_dir, 'assets.zip') - with ZipFile(assets_zip) as zip: - zip.extractall(assets_dir) +# Setup the Maui folder +precommands.new(template='maui-blazor', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) - os.remove(assets_zip) -else: - copyfile(apkname, os.path.join(PUBDIR, apkname)) +# Add the index.razor.cs file +with open(f"{const.APPDIR}/Pages/Index.razor.cs", "w") as indexCSFile: + indexCSFile.write(''' + using Microsoft.AspNetCore.Components; + #if ANDROID + using Android.App; + #endif\n\n''' + + f" namespace {EXENAME}.Pages" + +''' + { + public partial class Index + { + protected override void OnAfterRender(bool firstRender) + { + if (firstRender) + { + #if ANDROID + var activity = MainActivity.Context as Activity; + activity.ReportFullyDrawn(); + #else + System.Console.WriteLine(\"__MAUI_Blazor_WebView_OnAfterRender__\"); + #endif + } + } + } + } +''') + +# Replace line in the Android MainActivity.cs file +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "r") as mainActivityFile: + mainActivityFileLines = mainActivityFile.readlines() + +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "w") as mainActivityFile: + for line in mainActivityFileLines: + if line.startswith("{"): + mainActivityFile.write("{\npublic static Android.Content.Context Context { get; private set; }\npublic MainActivity() { Context = this; }") + else: + mainActivityFile.write(line) +# Build the APK +precommands.execute(['--no-restore', '--source', 'MauiNuget.config']) +# Remove the aab files as we don't need them, this saves space +output_file_partial_path = f"com.companyname.{str.lower(EXENAME)}" +if args.output_dir: + output_file_partial_path = os.path.join(args.output_dir, output_file_partial_path) +os.remove(f"{output_file_partial_path}-Signed.aab") +os.remove(f"{output_file_partial_path}.aab") From d9c9421b1cb947dcdb2fc60a4d21381139ee268b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 5 Dec 2022 15:27:22 -0800 Subject: [PATCH 02/48] Add testing run to yaml for running the android tests. --- azure-pipelines.yml | 31 ++++++++------------- eng/performance/build_machine_matrix.yml | 14 +++++++++- eng/performance/maui_scenarios_android.proj | 1 + 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 868448437c2..8958a5eebd8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -370,25 +370,18 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + # Maui scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-android-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + channels: + - release/7.0 ################################################ # Scheduled Private jobs (Weekly) diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/build_machine_matrix.yml index f525325cea7..da05dbd893b 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/performance/build_machine_matrix.yml @@ -91,4 +91,16 @@ jobs: vmImage: windows-2019 machinePool: Tiger queue: Windows.10.Arm64.Perf.Surf - ${{ insert }}: ${{ parameters.jobParameters }} \ No newline at end of file + ${{ insert }}: ${{ parameters.jobParameters }} + +- ${{ if and(containsValue(parameters.buildMachines, 'win-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently + - template: ${{ parameters.jobTemplate }} + parameters: + osName: windows + architecture: x64 + osVersion: 21H1 + pool: + vmImage: windows-2019 + queue: Windows.10.Amd64.Pixel.Perf + machinePool: Pixel + ${{ insert }}: ${{ parameters.jobParameters }} diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index b280c47dfdd..c2984d39cb2 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -1,6 +1,7 @@ + true 1.0.0-prerelease.21566.2 From cc2f11943351f5e81918f1b3a04b77a586e3f20e Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 5 Dec 2022 15:37:33 -0800 Subject: [PATCH 03/48] Fix incorrect $/% mixup in the maui_scenarios_android.proj and add publish for pre.py. --- eng/performance/maui_scenarios_android.proj | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index c2984d39cb2..4c3992708e1 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -1,7 +1,7 @@ - + true 1.0.0-prerelease.21566.2 @@ -34,7 +34,7 @@ - $(Python) pre.py -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained + $(Python) pre.py publish -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained %(PreparePayloadWorkItem.PayloadDirectory) @@ -42,24 +42,26 @@ - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\$(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\$(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub - $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + $(Python) test.py devicestartup --device-type android --package-path pub/%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\$(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub - $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + $(Python) test.py devicestartup --device-type android --package-path pub/%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations From 135b715afa5f548599312357940969fe150835e3 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 5 Dec 2022 16:08:01 -0800 Subject: [PATCH 04/48] Add debugging print variables and add no restore to mauiandroid test. --- src/scenarios/mauiandroid/pre.py | 4 +++- src/scenarios/mauiblazorandroid/pre.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 7b84c1c29ad..9c4ef173f30 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -23,9 +23,11 @@ help='capture of the output directory') args, unknown_args = parser.parse_known_args() +getLogger().info("Current directory: " + os.getcwd()) # Download what we need with open ("MauiNuGet.config", "wb") as f: f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) + getLogger().info("Downloaded MauiNuGet.config") precommands = PreCommands() precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) @@ -39,7 +41,7 @@ no_restore=False) # Build the APK -precommands.execute(['--source', 'MauiNuget.config']) +precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) # Remove the aab files as we don't need them, this saves space output_file_partial_path = f"com.companyname.{str.lower(EXENAME)}" diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index b85e7ac9cad..d8fa2d8b76d 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -23,9 +23,11 @@ help='capture of the output directory') args, unknown_args = parser.parse_known_args() +getLogger().info("Current directory: " + os.getcwd()) # Download what we need with open ("MauiNuGet.config", "wb") as f: f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) + getLogger().info("Downloaded MauiNuGet.config") precommands = PreCommands() precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) @@ -78,7 +80,7 @@ mainActivityFile.write(line) # Build the APK -precommands.execute(['--no-restore', '--source', 'MauiNuget.config']) +precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) # Remove the aab files as we don't need them, this saves space output_file_partial_path = f"com.companyname.{str.lower(EXENAME)}" From 8ca2dedbc573bb24e4533b489c4b8c54f076545e Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 6 Dec 2022 10:20:56 -0800 Subject: [PATCH 05/48] Try using MacOS to build the maui apps, then run on the android phones. --- azure-pipelines.yml | 2 +- eng/performance/build_machine_matrix.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8958a5eebd8..54470b7e1e0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -375,7 +375,7 @@ jobs: parameters: jobTemplate: /eng/performance/scenarios.yml buildMachines: - - win-android-arm64 + - mac-android-arm64 isPublic: false jobParameters: kind: maui_scenarios_android diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/build_machine_matrix.yml index da05dbd893b..60499be6773 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/performance/build_machine_matrix.yml @@ -93,14 +93,14 @@ jobs: queue: Windows.10.Arm64.Perf.Surf ${{ insert }}: ${{ parameters.jobParameters }} -- ${{ if and(containsValue(parameters.buildMachines, 'win-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently +- ${{ if and(containsValue(parameters.buildMachines, 'mac-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 - osVersion: 21H1 + osName: mac + architecture: arm64 + osVersion: Darwin pool: - vmImage: windows-2019 + vmImage: 'macos-12' queue: Windows.10.Amd64.Pixel.Perf machinePool: Pixel ${{ insert }}: ${{ parameters.jobParameters }} From 2306cbd83cdb22c0c41bacab3a848515cedfe443 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 6 Dec 2022 10:59:57 -0800 Subject: [PATCH 06/48] Try switch from $ to % for variabile interpolation for non-windows precommands. --- eng/performance/maui_scenarios_android.proj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 4c3992708e1..af835417904 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -43,7 +43,7 @@ xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" $(Python) test.py devicestartup --device-type android --package-path pub/%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/$(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations $(Python) test.py devicestartup --device-type android --package-path pub/%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations From 8b5d419244216af8efe6d31c581e5493a7ee1f58 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 6 Dec 2022 14:56:00 -0800 Subject: [PATCH 07/48] Remove non-windows run commands since all the android devices are currently connected to windows. --- eng/performance/maui_scenarios_android.proj | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index af835417904..71bff3fec0a 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -39,29 +39,23 @@ - + - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y $(Python) test.py sod --scenario-name "%(Identity)" - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub - $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" - $(Python) test.py devicestartup --device-type android --package-path pub/%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ApkName) $HELIX_WORKITEM_ROOT/pub - $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations - $(Python) test.py devicestartup --device-type android --package-path pub/%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations From 3ab14c614e789bdcfdf4a6179adeaa64ccacb1b5 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 6 Dec 2022 14:56:27 -0800 Subject: [PATCH 08/48] Setup testing upload of the corelation staging to see how big it is. --- eng/performance/scenarios.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 2e0bf2ea3f1..8a41547187e 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -152,6 +152,22 @@ jobs: TargetsWindows: 'false' WorkItemDirectory: $(Build.SourcesDirectory) HelixTargetQueues: ${{ parameters.queue }} + + # Upload the correlation staging + - task: ArchiveFiles@2 + displayName: 'Zip Correlation Staging' + inputs: + rootFolderOrFile: $(CorrelationStaging) + archiveFile: $(Build.StagingDirectory)/CorrelationStaging.zip + archiveType: 'zip' + includeRootFolder: true + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Correlation Staging' + inputs: + pathtoPublish: $(Build.StagingDirectory)/CorrelationStaging.zip + artifactName: CorrelationStaging + - template: /eng/performance/send-to-helix.yml parameters: HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ From 6d60691cf3a69c340f81a4d25510852e8d66a0b4 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 10:25:37 -0800 Subject: [PATCH 09/48] Try removing the dotnet directory since we don't need it and it is too big. --- eng/performance/maui_scenarios_android.proj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 71bff3fec0a..6a3a2549e83 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -15,6 +15,10 @@ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + + + From 98dbe75d0dadd7ac26f13d7060825adb1c536b92 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 12:06:45 -0800 Subject: [PATCH 10/48] Print the current rate limit for debugging purposes. --- scripts/dotnet.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 77010e0e5c8..31dd0bbabe6 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -625,6 +625,9 @@ def get_commit_date( build_timestamp = None sleep_time = 10 # Start with 10 second sleep timer + # Get current rate limit for testing + with urlopen("https://api.github.com/rate_limit") as response: + getLogger().info(response.read().decode('utf-8')) for retrycount in range(5): try: with urlopen(url) as response: From 811a61667ee56ea65e318fa811b910f5f433f7c3 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 13:58:08 -0800 Subject: [PATCH 11/48] Revert "Setup testing upload of the corelation staging to see how big it is." This reverts commit 3ab14c614e789bdcfdf4a6179adeaa64ccacb1b5. --- eng/performance/scenarios.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 8a41547187e..2e0bf2ea3f1 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -152,22 +152,6 @@ jobs: TargetsWindows: 'false' WorkItemDirectory: $(Build.SourcesDirectory) HelixTargetQueues: ${{ parameters.queue }} - - # Upload the correlation staging - - task: ArchiveFiles@2 - displayName: 'Zip Correlation Staging' - inputs: - rootFolderOrFile: $(CorrelationStaging) - archiveFile: $(Build.StagingDirectory)/CorrelationStaging.zip - archiveType: 'zip' - includeRootFolder: true - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Correlation Staging' - inputs: - pathtoPublish: $(Build.StagingDirectory)/CorrelationStaging.zip - artifactName: CorrelationStaging - - template: /eng/performance/send-to-helix.yml parameters: HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ From 6dc0a0cc153271976246bc57549bcb00c539aa57 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 13:59:58 -0800 Subject: [PATCH 12/48] Remove debugging print statements, adding no restore fixed the issue. --- src/scenarios/mauiandroid/pre.py | 2 -- src/scenarios/mauiblazorandroid/pre.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 9c4ef173f30..62c3581a5c8 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -23,11 +23,9 @@ help='capture of the output directory') args, unknown_args = parser.parse_known_args() -getLogger().info("Current directory: " + os.getcwd()) # Download what we need with open ("MauiNuGet.config", "wb") as f: f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) - getLogger().info("Downloaded MauiNuGet.config") precommands = PreCommands() precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index d8fa2d8b76d..c439ae9c927 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -23,11 +23,9 @@ help='capture of the output directory') args, unknown_args = parser.parse_known_args() -getLogger().info("Current directory: " + os.getcwd()) # Download what we need with open ("MauiNuGet.config", "wb") as f: f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) - getLogger().info("Downloaded MauiNuGet.config") precommands = PreCommands() precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) From 17df5a33d7b4d78ebc2b62001527ae09f762d3d6 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 15:50:11 -0800 Subject: [PATCH 13/48] Updated the get_commit_date method to use the github patches instead of github API to get the commit date. --- scripts/dotnet.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 31dd0bbabe6..ca429c0240f 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -5,13 +5,14 @@ """ import ssl +import datetime from argparse import Action, ArgumentParser, ArgumentTypeError, ArgumentError from collections import namedtuple from glob import iglob from json import loads from logging import getLogger from os import chmod, environ, listdir, makedirs, path, pathsep, system -from re import search +from re import search, match, MULTILINE from shutil import rmtree from stat import S_IRWXU from subprocess import CalledProcessError, check_output @@ -612,7 +613,7 @@ def get_commit_date( raise ValueError('.NET Commit sha was not defined.') url = None - urlformat = 'https://api.github.com/repos/%s/%s/commits/%s' + urlformat = 'https://github.com/%s/%s/commit/%s.patch' if repository is None: # The origin of the repo where the commit belongs to has changed # between release. Here we attempt to naively guess the repo. @@ -624,17 +625,17 @@ def get_commit_date( url = urlformat % (owner, repo, commit_sha) build_timestamp = None - sleep_time = 10 # Start with 10 second sleep timer - # Get current rate limit for testing - with urlopen("https://api.github.com/rate_limit") as response: - getLogger().info(response.read().decode('utf-8')) + sleep_time = 10 # Start with 10 second sleep timer for retrycount in range(5): try: with urlopen(url) as response: getLogger().info("Commit: %s", url) - item = loads(response.read().decode('utf-8')) - build_timestamp = item['commit']['committer']['date'] - break + patch = response.read().decode('utf-8') + dateMatch = search(r'^Date: (.+)$', patch, MULTILINE) + if dateMatch: + getLogger().info("Match: %s", dateMatch) + build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + break except URLError as error: getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}") sleep(sleep_time) From 6f26422f33f3001d5708d8d1d4876b1bb5673669 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 15:57:42 -0800 Subject: [PATCH 14/48] Fix dotnet.py get_commit_time Logger information. --- scripts/dotnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dotnet.py b/scripts/dotnet.py index ca429c0240f..938937b22ef 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -633,8 +633,8 @@ def get_commit_date( patch = response.read().decode('utf-8') dateMatch = search(r'^Date: (.+)$', patch, MULTILINE) if dateMatch: - getLogger().info("Match: %s", dateMatch) build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + getLogger().info(f"Got UTC timestamp {build_timestamp} from {dateMatch.group(1)}") break except URLError as error: getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}") From dc61f727b353585e23db20f5822c235ee6cbe05a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 16:21:18 -0800 Subject: [PATCH 15/48] Change osName to be the run machine, not build machine. --- eng/performance/build_machine_matrix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/build_machine_matrix.yml index 60499be6773..58c985f294b 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/performance/build_machine_matrix.yml @@ -96,9 +96,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'mac-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: mac - architecture: arm64 - osVersion: Darwin + osName: windows + architecture: x64 + osVersion: 19H1 pool: vmImage: 'macos-12' queue: Windows.10.Amd64.Pixel.Perf From b739648b27dd0b9ac9bf794084ea26dab5c30dce Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 16:45:58 -0800 Subject: [PATCH 16/48] Try using windows-2022 for android builds. --- azure-pipelines.yml | 2 +- eng/performance/build_machine_matrix.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 54470b7e1e0..cb240c650e1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -375,7 +375,7 @@ jobs: parameters: jobTemplate: /eng/performance/scenarios.yml buildMachines: - - mac-android-arm64 + - win-x64-android-arm64 isPublic: false jobParameters: kind: maui_scenarios_android diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/build_machine_matrix.yml index 58c985f294b..63b95f58f0e 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/performance/build_machine_matrix.yml @@ -93,14 +93,14 @@ jobs: queue: Windows.10.Arm64.Perf.Surf ${{ insert }}: ${{ parameters.jobParameters }} -- ${{ if and(containsValue(parameters.buildMachines, 'mac-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently +- ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: osName: windows architecture: x64 osVersion: 19H1 pool: - vmImage: 'macos-12' + vmImage: 'windows-2022' queue: Windows.10.Amd64.Pixel.Perf machinePool: Pixel ${{ insert }}: ${{ parameters.jobParameters }} From a9412d8fc554cafc42c8aa98b749166ed495361c Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 7 Dec 2022 17:45:53 -0800 Subject: [PATCH 17/48] Only remove the dotnet\packs path since they take up multiple gigabytes. --- eng/performance/maui_scenarios_android.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 6a3a2549e83..f2875f01446 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -17,7 +17,7 @@ - + From 451bbdd56e79a66d612db055c925da2c0ec51306 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 8 Dec 2022 10:53:16 -0800 Subject: [PATCH 18/48] Add timeout for the maui android scenarios helix work items. --- eng/performance/maui_scenarios_android.proj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index f2875f01446..bb56e75508d 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -48,18 +48,22 @@ xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y $(Python) test.py sod --scenario-name "%(Identity)" + 00:30 xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + 00:30 xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + 00:30 From cacece4175570df020d37ea66f8041d23e3e3202 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 8 Dec 2022 12:46:45 -0800 Subject: [PATCH 19/48] Fix the xcopy commands to ensure there is not a prompt since we are only transferring one file. --- eng/performance/maui_scenarios_android.proj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index bb56e75508d..1e596e7c4ce 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -46,22 +46,22 @@ - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub\ /Y $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y - $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub\ /Y + $(Python) test.py devicestartup --device-type android --package-path pub\\elixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" 00:30 - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub /E /I /Y + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub\ /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations 00:30 From ab8766b8357d116ad96a48ac9e59b82717a50937 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 8 Dec 2022 14:52:17 -0800 Subject: [PATCH 20/48] Changed APK Name to not include the file extension, and enabled extracted size sod testing by adding precommands to extract the apk zip. --- eng/performance/maui_scenarios_android.proj | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 1e596e7c4ce..125859aa027 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -24,13 +24,13 @@ mauiandroid $(ScenariosDir)%(ScenarioDirectoryName) - com.companyname.mauiandroiddefault-Signed.apk + com.companyname.mauiandroiddefault-Signed com.companyname.mauiandroiddefault mauiblazorandroid $(ScenariosDir)%(ScenarioDirectoryName) - com.companyname.mauiblazorandroiddefault-Signed.apk + com.companyname.mauiblazorandroiddefault-Signed com.companyname.mauiblazorandroiddefault @@ -46,23 +46,23 @@ - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub\ /Y + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - + - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub\ /Y - $(Python) test.py devicestartup --device-type android --package-path pub\\elixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" 00:30 - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName) %HELIX_WORKITEM_ROOT%\pub\ /Y - $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName) --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations 00:30 From b49edf49934e8eadc7888c6605bc5d36163196ef Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 8 Dec 2022 17:13:23 -0800 Subject: [PATCH 21/48] Added android podcast app testing and added XHARNESSPATH to the precommands. --- eng/performance/maui_scenarios_android.proj | 14 ++++-- src/scenarios/mauiandroid/pre.py | 2 + src/scenarios/mauiandroidpodcast/post.py | 9 ++++ src/scenarios/mauiandroidpodcast/pre.py | 49 +++++++++++++++++++++ src/scenarios/mauiandroidpodcast/test.py | 12 +++++ 5 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/scenarios/mauiandroidpodcast/post.py create mode 100644 src/scenarios/mauiandroidpodcast/pre.py create mode 100644 src/scenarios/mauiandroidpodcast/test.py diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 125859aa027..ab0cb9c702d 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -27,6 +27,12 @@ com.companyname.mauiandroiddefault-Signed com.companyname.mauiandroiddefault + + mauiandroidpodcast + $(ScenariosDir)%(ScenarioDirectoryName) + com.Microsoft.NetConf2021.Maui-Signed + com.Microsoft.NetConf2021.Maui + mauiblazorandroid $(ScenariosDir)%(ScenarioDirectoryName) @@ -46,22 +52,22 @@ - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" 00:30 - xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations 00:30 diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 62c3581a5c8..81ba7edd312 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -48,3 +48,5 @@ os.remove(f"{output_file_partial_path}-Signed.aab") os.remove(f"{output_file_partial_path}.aab") + +# Copy the MauiVersion to a file so we have it on the machine (EXTRA_VERSIONS?) diff --git a/src/scenarios/mauiandroidpodcast/post.py b/src/scenarios/mauiandroidpodcast/post.py new file mode 100644 index 00000000000..eebba678cb9 --- /dev/null +++ b/src/scenarios/mauiandroidpodcast/post.py @@ -0,0 +1,9 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories +from performance.common import remove_directory + +remove_directory('dotnet-podcasts') +clean_directories() diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py new file mode 100644 index 00000000000..7325f04ea1c --- /dev/null +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -0,0 +1,49 @@ +''' +pre-command +''' +import sys +import os +import requests +import subprocess +from zipfile import ZipFile +from performance.logger import setup_loggers, getLogger +from shutil import copyfile +from shared import const +from shared.precommands import PreCommands +from argparse import ArgumentParser +from test import EXENAME + +setup_loggers(True) + +parser = ArgumentParser(add_help=False) +parser.add_argument( + '-o', '--output', + dest='output_dir', + required=False, + type=str, + help='capture of the output directory') +args, unknown_args = parser.parse_known_args() + +# Download what we need +with open ("MauiNuGet.config", "wb") as f: + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) + +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', 'net7.0', '--single-branch']) + +precommands = PreCommands() +precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) +precommands.existing(projectdir='./dotnet-podcasts',projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') + +# Build the APK +precommands._restore() +precommands.execute(['--no-restore']) + +# Remove the aab files as we don't need them, this saves space +output_file_partial_path = f"com.Microsoft.NetConf2021.Maui" +if args.output_dir: + output_file_partial_path = os.path.join(args.output_dir, output_file_partial_path) + +os.remove(f"{output_file_partial_path}-Signed.aab") +os.remove(f"{output_file_partial_path}.aab") + +# Copy the MauiVersion to a file so we have it on the machine (EXTRA_VERSIONS?) diff --git a/src/scenarios/mauiandroidpodcast/test.py b/src/scenarios/mauiandroidpodcast/test.py new file mode 100644 index 00000000000..71edabdcaf7 --- /dev/null +++ b/src/scenarios/mauiandroidpodcast/test.py @@ -0,0 +1,12 @@ +''' +C# Console app +''' +from shared.runner import TestTraits, Runner + +EXENAME = 'MauiAndroidPodcast' + +if __name__ == "__main__": + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() From b3ab3c154809cce2080274d25495dcb46a9cbe0f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 8 Dec 2022 17:35:41 -0800 Subject: [PATCH 22/48] Don't try to remove the dotnet-podcasts directory as it is causing permission errors. --- src/scenarios/mauiandroidpodcast/post.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scenarios/mauiandroidpodcast/post.py b/src/scenarios/mauiandroidpodcast/post.py index eebba678cb9..9362e38f65f 100644 --- a/src/scenarios/mauiandroidpodcast/post.py +++ b/src/scenarios/mauiandroidpodcast/post.py @@ -5,5 +5,4 @@ from shared.postcommands import clean_directories from performance.common import remove_directory -remove_directory('dotnet-podcasts') clean_directories() From e820801fbe3a4944175d03174d3e9041ee70462b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 9 Dec 2022 11:31:13 -0800 Subject: [PATCH 23/48] Fixed .git folder delete issue. --- src/scenarios/mauiandroidpodcast/post.py | 1 + src/scenarios/mauiandroidpodcast/pre.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/scenarios/mauiandroidpodcast/post.py b/src/scenarios/mauiandroidpodcast/post.py index 9362e38f65f..abdb7d8db9d 100644 --- a/src/scenarios/mauiandroidpodcast/post.py +++ b/src/scenarios/mauiandroidpodcast/post.py @@ -5,4 +5,5 @@ from shared.postcommands import clean_directories from performance.common import remove_directory +remove_directory("dotnet-podcasts") clean_directories() diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index 7325f04ea1c..f9214da34fd 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -1,14 +1,10 @@ ''' pre-command ''' -import sys import os import requests import subprocess -from zipfile import ZipFile from performance.logger import setup_loggers, getLogger -from shutil import copyfile -from shared import const from shared.precommands import PreCommands from argparse import ArgumentParser from test import EXENAME @@ -28,7 +24,8 @@ with open ("MauiNuGet.config", "wb") as f: f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) -subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', 'net7.0', '--single-branch']) +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', 'net7.0', '--single-branch', '--depth', '1']) +subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, for their deletion seperately precommands = PreCommands() precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) From 2ec4bad597c3e245e3eb5357daab623abfd8a38f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 9 Dec 2022 15:21:10 -0800 Subject: [PATCH 24/48] Add MauiVersion stuff to the pre.py and test.py commands. --- src/scenarios/mauiandroid/pre.py | 12 ++++++++++-- src/scenarios/mauiandroid/test.py | 14 ++++++++++++++ src/scenarios/mauiandroidpodcast/pre.py | 12 ++++++++++-- src/scenarios/mauiandroidpodcast/test.py | 14 ++++++++++++++ src/scenarios/mauiblazorandroid/pre.py | 12 +++++++++++- src/scenarios/mauiblazorandroid/test.py | 14 ++++++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 81ba7edd312..3f374882ea1 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -4,13 +4,14 @@ import sys import os import requests +import subprocess from zipfile import ZipFile from performance.logger import setup_loggers, getLogger from shutil import copyfile from shared import const from shared.precommands import PreCommands from argparse import ArgumentParser -from test import EXENAME +from test import EXENAME, MAUIVERSIONFILE setup_loggers(True) @@ -49,4 +50,11 @@ os.remove(f"{output_file_partial_path}-Signed.aab") os.remove(f"{output_file_partial_path}.aab") -# Copy the MauiVersion to a file so we have it on the machine (EXTRA_VERSIONS?) +# Copy the MauiVersion to a file so we have it on the machine +result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem .\{const.APPDIR}\obj\Release\net7.0-android\android-arm64\linked\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) +maui_version = result.stdout.decode('utf-8').strip() +print(f'MAUI_VERSION: {maui_version}') +if("sha" not in maui_version or "azdo" not in maui_version): + raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") +with open(f'{args.output_dir}/{MAUIVERSIONFILE}', 'w') as f: + f.write(maui_version) diff --git a/src/scenarios/mauiandroid/test.py b/src/scenarios/mauiandroid/test.py index 349ef2ebcc7..05f71856dfc 100644 --- a/src/scenarios/mauiandroid/test.py +++ b/src/scenarios/mauiandroid/test.py @@ -1,11 +1,25 @@ ''' C# Console app ''' +import os from shared.runner import TestTraits, Runner EXENAME = 'MauiAndroidDefault' +MAUIVERSIONFILE = 'MAUI_VERSION.txt' if __name__ == "__main__": + try: + with open(f'pub/{MAUIVERSIONFILE}', 'r') as f: + maui_version = f.read() + if("sha" not in maui_version or "azdo" not in maui_version): + raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") + else: + print(f"Found MAUI_VERSION {maui_version}") + os.environ["MAUI_VERSION"] = maui_version + + except Exception as e: + print("Failed to read MAUI_VERSION.txt") + traits = TestTraits(exename=EXENAME, guiapp='false', ) diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index f9214da34fd..d8018ec73bc 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -6,8 +6,9 @@ import subprocess from performance.logger import setup_loggers, getLogger from shared.precommands import PreCommands +from shared import const from argparse import ArgumentParser -from test import EXENAME +from test import EXENAME, MAUIVERSIONFILE setup_loggers(True) @@ -43,4 +44,11 @@ os.remove(f"{output_file_partial_path}-Signed.aab") os.remove(f"{output_file_partial_path}.aab") -# Copy the MauiVersion to a file so we have it on the machine (EXTRA_VERSIONS?) +# Copy the MauiVersion to a file so we have it on the machine +result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem .\{const.APPDIR}\src\Mobile\obj\Release\net7.0-android\android-arm64\linked\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) +maui_version = result.stdout.decode('utf-8').strip() +print(f'MAUI_VERSION: {maui_version}') +if("sha" not in maui_version or "azdo" not in maui_version): + raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") +with open(f'{args.output_dir}/{MAUIVERSIONFILE}', 'w') as f: + f.write(maui_version) diff --git a/src/scenarios/mauiandroidpodcast/test.py b/src/scenarios/mauiandroidpodcast/test.py index 71edabdcaf7..ff47a061176 100644 --- a/src/scenarios/mauiandroidpodcast/test.py +++ b/src/scenarios/mauiandroidpodcast/test.py @@ -1,11 +1,25 @@ ''' C# Console app ''' +import os from shared.runner import TestTraits, Runner EXENAME = 'MauiAndroidPodcast' +MAUIVERSIONFILE = 'MAUI_VERSION.txt' if __name__ == "__main__": + try: + with open(f'pub/{MAUIVERSIONFILE}', 'r') as f: + maui_version = f.read() + if("sha" not in maui_version or "azdo" not in maui_version): + raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") + else: + print(f"Found MAUI_VERSION {maui_version}") + os.environ["MAUI_VERSION"] = maui_version + + except Exception as e: + print("Failed to read MAUI_VERSION.txt") + traits = TestTraits(exename=EXENAME, guiapp='false', ) diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index c439ae9c927..0def353e96f 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -1,6 +1,7 @@ ''' pre-command ''' +import subprocess import sys import os import requests @@ -10,7 +11,7 @@ from shared import const from shared.precommands import PreCommands from argparse import ArgumentParser -from test import EXENAME +from test import EXENAME, MAUIVERSIONFILE setup_loggers(True) @@ -87,3 +88,12 @@ os.remove(f"{output_file_partial_path}-Signed.aab") os.remove(f"{output_file_partial_path}.aab") + +# Copy the MauiVersion to a file so we have it on the machine +result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem .\{const.APPDIR}\obj\Release\net7.0-android\android-arm64\linked\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) +maui_version = result.stdout.decode('utf-8').strip() +print(f'MAUI_VERSION: {maui_version}') +if("sha" not in maui_version or "azdo" not in maui_version): + raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") +with open(f'{args.output_dir}/{MAUIVERSIONFILE}', 'w') as f: + f.write(maui_version) diff --git a/src/scenarios/mauiblazorandroid/test.py b/src/scenarios/mauiblazorandroid/test.py index 743d5eaeb65..b47ffa24b1c 100644 --- a/src/scenarios/mauiblazorandroid/test.py +++ b/src/scenarios/mauiblazorandroid/test.py @@ -1,11 +1,25 @@ ''' C# Console app ''' +import os from shared.runner import TestTraits, Runner EXENAME = 'MauiBlazorAndroidDefault' +MAUIVERSIONFILE = 'MAUI_VERSION.txt' if __name__ == "__main__": + try: + with open(f'pub/{MAUIVERSIONFILE}', 'r') as f: + maui_version = f.read() + if("sha" not in maui_version or "azdo" not in maui_version): + raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") + else: + print(f"Found MAUI_VERSION {maui_version}") + os.environ["MAUI_VERSION"] = maui_version + + except Exception as e: + print("Failed to read MAUI_VERSION.txt") + traits = TestTraits(exename=EXENAME, guiapp='false', ) From 425f6af0a667b9913c040f49713b4d38416ad3ae Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 9 Dec 2022 16:18:43 -0800 Subject: [PATCH 25/48] Ensure the Maui_Version file is downloaded with the apk file and make sure to delete it if it exists. --- eng/performance/maui_scenarios_android.proj | 8 ++++---- src/scenarios/mauiandroid/test.py | 1 + src/scenarios/mauiandroidpodcast/test.py | 1 + src/scenarios/mauiblazorandroid/test.py | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index ab0cb9c702d..14af1ec2db8 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -52,22 +52,22 @@ - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" 00:30 - echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ApkName).apk %HELIX_WORKITEM_ROOT%\pub\ /Y + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations 00:30 diff --git a/src/scenarios/mauiandroid/test.py b/src/scenarios/mauiandroid/test.py index 05f71856dfc..bca41dca8cf 100644 --- a/src/scenarios/mauiandroid/test.py +++ b/src/scenarios/mauiandroid/test.py @@ -16,6 +16,7 @@ else: print(f"Found MAUI_VERSION {maui_version}") os.environ["MAUI_VERSION"] = maui_version + os.remove(f'pub/{MAUIVERSIONFILE}') except Exception as e: print("Failed to read MAUI_VERSION.txt") diff --git a/src/scenarios/mauiandroidpodcast/test.py b/src/scenarios/mauiandroidpodcast/test.py index ff47a061176..c8fe16ad2a8 100644 --- a/src/scenarios/mauiandroidpodcast/test.py +++ b/src/scenarios/mauiandroidpodcast/test.py @@ -16,6 +16,7 @@ else: print(f"Found MAUI_VERSION {maui_version}") os.environ["MAUI_VERSION"] = maui_version + os.remove(f'pub/{MAUIVERSIONFILE}') except Exception as e: print("Failed to read MAUI_VERSION.txt") diff --git a/src/scenarios/mauiblazorandroid/test.py b/src/scenarios/mauiblazorandroid/test.py index b47ffa24b1c..f81beae1e68 100644 --- a/src/scenarios/mauiblazorandroid/test.py +++ b/src/scenarios/mauiblazorandroid/test.py @@ -16,6 +16,7 @@ else: print(f"Found MAUI_VERSION {maui_version}") os.environ["MAUI_VERSION"] = maui_version + os.remove(f'pub/{MAUIVERSIONFILE}') except Exception as e: print("Failed to read MAUI_VERSION.txt") From 0cd22d7b835d79e08a6cd6ff5c66a5e0f6f59892 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 11:24:10 -0800 Subject: [PATCH 26/48] Try different xcopy directory format to see if it gets found. --- eng/performance/maui_scenarios_android.proj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 14af1ec2db8..9cd11052c2a 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -27,7 +27,7 @@ com.companyname.mauiandroiddefault-Signed com.companyname.mauiandroiddefault - + @@ -52,22 +52,22 @@ - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip $(Python) test.py sod --scenario-name "%(Identity)" 00:30 - echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" 00:30 - echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\ %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations 00:30 From e3b2215caca35fdfffb7d6f21a787d17b06f3f4a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 12:54:08 -0800 Subject: [PATCH 27/48] Try using batch available tools and see if powershell is available. --- eng/performance/maui_scenarios_android.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 9cd11052c2a..ba305629d7c 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -57,7 +57,7 @@ 00:30 - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; Rename-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\; Remove-Item %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "& {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip $(Python) test.py sod --scenario-name "%(Identity)" 00:30 From 87a671f6f95bdcb15bec7f7df3bef745e856c29d Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 13:13:10 -0800 Subject: [PATCH 28/48] Fix ampersand. --- eng/performance/maui_scenarios_android.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index ba305629d7c..854197196b4 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -57,7 +57,7 @@ 00:30 - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "& {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "& {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip $(Python) test.py sod --scenario-name "%(Identity)" 00:30 From 25c648683bfeef8da3b27c02824fb73dbc44bfff Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 14:04:09 -0800 Subject: [PATCH 29/48] Readd podcast and blazor scenario, and remove rollback file for net7.0. --- eng/performance/maui_scenarios_android.proj | 4 ++-- src/scenarios/mauiandroid/pre.py | 2 +- src/scenarios/mauiandroidpodcast/pre.py | 2 +- src/scenarios/mauiblazorandroid/pre.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 854197196b4..26e2bfcc341 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -27,7 +27,7 @@ com.companyname.mauiandroiddefault-Signed com.companyname.mauiandroiddefault - + diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 3f374882ea1..3c7edb4aec1 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -29,7 +29,7 @@ f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) precommands = PreCommands() -precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) +precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) # Setup the Maui folder precommands.new(template='maui', diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index d8018ec73bc..a53ad5882f8 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -29,7 +29,7 @@ subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, for their deletion seperately precommands = PreCommands() -precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) +precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) precommands.existing(projectdir='./dotnet-podcasts',projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') # Build the APK diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index 0def353e96f..9d0be27fdbf 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -29,7 +29,7 @@ f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) precommands = PreCommands() -precommands.install_workload('maui', ['--from-rollback-file', f'https://aka.ms/dotnet/maui/net7.0.json', '--configfile', 'MauiNuGet.config']) +precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) # Setup the Maui folder precommands.new(template='maui-blazor', From b5bd30466e8ff2c6203e398e525709efa73217eb Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 14:09:20 -0800 Subject: [PATCH 30/48] Try using a specific version link and getting that version. --- azure-pipelines.yml | 3 ++- eng/performance/scenarios.yml | 31 +++++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cb240c650e1..7c6bcc50ced 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -380,8 +380,9 @@ jobs: jobParameters: kind: maui_scenarios_android projectFile: maui_scenarios_android.proj + channelHasVersionLink: true channels: - - release/7.0 + - [release/7.0, https://aka.ms/dotnet/sdk/maui/net7.0.json] ################################################ # Scheduled Private jobs (Weekly) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 2e0bf2ea3f1..454c87580e9 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -7,6 +7,7 @@ parameters: queue: '' # required -- name of the Helix queue container: '' # optional -- id of the container channels: [] # required -- list of channels to download .NET from + channelHasVersionLink: false # optional -- If the channels array consists of channel and version link projectFile: '' # required -- project file to build (current choices: scenarios.proj/sdk_scenarios.proj ) machinePool: '' # required -- Name of perf machine pool (Tiger, Owl, etc) @@ -82,15 +83,33 @@ jobs: strategy: matrix: ${{ each channel in parameters.channels }}: - ${{ channel }}: - _Channel: ${{ channel }} - _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" - _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net + ${{ if eq(channelHasVersionLink, true) }}: + ${{ channel }}: + _Channel: ${{ channel[0] }} + _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" + _VersionLink: ${{ channel[1] }} + _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net + ${{ if ne(channelHasVersionLink, true) }}: + ${{ channel }}: + _Channel: ${{ channel }} + _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" + _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net steps: - checkout: self clean: true - - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet - displayName: Run ci_setup.py + - ${{ if eq(channelHasVersionLink, true) }}: + # Get version from the link + - script: | + $json = Invoke-WebRequest -URI $(_VersionLink) + $content = ConvertFrom-Json $json.content + $version = $content.version + Write-Host "##vso[task.setvariable variable=dotnetVersion]$version" + displayName: Get dotnet-version to use + - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet --dotnet-versions $(dotnetVersion) + displayName: Run ci_setup.py + - ${{ if ne(channelHasVersionLink, true) }}: + - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet + displayName: Run ci_setup.py - ${{ if eq(parameters.osName, 'windows') }}: - script: xcopy .\NuGet.config $(CorrelationStaging) && xcopy .\scripts $(CorrelationStaging)scripts\/e && xcopy .\src\scenarios\shared $(CorrelationStaging)shared\/e && xcopy .\src\scenarios\staticdeps $(CorrelationStaging)staticdeps\/e displayName: Copy python libraries and NuGet.config From edaaf48308aa87ec917d2bdd801bbb108481e8de Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 14:12:25 -0800 Subject: [PATCH 31/48] Try using key values for the channels with links. --- azure-pipelines.yml | 3 ++- eng/performance/scenarios.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7c6bcc50ced..574c1f5ea8b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -382,7 +382,8 @@ jobs: projectFile: maui_scenarios_android.proj channelHasVersionLink: true channels: - - [release/7.0, https://aka.ms/dotnet/sdk/maui/net7.0.json] + - channel: release/7.0 + link: https://aka.ms/dotnet/sdk/maui/net7.0.json ################################################ # Scheduled Private jobs (Weekly) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 454c87580e9..5eb77df424a 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -85,9 +85,9 @@ jobs: ${{ each channel in parameters.channels }}: ${{ if eq(channelHasVersionLink, true) }}: ${{ channel }}: - _Channel: ${{ channel[0] }} + _Channel: ${{ channel['channel'] }} _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" - _VersionLink: ${{ channel[1] }} + _VersionLink: ${{ channel['link'] }} _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net ${{ if ne(channelHasVersionLink, true) }}: ${{ channel }}: From 7f34f2c36cb9bb9fcb0553b14d382d25b663636e Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 14:14:53 -0800 Subject: [PATCH 32/48] Testing how to pass link. --- azure-pipelines.yml | 3 +-- eng/performance/scenarios.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 574c1f5ea8b..eb91a72226e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -382,8 +382,7 @@ jobs: projectFile: maui_scenarios_android.proj channelHasVersionLink: true channels: - - channel: release/7.0 - link: https://aka.ms/dotnet/sdk/maui/net7.0.json + - release/7.0^https://aka.ms/dotnet/sdk/maui/net7.0.json ################################################ # Scheduled Private jobs (Weekly) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 5eb77df424a..1ce904b07a9 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -83,13 +83,13 @@ jobs: strategy: matrix: ${{ each channel in parameters.channels }}: - ${{ if eq(channelHasVersionLink, true) }}: + ${{ if eq(parameters.channelHasVersionLink, true) }}: ${{ channel }}: - _Channel: ${{ channel['channel'] }} + _Channel: ${{ split(channel, '^')[0] }} _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" - _VersionLink: ${{ channel['link'] }} + _VersionLink: ${{ split(channel, '^')[1] }} _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net - ${{ if ne(channelHasVersionLink, true) }}: + ${{ if ne(parameters.channelHasVersionLink, true) }}: ${{ channel }}: _Channel: ${{ channel }} _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" @@ -97,9 +97,9 @@ jobs: steps: - checkout: self clean: true - - ${{ if eq(channelHasVersionLink, true) }}: + - ${{ if eq(parameters.channelHasVersionLink, true) }}: # Get version from the link - - script: | + - powershell: | $json = Invoke-WebRequest -URI $(_VersionLink) $content = ConvertFrom-Json $json.content $version = $content.version @@ -107,7 +107,7 @@ jobs: displayName: Get dotnet-version to use - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet --dotnet-versions $(dotnetVersion) displayName: Run ci_setup.py - - ${{ if ne(channelHasVersionLink, true) }}: + - ${{ if ne(parameters.channelHasVersionLink, true) }}: - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet displayName: Run ci_setup.py - ${{ if eq(parameters.osName, 'windows') }}: From dc525374c2d7d3c97f0e313b0e3918c064201941 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 16:18:53 -0800 Subject: [PATCH 33/48] Removed version passing functionality as net7.0 shouldn't need it for maui, and added the run to the Private runs. --- azure-pipelines.yml | 16 ++++++++++++++-- eng/performance/scenarios.yml | 30 ++++++------------------------ 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eb91a72226e..01e17779d20 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -247,6 +247,19 @@ jobs: projectFile: maui_scenarios.proj channels: - release/7.0 + + # Maui scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64-android-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + channels: + - release/7.0 ################################################ # Scheduled Private jobs @@ -380,9 +393,8 @@ jobs: jobParameters: kind: maui_scenarios_android projectFile: maui_scenarios_android.proj - channelHasVersionLink: true channels: - - release/7.0^https://aka.ms/dotnet/sdk/maui/net7.0.json + - release/7.0 ################################################ # Scheduled Private jobs (Weekly) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 1ce904b07a9..240c2ca3caa 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -83,33 +83,15 @@ jobs: strategy: matrix: ${{ each channel in parameters.channels }}: - ${{ if eq(parameters.channelHasVersionLink, true) }}: - ${{ channel }}: - _Channel: ${{ split(channel, '^')[0] }} - _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" - _VersionLink: ${{ split(channel, '^')[1] }} - _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net - ${{ if ne(parameters.channelHasVersionLink, true) }}: - ${{ channel }}: - _Channel: ${{ channel }} - _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" - _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net + ${{ channel }}: + _Channel: ${{ channel }} + _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}" + _BuildConfig: ${{ parameters.architecture }}_$(_Channel)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net steps: - checkout: self clean: true - - ${{ if eq(parameters.channelHasVersionLink, true) }}: - # Get version from the link - - powershell: | - $json = Invoke-WebRequest -URI $(_VersionLink) - $content = ConvertFrom-Json $json.content - $version = $content.version - Write-Host "##vso[task.setvariable variable=dotnetVersion]$version" - displayName: Get dotnet-version to use - - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet --dotnet-versions $(dotnetVersion) - displayName: Run ci_setup.py - - ${{ if ne(parameters.channelHasVersionLink, true) }}: - - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet - displayName: Run ci_setup.py + - script: $(Python) scripts/ci_setup.py --channel $(_Channel) --architecture ${{parameters.architecture}} --perf-hash $(Build.SourceVersion) --queue ${{parameters.queue}} --build-number $(Build.BuildNumber) --build-configs $(_Configs) --output-file $(CorrelationStaging)machine-setup$(ScriptExtension) --install-dir $(CorrelationStaging)dotnet + displayName: Run ci_setup.py - ${{ if eq(parameters.osName, 'windows') }}: - script: xcopy .\NuGet.config $(CorrelationStaging) && xcopy .\scripts $(CorrelationStaging)scripts\/e && xcopy .\src\scenarios\shared $(CorrelationStaging)shared\/e && xcopy .\src\scenarios\staticdeps $(CorrelationStaging)staticdeps\/e displayName: Copy python libraries and NuGet.config From 65b7342b6e61b14d86a000c367da895d47c1b6fb Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 16:23:13 -0800 Subject: [PATCH 34/48] Remove channelHasVersionLink from scenarios.yml. --- eng/performance/scenarios.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 240c2ca3caa..2e0bf2ea3f1 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -7,7 +7,6 @@ parameters: queue: '' # required -- name of the Helix queue container: '' # optional -- id of the container channels: [] # required -- list of channels to download .NET from - channelHasVersionLink: false # optional -- If the channels array consists of channel and version link projectFile: '' # required -- project file to build (current choices: scenarios.proj/sdk_scenarios.proj ) machinePool: '' # required -- Name of perf machine pool (Tiger, Owl, etc) From 76b64a1b2f3e3cf661634c983f4bf2886f06e1d1 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 16:27:18 -0800 Subject: [PATCH 35/48] Fix spacing issue. --- azure-pipelines.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 01e17779d20..b21363082ec 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -250,16 +250,16 @@ jobs: # Maui scenario benchmarks - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64-android-arm64 - isPublic: false - jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj - channels: - - release/7.0 + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64-android-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + channels: + - release/7.0 ################################################ # Scheduled Private jobs From 833905ecde2871772eeadfc921629331ad4da2cf Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 12 Dec 2022 16:28:12 -0800 Subject: [PATCH 36/48] Final Cleanup commit. --- azure-pipelines.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b21363082ec..04eeeca0611 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -383,18 +383,25 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - # Maui scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64-android-arm64 - isPublic: false - jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj - channels: - - release/7.0 + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} ################################################ # Scheduled Private jobs (Weekly) From 4963e8d0fa73ca19d0a30509f3315853a78cc47e Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 13 Dec 2022 16:46:03 -0800 Subject: [PATCH 37/48] Makes timeouts global for maui_scenarios_android.proj and clarifies dotnet.py get commit date. --- eng/performance/maui_scenarios_android.proj | 11 +++++++---- scripts/dotnet.py | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 26e2bfcc341..a435f634691 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -20,6 +20,12 @@ + + + 30:00 + + + mauiandroid @@ -49,27 +55,24 @@ + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py sod --scenario-name "%(Identity)" - 00:30 echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "& {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip $(Python) test.py sod --scenario-name "%(Identity)" - 00:30 echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" - 00:30 echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations - 00:30 diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 938937b22ef..58284a3cc9b 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -612,17 +612,17 @@ def get_commit_date( if not commit_sha: raise ValueError('.NET Commit sha was not defined.') + # Example URL: https://github.com/dotnet/runtime/commit/2d76178d5faa97be86fc8d049c7dbcbdf66dc497.patch url = None - urlformat = 'https://github.com/%s/%s/commit/%s.patch' if repository is None: # The origin of the repo where the commit belongs to has changed # between release. Here we attempt to naively guess the repo. core_sdk_frameworks = ChannelMap.get_supported_frameworks() repo = 'core-sdk' if framework in core_sdk_frameworks else 'cli' - url = urlformat % ('dotnet', repo, commit_sha) + url = f'https://github.com/dotnet/{repo}/commit/{commit_sha}.patch' else: owner, repo = get_repository(repository) - url = urlformat % (owner, repo, commit_sha) + url = f'https://github.com/{owner}/{repo}/commit/{commit_sha}.patch' build_timestamp = None sleep_time = 10 # Start with 10 second sleep timer From be55f553659d0b12b4523fa9038e695d6682f3e9 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 13 Dec 2022 16:47:12 -0800 Subject: [PATCH 38/48] Update pre and test py's to use shared methods and add version management capabilties. --- src/scenarios/mauiandroid/pre.py | 41 +++++-------------- src/scenarios/mauiandroid/test.py | 21 +++------- src/scenarios/mauiandroidpodcast/pre.py | 39 +++++------------- src/scenarios/mauiandroidpodcast/test.py | 21 +++------- src/scenarios/mauiblazorandroid/pre.py | 41 +++++-------------- src/scenarios/mauiblazorandroid/test.py | 19 ++------- src/scenarios/mauishared/mauisharedpython.py | 13 ++++++ src/scenarios/shared/versionmanager.py | 26 ++++++++++++ src/tools/Reporting/Reporting/IEnvironment.cs | 5 +-- src/tools/Reporting/Reporting/Reporter.cs | 14 +++---- 10 files changed, 93 insertions(+), 147 deletions(-) create mode 100644 src/scenarios/mauishared/mauisharedpython.py create mode 100644 src/scenarios/shared/versionmanager.py diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 3c7edb4aec1..71d104f1e43 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -2,33 +2,22 @@ pre-command ''' import sys -import os import requests -import subprocess -from zipfile import ZipFile +from mauishared.mauisharedpython import RemoveAABFiles, GetVersionFromDll from performance.logger import setup_loggers, getLogger -from shutil import copyfile from shared import const from shared.precommands import PreCommands -from argparse import ArgumentParser -from test import EXENAME, MAUIVERSIONFILE +from shared.versionmanager import versionswritejson +from test import EXENAME setup_loggers(True) -parser = ArgumentParser(add_help=False) -parser.add_argument( - '-o', '--output', - dest='output_dir', - required=False, - type=str, - help='capture of the output directory') -args, unknown_args = parser.parse_known_args() +precommands = PreCommands() # Download what we need with open ("MauiNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) - -precommands = PreCommands() + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) + precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) # Setup the Maui folder @@ -43,18 +32,10 @@ precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) # Remove the aab files as we don't need them, this saves space -output_file_partial_path = f"com.companyname.{str.lower(EXENAME)}" -if args.output_dir: - output_file_partial_path = os.path.join(args.output_dir, output_file_partial_path) - -os.remove(f"{output_file_partial_path}-Signed.aab") -os.remove(f"{output_file_partial_path}.aab") +RemoveAABFiles(precommands.output) # Copy the MauiVersion to a file so we have it on the machine -result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem .\{const.APPDIR}\obj\Release\net7.0-android\android-arm64\linked\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) -maui_version = result.stdout.decode('utf-8').strip() -print(f'MAUI_VERSION: {maui_version}') -if("sha" not in maui_version or "azdo" not in maui_version): - raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") -with open(f'{args.output_dir}/{MAUIVERSIONFILE}', 'w') as f: - f.write(maui_version) +maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +version_dict = { "maui_version": maui_version } +versionswritejson(version_dict, rf".\{precommands.output}\versions.json") +print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiandroid/test.py b/src/scenarios/mauiandroid/test.py index bca41dca8cf..75fb182f2c7 100644 --- a/src/scenarios/mauiandroid/test.py +++ b/src/scenarios/mauiandroid/test.py @@ -1,25 +1,14 @@ ''' -C# Console app +Mobile Maui App ''' -import os +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versionsreadjsonfilesaveenv EXENAME = 'MauiAndroidDefault' -MAUIVERSIONFILE = 'MAUI_VERSION.txt' -if __name__ == "__main__": - try: - with open(f'pub/{MAUIVERSIONFILE}', 'r') as f: - maui_version = f.read() - if("sha" not in maui_version or "azdo" not in maui_version): - raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") - else: - print(f"Found MAUI_VERSION {maui_version}") - os.environ["MAUI_VERSION"] = maui_version - os.remove(f'pub/{MAUIVERSIONFILE}') - - except Exception as e: - print("Failed to read MAUI_VERSION.txt") +if __name__ == "__main__": + versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json") traits = TestTraits(exename=EXENAME, guiapp='false', diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index a53ad5882f8..166f2a666b1 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -1,34 +1,24 @@ ''' pre-command ''' -import os import requests import subprocess +from mauishared.mauisharedpython import RemoveAABFiles, GetVersionFromDll from performance.logger import setup_loggers, getLogger from shared.precommands import PreCommands +from shared.versionmanager import versionswritejson from shared import const -from argparse import ArgumentParser -from test import EXENAME, MAUIVERSIONFILE setup_loggers(True) - -parser = ArgumentParser(add_help=False) -parser.add_argument( - '-o', '--output', - dest='output_dir', - required=False, - type=str, - help='capture of the output directory') -args, unknown_args = parser.parse_known_args() +precommands = PreCommands() # Download what we need with open ("MauiNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) -subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', 'net7.0', '--single-branch', '--depth', '1']) +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', f'{precommands.framework[:6]}', '--single-branch', '--depth', '1']) subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, for their deletion seperately -precommands = PreCommands() precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) precommands.existing(projectdir='./dotnet-podcasts',projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') @@ -37,18 +27,9 @@ precommands.execute(['--no-restore']) # Remove the aab files as we don't need them, this saves space -output_file_partial_path = f"com.Microsoft.NetConf2021.Maui" -if args.output_dir: - output_file_partial_path = os.path.join(args.output_dir, output_file_partial_path) - -os.remove(f"{output_file_partial_path}-Signed.aab") -os.remove(f"{output_file_partial_path}.aab") +RemoveAABFiles(precommands.output) -# Copy the MauiVersion to a file so we have it on the machine -result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem .\{const.APPDIR}\src\Mobile\obj\Release\net7.0-android\android-arm64\linked\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) -maui_version = result.stdout.decode('utf-8').strip() -print(f'MAUI_VERSION: {maui_version}') -if("sha" not in maui_version or "azdo" not in maui_version): - raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") -with open(f'{args.output_dir}/{MAUIVERSIONFILE}', 'w') as f: - f.write(maui_version) +maui_version = GetVersionFromDll(f".\{const.APPDIR}\src\Mobile\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +version_dict = { "maui_version": maui_version } +versionswritejson(version_dict, rf".\{precommands.output}\versions.json") +print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiandroidpodcast/test.py b/src/scenarios/mauiandroidpodcast/test.py index c8fe16ad2a8..1e557a00e46 100644 --- a/src/scenarios/mauiandroidpodcast/test.py +++ b/src/scenarios/mauiandroidpodcast/test.py @@ -1,26 +1,15 @@ ''' -C# Console app +Mobile Maui App ''' -import os +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versionsreadjsonfilesaveenv EXENAME = 'MauiAndroidPodcast' -MAUIVERSIONFILE = 'MAUI_VERSION.txt' if __name__ == "__main__": - try: - with open(f'pub/{MAUIVERSIONFILE}', 'r') as f: - maui_version = f.read() - if("sha" not in maui_version or "azdo" not in maui_version): - raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") - else: - print(f"Found MAUI_VERSION {maui_version}") - os.environ["MAUI_VERSION"] = maui_version - os.remove(f'pub/{MAUIVERSIONFILE}') - - except Exception as e: - print("Failed to read MAUI_VERSION.txt") - + versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json") + traits = TestTraits(exename=EXENAME, guiapp='false', ) diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index 9d0be27fdbf..6e75fc16590 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -1,34 +1,22 @@ ''' pre-command ''' -import subprocess import sys -import os import requests -from zipfile import ZipFile +from mauishared.mauisharedpython import RemoveAABFiles, GetVersionFromDll from performance.logger import setup_loggers, getLogger -from shutil import copyfile from shared import const from shared.precommands import PreCommands -from argparse import ArgumentParser -from test import EXENAME, MAUIVERSIONFILE +from shared.versionmanager import versionswritejson +from test import EXENAME setup_loggers(True) - -parser = ArgumentParser(add_help=False) -parser.add_argument( - '-o', '--output', - dest='output_dir', - required=False, - type=str, - help='capture of the output directory') -args, unknown_args = parser.parse_known_args() +precommands = PreCommands() # Download what we need with open ("MauiNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/net7.0/NuGet.config', allow_redirects=True).content) + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) -precommands = PreCommands() precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) # Setup the Maui folder @@ -81,19 +69,10 @@ # Build the APK precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) -# Remove the aab files as we don't need them, this saves space -output_file_partial_path = f"com.companyname.{str.lower(EXENAME)}" -if args.output_dir: - output_file_partial_path = os.path.join(args.output_dir, output_file_partial_path) - -os.remove(f"{output_file_partial_path}-Signed.aab") -os.remove(f"{output_file_partial_path}.aab") +RemoveAABFiles(precommands.output) # Copy the MauiVersion to a file so we have it on the machine -result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem .\{const.APPDIR}\obj\Release\net7.0-android\android-arm64\linked\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) -maui_version = result.stdout.decode('utf-8').strip() -print(f'MAUI_VERSION: {maui_version}') -if("sha" not in maui_version or "azdo" not in maui_version): - raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") -with open(f'{args.output_dir}/{MAUIVERSIONFILE}', 'w') as f: - f.write(maui_version) +maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +version_dict = { "maui_version": maui_version } +versionswritejson(version_dict, rf".\{precommands.output}\versions.json") +print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiblazorandroid/test.py b/src/scenarios/mauiblazorandroid/test.py index f81beae1e68..c1b2ce9542a 100644 --- a/src/scenarios/mauiblazorandroid/test.py +++ b/src/scenarios/mauiblazorandroid/test.py @@ -1,25 +1,14 @@ ''' -C# Console app +Mobile Maui App ''' -import os +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versionsreadjsonfilesaveenv EXENAME = 'MauiBlazorAndroidDefault' -MAUIVERSIONFILE = 'MAUI_VERSION.txt' if __name__ == "__main__": - try: - with open(f'pub/{MAUIVERSIONFILE}', 'r') as f: - maui_version = f.read() - if("sha" not in maui_version or "azdo" not in maui_version): - raise ValueError(f"MAUI_VERSION does not contain sha and azdo indicating failure to retrieve or set the value. MAUI_VERSION: {maui_version}") - else: - print(f"Found MAUI_VERSION {maui_version}") - os.environ["MAUI_VERSION"] = maui_version - os.remove(f'pub/{MAUIVERSIONFILE}') - - except Exception as e: - print("Failed to read MAUI_VERSION.txt") + versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json") traits = TestTraits(exename=EXENAME, guiapp='false', diff --git a/src/scenarios/mauishared/mauisharedpython.py b/src/scenarios/mauishared/mauisharedpython.py new file mode 100644 index 00000000000..d3b97657fb4 --- /dev/null +++ b/src/scenarios/mauishared/mauisharedpython.py @@ -0,0 +1,13 @@ +import subprocess +import os + +# Remove the aab files as we don't need them, this saves space in the correlation payload +def RemoveAABFiles(output_dir="."): + file_list = os.listdir(output_dir) + for file in file_list: + if file.endswith(".aab"): + os.remove(os.path.join(output_dir, file)) + +def GetVersionFromDll(dll_path: str): + result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + return result.stdout.decode('utf-8').strip() diff --git a/src/scenarios/shared/versionmanager.py b/src/scenarios/shared/versionmanager.py new file mode 100644 index 00000000000..e6c26505a42 --- /dev/null +++ b/src/scenarios/shared/versionmanager.py @@ -0,0 +1,26 @@ +''' +Version File Manager +''' +import json +import os + +def versionswritejson(versiondict: dict, outputfile = 'versions.json'): + with open(outputfile, 'w') as file: + json.dump(versiondict, file) + +def versionsreadjson(inputfile = 'versions.json'): + with open(inputfile, 'r') as file: + return json.load(file) + +def versionswriteenv(versiondict: dict): + for key, value in versiondict.items(): + os.environ[key] = value + +def versionsreadjsonfilesaveenv(inputfile = 'versions.json'): + versions = versionsreadjson(inputfile) + print(f"Versions: {versions}") + versionswriteenv(versions) + + # Remove the versions.json file if we are in the lab to ensure SOD doesn't pick it up + if "PERFLAB_INLAB" in os.environ and os.environ["PERFLAB_INLAB"] == "1": + os.remove(inputfile) \ No newline at end of file diff --git a/src/tools/Reporting/Reporting/IEnvironment.cs b/src/tools/Reporting/Reporting/IEnvironment.cs index c7dbfb9b002..deda72fc83e 100644 --- a/src/tools/Reporting/Reporting/IEnvironment.cs +++ b/src/tools/Reporting/Reporting/IEnvironment.cs @@ -2,14 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.Collections.Generic; -using System.Text; +using System.Collections; namespace Reporting { public interface IEnvironment { string GetEnvironmentVariable(string variable); + IDictionary GetEnvironmentVariables(); } } diff --git a/src/tools/Reporting/Reporting/Reporter.cs b/src/tools/Reporting/Reporting/Reporter.cs index ed797a01bec..57e896c346f 100644 --- a/src/tools/Reporting/Reporting/Reporter.cs +++ b/src/tools/Reporting/Reporting/Reporter.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -88,14 +89,13 @@ private void Init() BuildName = environment.GetEnvironmentVariable("PERFLAB_BUILDNUM"), TimeStamp = DateTime.Parse(environment.GetEnvironmentVariable("PERFLAB_BUILDTIMESTAMP")), }; - build.AdditionalData["productVersion"] = environment.GetEnvironmentVariable("DOTNET_VERSION"); - // Additional Data we only want populated if it is available - if (environment.GetEnvironmentVariable("MAUI_VERSION") != null) - { - build.AdditionalData["mauiVersion"] = environment.GetEnvironmentVariable("MAUI_VERSION"); - } - + foreach (DictionaryEntry entry in environment.GetEnvironmentVariables()){ + if (entry.Key.ToString().EndsWith("version", ignoreCase: true, culture: CultureInfo.InvariantCulture)) + { + build.AdditionalData[entry.Key.ToString()] = entry.Value.ToString(); + } + } } public string GetJson() { From 2cfb5fea34f32ae38e0335b85b3047e86c9d4820 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 13 Dec 2022 17:01:40 -0800 Subject: [PATCH 39/48] Resetup testing. --- azure-pipelines.yml | 50 ++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 04eeeca0611..0b10495f794 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -248,7 +248,7 @@ jobs: channels: - release/7.0 - # Maui scenario benchmarks + # Maui Android scenario benchmarks - template: /eng/performance/build_machine_matrix.yml parameters: jobTemplate: /eng/performance/scenarios.yml @@ -383,25 +383,37 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64-android-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + channels: + - release/7.0 - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + # - job: Synchronize + # pool: + # name: NetCore1ESPool-Internal-NoMSI + # demands: ImageOverride -equals 1es-windows-2019 + # steps: + # - task: UseDotNet@2 + # displayName: Install .NET 6.0 runtime + # inputs: + # version: 6.x + + # - script: dotnet tool restore + + # - task: AzureCLI@2 + # inputs: + # azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + # scriptType: ps + # scriptLocation: inlineScript + # inlineScript: | + # Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} ################################################ # Scheduled Private jobs (Weekly) From 06139f55c3fa546a9646b1c90b7c16a57b1d2a0f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 13 Dec 2022 17:58:54 -0800 Subject: [PATCH 40/48] Fix GetEnvironmentVariables not being found for environment. --- src/tools/Reporting/Reporting/EnvironmentProvider.cs | 1 + src/tools/Reporting/Reporting/IEnvironment.cs | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tools/Reporting/Reporting/EnvironmentProvider.cs b/src/tools/Reporting/Reporting/EnvironmentProvider.cs index 90d28729284..11336a24fce 100644 --- a/src/tools/Reporting/Reporting/EnvironmentProvider.cs +++ b/src/tools/Reporting/Reporting/EnvironmentProvider.cs @@ -11,5 +11,6 @@ namespace Reporting public class EnvironmentProvider : IEnvironment { public string GetEnvironmentVariable(string variable) => Environment.GetEnvironmentVariable(variable); + public System.Collections.IDictionary GetEnvironmentVariables() => Environment.GetEnvironmentVariables(); } } diff --git a/src/tools/Reporting/Reporting/IEnvironment.cs b/src/tools/Reporting/Reporting/IEnvironment.cs index deda72fc83e..409edd14285 100644 --- a/src/tools/Reporting/Reporting/IEnvironment.cs +++ b/src/tools/Reporting/Reporting/IEnvironment.cs @@ -2,13 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections; - namespace Reporting { public interface IEnvironment { string GetEnvironmentVariable(string variable); - IDictionary GetEnvironmentVariables(); + System.Collections.IDictionary GetEnvironmentVariables(); } } From d7e7405f17ef8b5dd8d5bf842e7524334a2dddc4 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 14 Dec 2022 11:09:36 -0800 Subject: [PATCH 41/48] Fix versionswritejson path in the pre.pys to not start with .\. --- src/scenarios/mauiandroid/pre.py | 2 +- src/scenarios/mauiandroidpodcast/pre.py | 2 +- src/scenarios/mauiblazorandroid/pre.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 71d104f1e43..63d98914d75 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -37,5 +37,5 @@ # Copy the MauiVersion to a file so we have it on the machine maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "maui_version": maui_version } -versionswritejson(version_dict, rf".\{precommands.output}\versions.json") +versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index 166f2a666b1..f6329e9cbf2 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -31,5 +31,5 @@ maui_version = GetVersionFromDll(f".\{const.APPDIR}\src\Mobile\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "maui_version": maui_version } -versionswritejson(version_dict, rf".\{precommands.output}\versions.json") +versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index 6e75fc16590..f9221a70d39 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -74,5 +74,5 @@ # Copy the MauiVersion to a file so we have it on the machine maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "maui_version": maui_version } -versionswritejson(version_dict, rf".\{precommands.output}\versions.json") +versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") From 40a176d10e0ea483c5c6b5dc549f425d923fed9c Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 14 Dec 2022 11:26:15 -0800 Subject: [PATCH 42/48] Fix Reporting.Tests by adding in HELIX_WORKITEM_FRIENDLYNAME environment variable. --- .../Reporting/Reporting.Tests/EnvironmentProviderMock.cs | 8 +++++++- src/tools/Reporting/Reporting.Tests/ReporterTests.cs | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs b/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs index cf33c408fe3..1939998e176 100644 --- a/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs +++ b/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections; using System.Collections.Generic; namespace Reporting.Tests @@ -14,6 +15,10 @@ public string GetEnvironmentVariable(string variable) { return Vars?[variable]; } + public IDictionary GetEnvironmentVariables() + { + return Vars; + } } internal class PerfLabEnvironmentProviderMock : EnvironmentProviderMockBase { @@ -23,6 +28,7 @@ public PerfLabEnvironmentProviderMock() { {"PERFLAB_INLAB", "1" }, {"HELIX_CORRELATION_ID","testCorrelationId" }, + {"HELIX_WORKITEM_FRIENDLYNAME","Test Friendly Name" }, {"PERFLAB_PERFHASH","testPerfHash" }, {"PERFLAB_QUEUE","testQueue" }, {"PERFLAB_REPO","testRepo" }, @@ -34,7 +40,7 @@ public PerfLabEnvironmentProviderMock() {"PERFLAB_HIDDEN", "false" }, {"PERFLAB_RUNNAME", "testRunName" }, {"PERFLAB_CONFIGS", "KEY1=VALUE1;KEY2=VALUE2" }, - {"PERFLAB_BUILDTIMESTAMP", new DateTime(1970,1,1).ToString("o") }, + {"PERFLAB_BUILDTIMESTAMP", new DateTime(1970,1,1).ToString("o") } }; } diff --git a/src/tools/Reporting/Reporting.Tests/ReporterTests.cs b/src/tools/Reporting/Reporting.Tests/ReporterTests.cs index c354d23b90f..dca49fe5314 100644 --- a/src/tools/Reporting/Reporting.Tests/ReporterTests.cs +++ b/src/tools/Reporting/Reporting.Tests/ReporterTests.cs @@ -106,6 +106,7 @@ public void JsonCanBeGenerated() var jsonObj = JsonConvert.DeserializeAnonymousType(jsonString, jsonType); Assert.Equal(environment.GetEnvironmentVariable("HELIX_CORRELATION_ID"), jsonObj.Run.CorrelationId); + Assert.Equal(environment.GetEnvironmentVariable("HELIX_WORKITEM_FRIENDLYNAME"), jsonObj.Run.WorkItemName); Assert.Equal(environment.GetEnvironmentVariable("PERFLAB_PERFHASH"), jsonObj.Run.PerfRepoHash); Assert.Equal(environment.GetEnvironmentVariable("PERFLAB_QUEUE"), jsonObj.Run.Queue); Assert.Equal(environment.GetEnvironmentVariable("PERFLAB_REPO"), jsonObj.Build.Repo); From 330d4574ef019b2fa3f8e9428b954bb959ecd319 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 14 Dec 2022 11:36:18 -0800 Subject: [PATCH 43/48] Fix timeout time, should be parsable now. --- eng/performance/maui_scenarios_android.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index a435f634691..270288b148a 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -22,7 +22,7 @@ - 30:00 + 00:30 From aecc325fb529506f620bedfbe4b0d072609a2768 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 15 Dec 2022 11:09:00 -0800 Subject: [PATCH 44/48] Add special cases to the reporter for environment variable versions and setup the current maui android scenarios to use the non-special case. --- src/scenarios/mauiandroid/pre.py | 2 +- src/scenarios/mauiandroidpodcast/pre.py | 2 +- src/scenarios/mauiblazorandroid/pre.py | 2 +- src/tools/Reporting/Reporting/Reporter.cs | 9 ++++++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 63d98914d75..145fde0473f 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -36,6 +36,6 @@ # Copy the MauiVersion to a file so we have it on the machine maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") -version_dict = { "maui_version": maui_version } +version_dict = { "mauiVersion": maui_version } versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index f6329e9cbf2..ae724c551bb 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -30,6 +30,6 @@ RemoveAABFiles(precommands.output) maui_version = GetVersionFromDll(f".\{const.APPDIR}\src\Mobile\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") -version_dict = { "maui_version": maui_version } +version_dict = { "mauiVersion": maui_version } versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index f9221a70d39..2a028fe28ed 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -73,6 +73,6 @@ # Copy the MauiVersion to a file so we have it on the machine maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") -version_dict = { "maui_version": maui_version } +version_dict = { "mauiVersion": maui_version } versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/tools/Reporting/Reporting/Reporter.cs b/src/tools/Reporting/Reporting/Reporter.cs index 57e896c346f..d356418f3ea 100644 --- a/src/tools/Reporting/Reporting/Reporter.cs +++ b/src/tools/Reporting/Reporting/Reporter.cs @@ -93,7 +93,14 @@ private void Init() foreach (DictionaryEntry entry in environment.GetEnvironmentVariables()){ if (entry.Key.ToString().EndsWith("version", ignoreCase: true, culture: CultureInfo.InvariantCulture)) { - build.AdditionalData[entry.Key.ToString()] = entry.Value.ToString(); + // Special case the original two special cases, MAUI_VERSION is only needed because runtime based runs use MAUI_VERSION + if(entry.Key.ToString().Equals("DOTNET_VERSION", StringComparison.InvariantCultureIgnoreCase)){ + build.AdditionalData["productVersion"] = entry.Value.ToString(); + } else if(entry.Key.ToString().Equals("MAUI_VERSION", StringComparison.InvariantCultureIgnoreCase)){ + build.AdditionalData["mauiVersion"] = entry.Value.ToString(); + } else { + build.AdditionalData[entry.Key.ToString()] = entry.Value.ToString(); + } } } } From b13d06a09c6425f6bfaab8d129409c970c4b7930 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 16 Dec 2022 10:54:09 -0800 Subject: [PATCH 45/48] Move GetVersionFromDLL to the version manager shared script since it can be used to get values from any Dll on systems with powershell support. --- src/scenarios/mauiandroid/pre.py | 6 +++--- src/scenarios/mauiandroidpodcast/pre.py | 6 +++--- src/scenarios/mauiblazorandroid/pre.py | 6 +++--- src/scenarios/mauishared/mauisharedpython.py | 4 ---- src/scenarios/shared/versionmanager.py | 7 ++++++- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 145fde0473f..4ae4d0c5714 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -3,11 +3,11 @@ ''' import sys import requests -from mauishared.mauisharedpython import RemoveAABFiles, GetVersionFromDll +from mauishared.mauisharedpython import RemoveAABFiles from performance.logger import setup_loggers, getLogger from shared import const from shared.precommands import PreCommands -from shared.versionmanager import versionswritejson +from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell from test import EXENAME setup_loggers(True) @@ -35,7 +35,7 @@ RemoveAABFiles(precommands.output) # Copy the MauiVersion to a file so we have it on the machine -maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +maui_version = GetVersionFromDllPowershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index ae724c551bb..8fc9c1ab665 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -3,10 +3,10 @@ ''' import requests import subprocess -from mauishared.mauisharedpython import RemoveAABFiles, GetVersionFromDll +from mauishared.mauisharedpython import RemoveAABFiles from performance.logger import setup_loggers, getLogger from shared.precommands import PreCommands -from shared.versionmanager import versionswritejson +from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell from shared import const setup_loggers(True) @@ -29,7 +29,7 @@ # Remove the aab files as we don't need them, this saves space RemoveAABFiles(precommands.output) -maui_version = GetVersionFromDll(f".\{const.APPDIR}\src\Mobile\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +maui_version = GetVersionFromDllPowershell(f".\{const.APPDIR}\src\Mobile\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index 2a028fe28ed..1b086f8d763 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -3,11 +3,11 @@ ''' import sys import requests -from mauishared.mauisharedpython import RemoveAABFiles, GetVersionFromDll +from mauishared.mauisharedpython import RemoveAABFiles from performance.logger import setup_loggers, getLogger from shared import const from shared.precommands import PreCommands -from shared.versionmanager import versionswritejson +from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell from test import EXENAME setup_loggers(True) @@ -72,7 +72,7 @@ RemoveAABFiles(precommands.output) # Copy the MauiVersion to a file so we have it on the machine -maui_version = GetVersionFromDll(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +maui_version = GetVersionFromDllPowershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versionswritejson(version_dict, rf"{precommands.output}\versions.json") print(f"Versions: {version_dict}") diff --git a/src/scenarios/mauishared/mauisharedpython.py b/src/scenarios/mauishared/mauisharedpython.py index d3b97657fb4..127c712bfee 100644 --- a/src/scenarios/mauishared/mauisharedpython.py +++ b/src/scenarios/mauishared/mauisharedpython.py @@ -7,7 +7,3 @@ def RemoveAABFiles(output_dir="."): for file in file_list: if file.endswith(".aab"): os.remove(os.path.join(output_dir, file)) - -def GetVersionFromDll(dll_path: str): - result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - return result.stdout.decode('utf-8').strip() diff --git a/src/scenarios/shared/versionmanager.py b/src/scenarios/shared/versionmanager.py index e6c26505a42..acaf5ed0bb2 100644 --- a/src/scenarios/shared/versionmanager.py +++ b/src/scenarios/shared/versionmanager.py @@ -3,6 +3,7 @@ ''' import json import os +import subprocess def versionswritejson(versiondict: dict, outputfile = 'versions.json'): with open(outputfile, 'w') as file: @@ -23,4 +24,8 @@ def versionsreadjsonfilesaveenv(inputfile = 'versions.json'): # Remove the versions.json file if we are in the lab to ensure SOD doesn't pick it up if "PERFLAB_INLAB" in os.environ and os.environ["PERFLAB_INLAB"] == "1": - os.remove(inputfile) \ No newline at end of file + os.remove(inputfile) + +def GetVersionFromDllPowershell(dll_path: str): + result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + return result.stdout.decode('utf-8').strip() From 47aeff583e9a9892595cf600d97d40d74dcafe9a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 16 Dec 2022 12:27:25 -0800 Subject: [PATCH 46/48] Undo yml testing changes. --- azure-pipelines.yml | 49 ++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0b10495f794..f1147508d75 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -380,40 +380,25 @@ jobs: ################################################ # Manually Triggered Job ################################################ + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + - script: dotnet tool restore -- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64-android-arm64 - isPublic: false - jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj - channels: - - release/7.0 - - # - job: Synchronize - # pool: - # name: NetCore1ESPool-Internal-NoMSI - # demands: ImageOverride -equals 1es-windows-2019 - # steps: - # - task: UseDotNet@2 - # displayName: Install .NET 6.0 runtime - # inputs: - # version: 6.x - - # - script: dotnet tool restore - - # - task: AzureCLI@2 - # inputs: - # azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - # scriptType: ps - # scriptLocation: inlineScript - # inlineScript: | - # Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} ################################################ # Scheduled Private jobs (Weekly) From 966861d221624e3c6c01be05438dda8baa66dda9 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 28 Dec 2022 13:59:13 -0800 Subject: [PATCH 47/48] Fixed type and added in manual run conditional. --- azure-pipelines.yml | 3 +++ src/scenarios/mauiandroidpodcast/pre.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f1147508d75..3f85b7f5d94 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -380,6 +380,9 @@ jobs: ################################################ # Manually Triggered Job ################################################ + + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - job: Synchronize pool: name: NetCore1ESPool-Internal-NoMSI diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index 8fc9c1ab665..9714e5c9d52 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -17,7 +17,7 @@ f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', f'{precommands.framework[:6]}', '--single-branch', '--depth', '1']) -subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, for their deletion seperately +subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) precommands.existing(projectdir='./dotnet-podcasts',projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') From 65f5404ffad4a00d9575500990f5094f3dba1012 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Jan 2023 11:27:59 -0800 Subject: [PATCH 48/48] Backport changes made to main for final version. --- src/scenarios/mauiandroid/pre.py | 23 ++++++++----------- src/scenarios/mauiandroid/test.py | 4 ++-- src/scenarios/mauiandroidpodcast/pre.py | 28 +++++++++++------------ src/scenarios/mauiandroidpodcast/test.py | 6 ++--- src/scenarios/mauiblazorandroid/pre.py | 23 ++++++++----------- src/scenarios/mauiblazorandroid/test.py | 6 ++--- src/scenarios/shared/mauisharedpython.py | 24 +++++++++++++++++++ src/scenarios/shared/runner.py | 18 +++++++-------- src/scenarios/shared/versionmanager.py | 16 ++++++------- src/tools/Reporting/Reporting/Reporter.cs | 11 ++++++--- 10 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 src/scenarios/shared/mauisharedpython.py diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index 4ae4d0c5714..ce508b17fe6 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -2,23 +2,17 @@ pre-command ''' import sys -import requests -from mauishared.mauisharedpython import RemoveAABFiles from performance.logger import setup_loggers, getLogger from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui from shared.precommands import PreCommands -from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell from test import EXENAME setup_loggers(True) precommands = PreCommands() - -# Download what we need -with open ("MauiNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) - -precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) +install_versioned_maui(precommands) # Setup the Maui folder precommands.new(template='maui', @@ -32,10 +26,13 @@ precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) # Remove the aab files as we don't need them, this saves space -RemoveAABFiles(precommands.output) +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) # Copy the MauiVersion to a file so we have it on the machine -maui_version = GetVersionFromDllPowershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } -versionswritejson(version_dict, rf"{precommands.output}\versions.json") -print(f"Versions: {version_dict}") +versions_write_json(version_dict, rf"{output_dir}\versions.json") +print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiandroid/test.py b/src/scenarios/mauiandroid/test.py index 75fb182f2c7..5976d343ee1 100644 --- a/src/scenarios/mauiandroid/test.py +++ b/src/scenarios/mauiandroid/test.py @@ -3,12 +3,12 @@ ''' from shared.const import PUBDIR from shared.runner import TestTraits, Runner -from shared.versionmanager import versionsreadjsonfilesaveenv +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiAndroidDefault' if __name__ == "__main__": - versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json") + versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") traits = TestTraits(exename=EXENAME, guiapp='false', diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py index 9714e5c9d52..30892a686f7 100644 --- a/src/scenarios/mauiandroidpodcast/pre.py +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -1,35 +1,35 @@ ''' pre-command ''' -import requests import subprocess -from mauishared.mauisharedpython import RemoveAABFiles from performance.logger import setup_loggers, getLogger from shared.precommands import PreCommands -from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell from shared import const setup_loggers(True) precommands = PreCommands() +install_versioned_maui(precommands) -# Download what we need -with open ("MauiNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) - -subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', f'{precommands.framework[:6]}', '--single-branch', '--depth', '1']) +branch = f'{precommands.framework[:6]}' +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1']) subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately -precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) -precommands.existing(projectdir='./dotnet-podcasts',projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') +precommands.existing(projectdir='./dotnet-podcasts', projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') # Build the APK precommands._restore() precommands.execute(['--no-restore']) # Remove the aab files as we don't need them, this saves space -RemoveAABFiles(precommands.output) +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) -maui_version = GetVersionFromDllPowershell(f".\{const.APPDIR}\src\Mobile\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } -versionswritejson(version_dict, rf"{precommands.output}\versions.json") -print(f"Versions: {version_dict}") +versions_write_json(version_dict, rf"{output_dir}\versions.json") +print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiandroidpodcast/test.py b/src/scenarios/mauiandroidpodcast/test.py index 1e557a00e46..d715ae7eb3a 100644 --- a/src/scenarios/mauiandroidpodcast/test.py +++ b/src/scenarios/mauiandroidpodcast/test.py @@ -3,14 +3,14 @@ ''' from shared.const import PUBDIR from shared.runner import TestTraits, Runner -from shared.versionmanager import versionsreadjsonfilesaveenv +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiAndroidPodcast' if __name__ == "__main__": - versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json") + versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") traits = TestTraits(exename=EXENAME, guiapp='false', ) - Runner(traits).run() + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py index 1b086f8d763..c2c9f71f41d 100644 --- a/src/scenarios/mauiblazorandroid/pre.py +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -2,22 +2,16 @@ pre-command ''' import sys -import requests -from mauishared.mauisharedpython import RemoveAABFiles from performance.logger import setup_loggers, getLogger from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui from shared.precommands import PreCommands -from shared.versionmanager import versionswritejson, GetVersionFromDllPowershell +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell from test import EXENAME setup_loggers(True) precommands = PreCommands() - -# Download what we need -with open ("MauiNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{precommands.framework[:6]}/NuGet.config', allow_redirects=True).content) - -precommands.install_workload('maui', ['--configfile', 'MauiNuGet.config']) +install_versioned_maui(precommands) # Setup the Maui folder precommands.new(template='maui-blazor', @@ -69,10 +63,13 @@ # Build the APK precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) -RemoveAABFiles(precommands.output) +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) # Copy the MauiVersion to a file so we have it on the machine -maui_version = GetVersionFromDllPowershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } -versionswritejson(version_dict, rf"{precommands.output}\versions.json") -print(f"Versions: {version_dict}") +versions_write_json(version_dict, rf"{output_dir}\versions.json") +print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiblazorandroid/test.py b/src/scenarios/mauiblazorandroid/test.py index c1b2ce9542a..9363fdfeb5b 100644 --- a/src/scenarios/mauiblazorandroid/test.py +++ b/src/scenarios/mauiblazorandroid/test.py @@ -3,14 +3,14 @@ ''' from shared.const import PUBDIR from shared.runner import TestTraits, Runner -from shared.versionmanager import versionsreadjsonfilesaveenv +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiBlazorAndroidDefault' if __name__ == "__main__": - versionsreadjsonfilesaveenv(rf".\{PUBDIR}\versions.json") + versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") traits = TestTraits(exename=EXENAME, guiapp='false', ) - Runner(traits).run() + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/shared/mauisharedpython.py b/src/scenarios/shared/mauisharedpython.py new file mode 100644 index 00000000000..a61be2cfa65 --- /dev/null +++ b/src/scenarios/shared/mauisharedpython.py @@ -0,0 +1,24 @@ +import subprocess +import os +import requests +from shared.precommands import PreCommands + +# Remove the aab files as we don't need them, this saves space in the correlation payload +def remove_aab_files(output_dir="."): + file_list = os.listdir(output_dir) + for file in file_list: + if file.endswith(".aab"): + os.remove(os.path.join(output_dir, file)) + +def install_versioned_maui(precommands): + target_framework_wo_platform = precommands.framework.split('-')[0] + + # Download what we need + with open ("MauiNuGet.config", "wb") as f: + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content) + + workload_install_args = ['--configfile', 'MauiNuGet.config'] + if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 + workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] + + precommands.install_workload('maui', workload_install_args) \ No newline at end of file diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 147c1f5b225..86acf00715d 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -440,6 +440,14 @@ def run(self): getLogger().info(f"Animation values successfully set to {animationValue}.") try: + stopAppCmd = [ + adb.stdout.strip(), + 'shell', + 'am', + 'force-stop', + self.packagename + ] + installCmd = xharnesscommand() + [ 'android', 'install', @@ -493,7 +501,7 @@ def run(self): # Actual testing some run stuff getLogger().info("Test run to check if permissions are needed") - activityname = getActivity.stdout + activityname = getActivity.stdout.strip() # -W in the start command waits for the app to finish initial draw. startAppCmd = [ @@ -509,16 +517,8 @@ def run(self): testRun.run() testRunStats = re.findall(runSplitRegex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") - time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle - stopAppCmd = [ - adb.stdout.strip(), - 'shell', - 'am', - 'force-stop', - self.packagename - ] RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: diff --git a/src/scenarios/shared/versionmanager.py b/src/scenarios/shared/versionmanager.py index acaf5ed0bb2..3eeaf795a5a 100644 --- a/src/scenarios/shared/versionmanager.py +++ b/src/scenarios/shared/versionmanager.py @@ -5,27 +5,27 @@ import os import subprocess -def versionswritejson(versiondict: dict, outputfile = 'versions.json'): +def versions_write_json(versiondict: dict, outputfile = 'versions.json'): with open(outputfile, 'w') as file: json.dump(versiondict, file) -def versionsreadjson(inputfile = 'versions.json'): +def versions_read_json(inputfile = 'versions.json'): with open(inputfile, 'r') as file: return json.load(file) -def versionswriteenv(versiondict: dict): +def versions_write_env(versiondict: dict): for key, value in versiondict.items(): os.environ[key] = value -def versionsreadjsonfilesaveenv(inputfile = 'versions.json'): - versions = versionsreadjson(inputfile) +def versions_read_json_file_save_env(inputfile = 'versions.json'): + versions = versions_read_json(inputfile) print(f"Versions: {versions}") - versionswriteenv(versions) + versions_write_env(versions) # Remove the versions.json file if we are in the lab to ensure SOD doesn't pick it up if "PERFLAB_INLAB" in os.environ and os.environ["PERFLAB_INLAB"] == "1": os.remove(inputfile) -def GetVersionFromDllPowershell(dll_path: str): +def get_version_from_dll_powershell(dll_path: str): result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - return result.stdout.decode('utf-8').strip() + return result.stdout.decode('utf-8').strip() \ No newline at end of file diff --git a/src/tools/Reporting/Reporting/Reporter.cs b/src/tools/Reporting/Reporting/Reporter.cs index d356418f3ea..097a92aaf2d 100644 --- a/src/tools/Reporting/Reporting/Reporter.cs +++ b/src/tools/Reporting/Reporting/Reporter.cs @@ -90,8 +90,13 @@ private void Init() TimeStamp = DateTime.Parse(environment.GetEnvironmentVariable("PERFLAB_BUILDTIMESTAMP")), }; - foreach (DictionaryEntry entry in environment.GetEnvironmentVariables()){ - if (entry.Key.ToString().EndsWith("version", ignoreCase: true, culture: CultureInfo.InvariantCulture)) + foreach (DictionaryEntry entry in environment.GetEnvironmentVariables()) + { + if (entry.Key.ToString().Equals("PERFLAB_TARGET_FRAMEWORKS", StringComparison.InvariantCultureIgnoreCase)) + { + build.AdditionalData["targetFrameworks"] = entry.Value.ToString(); + } + else if(entry.Key.ToString().EndsWith("version", true, CultureInfo.InvariantCulture)) { // Special case the original two special cases, MAUI_VERSION is only needed because runtime based runs use MAUI_VERSION if(entry.Key.ToString().Equals("DOTNET_VERSION", StringComparison.InvariantCultureIgnoreCase)){ @@ -166,4 +171,4 @@ private string LeftJustify(string str, int width) public bool InLab => environment.GetEnvironmentVariable("PERFLAB_INLAB")?.Equals("1") ?? false; } -} +} \ No newline at end of file