From 734574602820a32ff4a3abc7698d6f3f1a4492b7 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 01:33:05 +0530 Subject: [PATCH 01/18] feat(playwrighttesting): separate api, data processing and utility layers --- .../src/Constants.cs | 121 +++ .../src/Implementation/CloudRunErrorParser.cs | 69 ++ .../src/Implementation/ConsoleWriter.cs | 23 + .../src/Implementation/Logger.cs | 50 ++ .../src/Implementation/ServiceClient.cs | 63 ++ .../src/Interface/ICloudRunErrorParser.cs | 13 + .../src/Interface/IConsoleWriter.cs | 10 + .../src/Interface/IDataProcessor.cs | 15 + .../src/Interface/ILogger.cs | 13 + .../src/Interface/IServiceClient.cs | 15 + .../src/Interface/ITestProcessor.cs | 15 + .../src/Model/CloudRunMetadata.cs | 19 + .../src/PlaywrightReporter.cs | 710 ++---------------- .../src/Processor/DataProcessor.cs | 199 +++++ .../src/Processor/TestProcessor.cs | 287 +++++++ .../src/Utility/CiInfoProvider.cs | 17 +- .../src/Utility/Constants.cs | 128 ---- .../src/Utility/Logger.cs | 25 - .../src/Utility/ReporterUtils.cs | 66 +- 19 files changed, 1034 insertions(+), 824 deletions(-) create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs delete mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs delete mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 4b1f0b43a2bb..5753ed87f524 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -1,6 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using System.Collections.Generic; +using System.Text.RegularExpressions; + namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; /// @@ -183,6 +187,7 @@ internal class Constants // Default constants internal static readonly string s_default_os = ServiceOs.Linux; internal static readonly string s_default_expose_network = ""; + internal static readonly string s_pLAYWRIGHT_SERVICE_DEBUG = "PLAYWRIGHT_SERVICE_DEBUG"; // Entra id access token constants internal static readonly int s_entra_access_token_lifetime_left_threshold_in_minutes_for_rotation = 15; @@ -204,3 +209,119 @@ internal class Constants internal static readonly string s_playwright_service_reporting_url_environment_variable = "PLAYWRIGHT_SERVICE_REPORTING_URL"; internal static readonly string s_playwright_service_workspace_id_environment_variable = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; } + +internal class OSConstants +{ + internal static readonly string s_lINUX = "Linux"; + internal static readonly string s_wINDOWS = "Windows"; + internal static readonly string s_mACOS = "MacOS"; +} + +internal class ReporterConstants +{ + internal static readonly string s_executionIdPropertyIdentifier = "ExecutionId"; + internal static readonly string s_parentExecutionIdPropertyIdentifier = "ParentExecId"; + internal static readonly string s_testTypePropertyIdentifier = "TestType"; + internal static readonly string s_sASUriSeparator = "?"; + internal static readonly string s_portalBaseUrl = "https://playwright.microsoft.com/workspaces/"; + internal static readonly string s_reportingRoute = "/runs/"; + internal static readonly string s_reportingAPIVersion_2024_04_30_preview = "2024-04-30-preview"; + internal static readonly string s_reportingAPIVersion_2024_05_20_preview = "2024-05-20-preview"; + internal static readonly string s_pLAYWRIGHT_SERVICE_REPORTING_URL = "PLAYWRIGHT_SERVICE_REPORTING_URL"; + internal static readonly string s_pLAYWRIGHT_SERVICE_WORKSPACE_ID = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; + internal static readonly string s_aPPLICATION_JSON = "application/json"; +} + +internal class CIConstants +{ + internal static readonly string s_gITHUB_ACTIONS = "GitHub Actions"; + internal static readonly string s_aZURE_DEVOPS = "Azure DevOps"; + internal static readonly string s_dEFAULT = "Default"; +} + +internal class TestCaseResultStatus +{ + internal static readonly string s_pASSED = "passed"; + internal static readonly string s_fAILED = "failed"; + internal static readonly string s_sKIPPED = "skipped"; + internal static readonly string s_iNCONCLUSIVE = "inconclusive"; +} + +internal class TestResultError +{ + internal string? Key { get; set; } = string.Empty; + internal string? Message { get; set; } = string.Empty; + internal Regex Pattern { get; set; } = new Regex(string.Empty); + internal TestErrorType Type { get; set; } +} + +internal enum TestErrorType +{ + Scalable +} + +internal static class TestResultErrorConstants +{ + public static List ErrorConstants = new() + { + new TestResultError + { + Key = "Unauthorized_Scalable", + Message = "The authentication token provided is invalid. Please check the token and try again.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*401 Unauthorized)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "NoPermissionOnWorkspace_Scalable", + Message = @"You do not have the required permissions to run tests. This could be because: + + a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator. + b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant '.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*CheckAccess API call with non successful response)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "InvalidWorkspace_Scalable", + Message = "The specified workspace does not exist. Please verify your workspace settings.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "AccessKeyBasedAuthNotSupported_Scalable", + Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "ServiceUnavailable_Scalable", + Message = "The service is currently unavailable. Please check the service status and try again.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*503 Service Unavailable)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "GatewayTimeout_Scalable", + Message = "The request to the service timed out. Please try again later.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*504 Gateway Timeout)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "QuotaLimitError_Scalable", + Message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded.", + Pattern = new Regex(@"(Timeout .* exceeded)(?=[\s\S]*ws connecting)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "BrowserConnectionError_Scalable", + Message = "The service is currently unavailable. Please try again after some time.", + Pattern = new Regex(@"Target page, context or browser has been closed", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + } + }; +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs new file mode 100644 index 000000000000..bc0dae518d9f --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation +{ + internal class CloudRunErrorParser : ICloudRunErrorParser + { + private List InformationalMessages { get; set; } = new(); + private List ProcessedErrorMessageKeys { get; set; } = new(); + private readonly ILogger _logger; + private readonly IConsoleWriter _consoleWriter; + public CloudRunErrorParser(ILogger? logger = null, IConsoleWriter? consoleWriter = null) + { + _logger = logger ?? new Logger(); + _consoleWriter = consoleWriter ?? new ConsoleWriter(); + } + + public bool TryPushMessageAndKey(string? message, string? key) + { + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(message)) + { + return false; + } + if (ProcessedErrorMessageKeys.Contains(key!)) + { + return false; + } + _logger.Info($"Adding message with key: {key}"); + + ProcessedErrorMessageKeys.Add(key!); + InformationalMessages.Add(message!); + return true; + } + + public void PushMessage(string message) + { + InformationalMessages.Add(message); + } + + public void DisplayMessages() + { + if (InformationalMessages.Count > 0) + _consoleWriter.WriteLine(); + int index = 1; + foreach (string message in InformationalMessages) + { + _consoleWriter.WriteLine($"{index++}) {message}"); + } + } + + public void HandleScalableRunErrorMessage(string? message) + { + if (string.IsNullOrEmpty(message)) + { + return; + } + foreach (TestResultError testResultErrorObj in TestResultErrorConstants.ErrorConstants) + { + if (testResultErrorObj.Pattern.IsMatch(message)) + { + TryPushMessageAndKey(testResultErrorObj.Message, testResultErrorObj.Key); + } + } + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs new file mode 100644 index 000000000000..5948545c6593 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation +{ + internal class ConsoleWriter : IConsoleWriter + { + public void WriteLine(string? message = null) + { + if (message == null) + { + Console.WriteLine(); + } + else + { + Console.WriteLine(message); + } + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs new file mode 100644 index 000000000000..b1d508840637 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; + +internal enum LogLevel +{ + Debug, + Info, + Warning, + Error +} + +internal class Logger : ILogger +{ + internal static bool EnableDebug { get { return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.s_pLAYWRIGHT_SERVICE_DEBUG)); } set { } } + +#pragma warning disable CA1822 // Mark members as static + private void Log(LogLevel level, string message) +#pragma warning restore CA1822 // Mark members as static + { + if (EnableDebug) + { + Console.WriteLine($"{DateTime.Now} [{level}]: {message}"); + } + } + + public void Debug(string message) + { + Log(LogLevel.Debug, message); + } + + public void Error(string message) + { + Log(LogLevel.Error, message); + } + + public void Info(string message) + { + Log(LogLevel.Info, message); + } + + public void Warning(string message) + { + Log(LogLevel.Warning, message); + } +}; diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs new file mode 100644 index 000000000000..1a459b574f8c --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +using System; +using Azure.Core.Serialization; +using Azure.Core; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using System.Text.Json; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation +{ + internal class ServiceClient : IServiceClient + { + private readonly ReportingTestResultsClient _reportingTestResultsClient; + private readonly ReportingTestRunsClient _reportingTestRunsClient; + private readonly CloudRunMetadata _cloudRunMetadata; + private static string AccessToken { get => $"Bearer {Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken)}"; set { } } + private static string CorrelationId { get => Guid.NewGuid().ToString(); set { } } + + public ServiceClient(CloudRunMetadata cloudRunMetadata, ReportingTestResultsClient? reportingTestResultsClient = null, ReportingTestRunsClient? reportingTestRunsClient = null) + { + _cloudRunMetadata = cloudRunMetadata; + _reportingTestResultsClient = reportingTestResultsClient ?? new ReportingTestResultsClient(_cloudRunMetadata.BaseUri); + _reportingTestRunsClient = reportingTestRunsClient ?? new ReportingTestRunsClient(_cloudRunMetadata.BaseUri); + } + + public TestRunDtoV2? PatchTestRunInfo(TestRunDtoV2 run) + { + Response apiResponse = _reportingTestRunsClient.PatchTestRunInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(run), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + if (apiResponse.Content != null) + { + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + } + return null; + } + + public TestRunShardDto? PatchTestRunShardInfo(int shardId, TestRunShardDto runShard) + { + Response apiResponse = _reportingTestRunsClient.PatchTestRunShardInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, shardId.ToString(), RequestContent.Create(runShard), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + if (apiResponse.Content != null) + { + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + } + return null; + } + + public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsRequest) + { + _reportingTestResultsClient.UploadBatchTestResults(_cloudRunMetadata.WorkspaceId!, RequestContent.Create(JsonSerializer.Serialize(uploadTestResultsRequest)), AccessToken, CorrelationId, null); + } + + public TestResultsUri? GetTestRunResultsUri() + { + Response response = _reportingTestRunsClient.GetTestRunResultsUri(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, AccessToken, CorrelationId, null); + if (response.Content != null) + { + return response.Content!.ToObject(new JsonObjectSerializer()); + } + return null; + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs new file mode 100644 index 000000000000..f2c507b3bc50 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface ICloudRunErrorParser + { + void HandleScalableRunErrorMessage(string? message); + bool TryPushMessageAndKey(string? message, string? key); + void PushMessage(string message); + void DisplayMessages(); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs new file mode 100644 index 000000000000..30a051fda807 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface IConsoleWriter + { + void WriteLine(string? message = null); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs new file mode 100644 index 000000000000..37f49cbd1ccf --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface IDataProcessor + { + TestRunDtoV2 GetTestRun(); + TestRunShardDto GetTestRunShard(); + TestResults GetTestCaseResultData(TestResult testResultSource); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs new file mode 100644 index 000000000000..c1912bcfaa50 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface ILogger + { + void Info(string message); + void Debug(string message); + void Warning(string message); + void Error(string message); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs new file mode 100644 index 000000000000..a9e14963acb8 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface IServiceClient + { + TestRunDtoV2? PatchTestRunInfo(TestRunDtoV2 run); + TestRunShardDto? PatchTestRunShardInfo(int shardId, TestRunShardDto runShard); + void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsRequest); + TestResultsUri? GetTestRunResultsUri(); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs new file mode 100644 index 000000000000..205332294b2e --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface ITestProcessor + { + void TestCaseResultHandler(object? sender, TestResultEventArgs e); + void TestRunStartHandler(object? sender, TestRunStartEventArgs e); + void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs new file mode 100644 index 000000000000..d80d56b4cb98 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model +{ + internal class CloudRunMetadata + { + internal string? WorkspaceId { get; set; } + internal string? RunId { get; set; } + internal Uri? BaseUri { get; set; } + internal string? PortalUrl { get; set; } + internal bool EnableResultPublish { get; set; } = true; + internal bool EnableGithubSummary { get; set; } = true; + internal DateTime TestRunStartTime { get; set; } + internal TokenDetails? AccessTokenDetails { get; set; } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs index bccfa0632517..3f5d660ee0b3 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs @@ -1,26 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using Azure.Storage.Blobs; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; -using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client; -using Microsoft.IdentityModel.JsonWebTokens; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.IO; -using System.Text.Json; -using PlaywrightConstants = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility.Constants; -using Azure.Core; -using Azure.Core.Serialization; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; @@ -29,58 +20,15 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; internal class PlaywrightReporter : ITestLoggerWithParameters { private Dictionary? _parametersDictionary; + private PlaywrightService? playwrightService; + private readonly ILogger _logger; + private TestProcessor? _testProcessor; - private bool IsInitialized { get; set; } - - private HttpClient? _httpClient; - - private ReportingTestResultsClient? _reportingTestResultsClient; - private ReportingTestRunsClient? _reportingTestRunsClient; - - private static readonly JsonWebTokenHandler s_tokenHandler = new(); - - private readonly LogLevel _logLevel = LogLevel.Debug; - - internal static string EnableConsoleLog { get => Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_DEBUG) ?? "false"; set { } } - - internal string? PortalUrl { get; set; } - - internal static string? BaseUrl { get => Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_REPORTING_URL); private set { } } - - internal static string AccessToken { get => Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_ACCESS_TOKEN) ?? ""; set { } } - - internal string? WorkspaceId { get; set; } - - internal TokenDetails? TokenDetails { get; set; } - - internal CIInfo? CIInfo { get; set; } - - internal string? RunId { get; set; } - - internal DateTime TestRunStartTime { get; private set; } - - internal int TotalTestCount { get; private set; } - - internal int PassedTestCount { get; private set; } - - internal int FailedTestCount { get; private set; } - - internal int SkippedTestCount { get; private set; } - - internal TestRunDtoV2? TestRun { get; set; } - - internal TestRunShardDto? TestRunShard { get; set; } - - internal bool EnableGithubSummary { get; set; } = true; - internal bool EnableResultPublish { get; set; } = true; - - internal List TestResults = new(); - - internal ConcurrentDictionary RawTestResultsMap = new(); - - internal PlaywrightService? playwrightService; - private List informationalMessages = new(); - private List processedErrorMessageKeys = new(); + public PlaywrightReporter() : this(null) { } // no-op + public PlaywrightReporter(ILogger? logger) + { + _logger = logger ?? new Logger(); + } public void Initialize(TestLoggerEvents events, Dictionary parameters) { @@ -88,570 +36,48 @@ public void Initialize(TestLoggerEvents events, Dictionary para _parametersDictionary = parameters; Initialize(events, _parametersDictionary[DefaultLoggerParameterNames.TestRunDirectory]!); } - public void Initialize(TestLoggerEvents events, string testResultsDirPath) { ValidateArg.NotNull(events, nameof(events)); ValidateArg.NotNullOrEmpty(testResultsDirPath, nameof(testResultsDirPath)); // Register for the events. - events.TestRunMessage += TestMessageHandler; - events.TestResult += TestResultHandler; - events.TestRunComplete += TestRunCompleteHandler; - events.TestRunStart += TestRunStartHandler; + events.TestResult += TestResultHandler; // each test run end + events.TestRunComplete += TestRunCompleteHandler; // test suite end + events.TestRunStart += TestRunStartHandler; // test suite start } #region Event Handlers - internal void TestRunStartHandler(object? sender, TestRunStartEventArgs e) { InitializePlaywrightReporter(e.TestRunCriteria.TestRunSettings!); - LogMessage("Test Run start Handler"); - if (!EnableResultPublish) - { - return; - } - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null) - { - LogErrorMessage("Test Run setup issue exiting handler"); - return; - } - - var startTime = TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); - LogMessage("Test Run start time: " + startTime); - var corelationId = Guid.NewGuid().ToString(); - var gitBasedRunName = ReporterUtils.GetRunName(CiInfoProvider.GetCIInfo()); - var runName = string.IsNullOrEmpty(gitBasedRunName) ? Guid.NewGuid().ToString() : gitBasedRunName; - var run = new TestRunDtoV2 - { - TestRunId = RunId!, - DisplayName = runName, - StartTime = startTime, - CreatorId = TokenDetails!.oid ?? "", - CreatorName = TokenDetails.userName ?? "", - //CloudRunEnabled = "false", - CloudReportingEnabled = "true", - Summary = new TestRunSummary - { - Status = "RUNNING", - StartTime = startTime, - //Projects = ["playwright-dotnet"], - //Tags = ["Nunit", "dotnet"], - //Jobs = ["playwright-dotnet"], - }, - CiConfig = new CIConfig // TODO fetch dynamically - { - Branch = CIInfo!.Branch ?? "", - Author = CIInfo.Author ?? "", - CommitId = CIInfo.CommitId ?? "", - RevisionUrl = CIInfo.RevisionUrl ?? "" - }, - TestRunConfig = new ClientConfig // TODO fetch some of these dynamically - { - Workers = 1, - PwVersion = "1.40", - Timeout = 60000, - TestType = "WebTest", - TestSdkLanguage = "Dotnet", - TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit/MSTest", Version = "3.1" }, // TODO fetch runner name MSTest/Nunit - ReporterPackageVersion = "0.0.1-dotnet", - Shards = new Shard() { Current = 0, Total = 1 } - } - }; - var shard = new TestRunShardDto - { - UploadCompleted = "false", - Summary = new TestRunShardSummary - { - Status = "RUNNING", - StartTime = startTime, - }, - TestRunConfig = new ClientConfig // TODO fetch some of these dynamically - { - Workers = 1, - PwVersion = "1.40", - Timeout = 60000, - TestType = "Functional", - TestSdkLanguage = "dotnet", - TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit", Version = "3.1" }, - ReporterPackageVersion = "0.0.1-dotnet", - Shards = new Shard() { Current = 0, Total = 1 }, - } - }; - var token = "Bearer " + AccessToken; - TestRunDtoV2? response = null; - try - { - Response apiResponse = _reportingTestRunsClient.PatchTestRunInfo(WorkspaceId, RunId, RequestContent.Create(run), "application/json", token, corelationId); - if (apiResponse.Content != null) - { - response = apiResponse.Content!.ToObject(new JsonObjectSerializer()); - } - } - catch (Exception ex) - { - Logger.Log(true, LogLevel.Error, ex.ToString()); - throw; - } - if (response != null) - { - TestRun = response; - - // Start shard - corelationId = Guid.NewGuid().ToString(); - TestRunShardDto? response1 = null; - try - { - Response apiResponse = _reportingTestRunsClient.PatchTestRunShardInfo(WorkspaceId, RunId, "1", RequestContent.Create(shard), "application/json", token, corelationId); - if (apiResponse.Content != null) - { - response1 = apiResponse.Content!.ToObject(new JsonObjectSerializer()); - } - } - catch (Exception ex) - { - Logger.Log(true, LogLevel.Error, ex.ToString()); - throw; - } - if (response1 != null) - { - TestRunShard = shard; // due to wrong response type TODO - } - else - { - Logger.Log(true, LogLevel.Error, "Run shard creation Failed"); - } - } - else - { - Logger.Log(true, LogLevel.Error, "Run creation Failed"); - } - LogMessage("Test Run start Handler completed"); - } - - internal void TestMessageHandler(object? sender, TestRunMessageEventArgs e) - { - LogMessage("Test Message Handler"); - ValidateArg.NotNull(sender, nameof(sender)); - ValidateArg.NotNull(e, nameof(e)); - LogMessage(e.Message); + _testProcessor?.TestRunStartHandler(sender, e); } internal void TestResultHandler(object? sender, TestResultEventArgs e) { - LogMessage("Test Result Handler"); - TestResults? testResult = GetTestCaseResultData(e.Result); - if (!EnableResultPublish) - { - return; - } - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null) - { - LogErrorMessage("Test Run setup issue exiting handler"); - return; - } - // Set various counts (passed tests, failed tests, total tests) - if (testResult != null) - { - TotalTestCount++; - if (testResult.Status == "failed") - { - FailedTestCount++; - } - else if (testResult.Status == "passed") - { - PassedTestCount++; - } - else if (testResult.Status == "skipped") - { - SkippedTestCount++; - } - } - if (testResult != null) - { - TestResults.Add(testResult); - } + _testProcessor?.TestCaseResultHandler(sender, e); } internal void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) { - LogMessage("Test Run End Handler"); - if (!EnableResultPublish) - { - UpdateTestRun(e); // will not publish results, but will print informational messages - return; - } - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null || TestRun == null) - { - LogErrorMessage("Test Run setup issue exiting handler"); - EnableResultPublish = false; - UpdateTestRun(e); // will not publish results, but will print informational messages - return; - } - // Upload TestResults - var corelationId = Guid.NewGuid().ToString(); - var token = "Bearer " + AccessToken; - - var body = new UploadTestResultsRequest() { Value = TestResults }; - try - { - _reportingTestResultsClient.UploadBatchTestResults(WorkspaceId, RequestContent.Create(JsonSerializer.Serialize(body)), token, corelationId, null); - LogMessage("Test Result Uploaded"); - } - catch (Exception ex) - { - LogErrorMessage(ex.Message); - } - - corelationId = Guid.NewGuid().ToString(); - TestResultsUri? sasUri = null; - Response response = _reportingTestRunsClient.GetTestRunResultsUri(WorkspaceId, RunId, token, corelationId, null); - var serializer = new JsonObjectSerializer(); - if (response.Content != null) - { - sasUri = response.Content.ToObject(serializer); - } - if (sasUri != null && !string.IsNullOrEmpty(sasUri.Uri)) - { - LogMessage("Test Run Uri: " + sasUri.ToString()); - foreach (TestResults testResult in TestResults) - { - if (RawTestResultsMap.TryGetValue(testResult.TestExecutionId!, out RawTestResult? rawResult) && rawResult != null) - { - // Upload rawResult to blob storage using sasUri - var rawTestResultJson = JsonSerializer.Serialize(rawResult); - var filePath = $"{testResult.TestExecutionId}/rawTestResult.json"; - UploadBuffer(sasUri.Uri!, rawTestResultJson, filePath); - } - else - { - LogMessage("Couldnt find rawResult for Id: " + testResult.TestExecutionId); - } - } - } - else - { - LogMessage("MPT API error: failed to upload artifacts"); - } - LogMessage("Test Results uploaded"); - // Update TestRun with CLIENT_COMPLETE - if (UpdateTestRun(e) == false) - { - LogErrorMessage("Test Run setup issue, Failed to update TestRun"); - } + _testProcessor?.TestRunCompleteHandler(sender, e); + playwrightService?.Cleanup(); } #endregion - private bool UpdateTestRun(TestRunCompleteEventArgs e) - { - if (EnableResultPublish) - { - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null || TestRun == null || TestRunShard == null) - { - // no-op - } - else - { - DateTime testRunStartedOn = DateTime.MinValue; - DateTime testRunEndedOn = DateTime.UtcNow; - long durationInMs = 0; - - var result = FailedTestCount > 0 ? "failed" : "passed"; - - if (e.ElapsedTimeInRunningTests != null) - { - testRunEndedOn = TestRunStartTime.Add(e.ElapsedTimeInRunningTests); - durationInMs = (long)e.ElapsedTimeInRunningTests.TotalMilliseconds; - } - - // Update Shard End - if (TestRunShard.Summary == null) - TestRunShard.Summary = new TestRunShardSummary(); - TestRunShard.Summary.Status = "CLIENT_COMPLETE"; - TestRunShard.Summary.StartTime = TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); - TestRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ"); - TestRunShard.Summary.TotalTime = durationInMs; - TestRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 }; - LogMessage("duration:" + durationInMs); - LogMessage("StartTime:" + TestRunShard.Summary.StartTime); - LogMessage("EndTime:" + TestRunShard.Summary.EndTime); - TestRunShard.ResultsSummary = new TestRunResultsSummary - { - NumTotalTests = TotalTestCount, - NumPassedTests = PassedTestCount, - NumFailedTests = FailedTestCount, - NumSkippedTests = SkippedTestCount, - NumFlakyTests = 0, // TODO: Implement flaky tests - Status = result - }; - TestRunShard.UploadCompleted = "true"; - var token = "Bearer " + AccessToken; - var corelationId = Guid.NewGuid().ToString(); - try - { - _reportingTestRunsClient.PatchTestRunShardInfo(WorkspaceId, RunId, "1", RequestContent.Create(TestRunShard), "application/json", token, corelationId); - } - catch (Exception ex) - { - LogErrorMessage("Test Run shard failed: " + ex.ToString()); - throw; - } - - LogMessage("TestRun Shard updated"); - playwrightService?.Cleanup(); - Console.WriteLine("Visit MPT Portal for Debugging: " + Uri.EscapeUriString(PortalUrl!)); - if (EnableGithubSummary) - GenerateMarkdownSummary(); - } - } - if (informationalMessages.Count > 0) - Console.WriteLine(); - int index = 1; - foreach (string message in informationalMessages) - { - Console.WriteLine($"{index}) {message}"); - } - return true; - } - - private TestResults GetTestCaseResultData(TestResult testResultSource) - { - if (testResultSource == null) - return new TestResults(); - - LogMessage(testResultSource.TestCase.DisplayName); - TestResults testCaseResultData = new() - { - ArtifactsPath = new List(), - - AccountId = WorkspaceId!, - RunId = RunId!, - TestExecutionId = GetExecutionId(testResultSource).ToString() - }; - testCaseResultData.TestCombinationId = testCaseResultData.TestExecutionId; // TODO check - testCaseResultData.TestId = testResultSource.TestCase.Id.ToString(); - testCaseResultData.TestTitle = testResultSource.TestCase.DisplayName; - var className = FetchTestClassName(testResultSource.TestCase.FullyQualifiedName); - testCaseResultData.SuiteTitle = className; - testCaseResultData.SuiteId = className; - testCaseResultData.FileName = FetchFileName(testResultSource.TestCase.Source); - testCaseResultData.LineNumber = testResultSource.TestCase.LineNumber; - testCaseResultData.Retry = 0; // TODO Retry and PreviousRetries - testCaseResultData.WebTestConfig = new WebTestConfig - { - JobName = CIInfo!.JobId ?? "", - //ProjectName = "playwright-dotnet", // TODO no project concept NA?? - //BrowserName = "chromium", // TODO check if possible to get from test - Os = GetCurrentOS(), - }; - //testCaseResultData.Annotations = ["windows"]; // TODO MSTest/Nunit annotation ?? - //testCaseResultData.Tags = ["windows"]; // TODO NA ?? - - TimeSpan duration = testResultSource.Duration; - testCaseResultData.ResultsSummary = new TestResultsSummary - { - Duration = (long)duration.TotalMilliseconds, // TODO fallback get from End-Start - StartTime = testResultSource.StartTime.UtcDateTime.ToString(), - Status = "inconclusive" - }; - TestOutcome outcome = testResultSource.Outcome; - switch (outcome) - { - case TestOutcome.Passed: - testCaseResultData.ResultsSummary.Status = "passed"; - testCaseResultData.Status = "passed"; - break; - case TestOutcome.Failed: - testCaseResultData.ResultsSummary.Status = "failed"; - testCaseResultData.Status = "failed"; - break; - case TestOutcome.Skipped: - testCaseResultData.ResultsSummary.Status = "skipped"; - testCaseResultData.Status = "skipped"; - break; - default: - testCaseResultData.ResultsSummary.Status = "inconclusive"; - testCaseResultData.Status = "inconclusive"; - break; - } - // errorMessage, Stacktrace - RawTestResult rawResult = GetRawResultObject(testResultSource); - RawTestResultsMap.TryAdd(testCaseResultData.TestExecutionId, rawResult); - - if (!string.IsNullOrEmpty(testResultSource.ErrorMessage)) - { - ProcessTestResultMessage(testResultSource.ErrorMessage); - // TODO send it in blob - } - if (!string.IsNullOrEmpty(testResultSource.ErrorStackTrace)) - { - ProcessTestResultMessage(testResultSource.ErrorStackTrace); - // TODO send it in blob - } - - // TODO ArtifactsPaths - return testCaseResultData; - } - - private void ProcessTestResultMessage(string? message) - { - if (string.IsNullOrEmpty(message)) - { - return; - } - foreach (TestResultError testResultErrorObj in TestResultErrorConstants.ErrorConstants) - { - if (processedErrorMessageKeys.Contains(testResultErrorObj.Key!)) - continue; - if (testResultErrorObj.Pattern.IsMatch(message)) - { - AddInformationalMessage(testResultErrorObj.Message!); - processedErrorMessageKeys.Add(testResultErrorObj.Key!); - } - } - } - - private TokenDetails ParseWorkspaceIdFromAccessToken(string accessToken) - { - TokenDetails tokenDetails = new(); - if (accessToken == null) - { - if (string.IsNullOrEmpty(accessToken)) - { - throw new ArgumentNullException("AccessToken is null or empty"); - } - } - try - { - JsonWebToken inputToken = (JsonWebToken)s_tokenHandler.ReadToken(accessToken); - var aid = inputToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value ?? string.Empty; - - if (!string.IsNullOrEmpty(aid)) // Custom Token - { - LogMessage("Custom Token parsing"); - tokenDetails.aid = aid; - tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; - tokenDetails.id = inputToken.Claims.FirstOrDefault(c => c.Type == "id")?.Value ?? string.Empty; - tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; - } - else // Entra Token - { - LogMessage("Entra Token parsing"); - tokenDetails.aid = Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_WORKSPACE_ID) ?? string.Empty; - tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; - tokenDetails.id = string.Empty; - tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; - // TODO add back suport for old claims https://devdiv.visualstudio.com/OnlineServices/_git/PlaywrightService?path=/src/Common/Authorization/JwtSecurityTokenValidator.cs&version=GBmain&line=200&lineEnd=200&lineStartColumn=30&lineEndColumn=52&lineStyle=plain&_a=contents - } - - return tokenDetails; - } - catch (Exception ex) - { - LogErrorMessage(ex.Message); - throw; - } - } - - private static Guid GetExecutionId(TestResult testResult) - { - TestProperty? executionIdProperty = testResult.Properties.FirstOrDefault( - property => property.Id.Equals(PlaywrightConstants.ExecutionIdPropertyIdentifier)); - - Guid executionId = Guid.Empty; - if (executionIdProperty != null) - executionId = testResult.GetPropertyValue(executionIdProperty, Guid.Empty); - - return executionId.Equals(Guid.Empty) ? Guid.NewGuid() : executionId; - } - - private static RawTestResult GetRawResultObject(TestResult testResultSource) - { - List errors = new();//[testResultSource.ErrorMessage]; - if (testResultSource.ErrorMessage != null) - errors.Add(new MPTError() { message = testResultSource.ErrorMessage }); - var rawTestResult = new RawTestResult - { - errors = JsonSerializer.Serialize(errors), - stdErr = testResultSource?.ErrorStackTrace ?? string.Empty - }; - return rawTestResult; - } - - private static string GetCloudFilePath(string uri, string fileRelativePath) - { - // Assuming Constants.SAS_URI_SEPARATOR is a static property or field in a class named Constants - // that holds the character used to split the URI and the SAS token. - string[] parts = uri.Split(new string[] { PlaywrightConstants.SASUriSeparator }, StringSplitOptions.None); - string containerUri = parts[0]; - string sasToken = parts.Length > 1 ? parts[1] : string.Empty; - - return $"{containerUri}/{fileRelativePath}?{sasToken}"; - } - - private void UploadBuffer(string uri, string buffer, string fileRelativePath) - { - string cloudFilePath = GetCloudFilePath(uri, fileRelativePath); - LogMessage(cloudFilePath); - LogMessage(buffer); - BlobClient blobClient = new(new Uri(cloudFilePath)); - byte[] bufferBytes = Encoding.UTF8.GetBytes(buffer); - blobClient.Upload(new BinaryData(bufferBytes), overwrite: true); - LogMessage($"Uploaded buffer to {fileRelativePath}"); - } - - private static string FetchTestClassName(string fullyQualifiedName) - { - string[] parts = fullyQualifiedName.Split('.'); - return string.Join(".", parts.Take(parts.Length - 1)); - } - - private static string FetchFileName(string fullFilePath) - { - char[] delimiters = { '\\', '/' }; - string[] parts = fullFilePath.Split(delimiters); - return parts.Last(); - } - - private static string GetCurrentOS() - { - PlatformID platform = Environment.OSVersion.Platform; - if (platform == PlatformID.Unix) - return "Linux"; - else if (platform == PlatformID.MacOSX) - return "MacOS"; - else - return "Windows"; - } - - private void LogMessage(string message) - { - bool enable = bool.TryParse(EnableConsoleLog, out enable) == true && enable; - Logger.Log(enable, _logLevel, message); - } - - private static void LogErrorMessage(string message) - { - Logger.Log(true, LogLevel.Error, message); - } - private void InitializePlaywrightReporter(string xmlSettings) { - if (IsInitialized) - { - return; - } - Dictionary runParameters = XmlRunSettingsUtilities.GetTestRunParameters(xmlSettings); runParameters.TryGetValue(RunSettingKey.RunId, out var runId); // If run id is not provided and not set via env, try fetching it from CI info. - CIInfo = CiInfoProvider.GetCIInfo(); - if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID))) + CIInfo cIInfo = CiInfoProvider.GetCIInfo(); + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId))) { if (string.IsNullOrEmpty(runId?.ToString())) - Environment.SetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID, ReporterUtils.GetRunId(CIInfo)); + Environment.SetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId, ReporterUtils.GetRunId(cIInfo)); else - Environment.SetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID, runId!.ToString()); + Environment.SetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId, runId!.ToString()); } else { @@ -666,8 +92,8 @@ private void InitializePlaywrightReporter(string xmlSettings) string? enableGithubSummaryString = enableGithubSummary?.ToString(); string? enableResultPublishString = enableResultPublish?.ToString(); - EnableGithubSummary = string.IsNullOrEmpty(enableGithubSummaryString) || bool.Parse(enableGithubSummaryString!); - EnableResultPublish = string.IsNullOrEmpty(enableResultPublishString) || bool.Parse(enableResultPublishString!); + bool _enableGitHubSummary = string.IsNullOrEmpty(enableGithubSummaryString) || bool.Parse(enableGithubSummaryString!); + bool _enableResultPublish = string.IsNullOrEmpty(enableResultPublishString) || bool.Parse(enableResultPublishString!); PlaywrightServiceOptions? playwrightServiceSettings = null; try @@ -676,7 +102,7 @@ private void InitializePlaywrightReporter(string xmlSettings) } catch (Exception ex) { - Console.Error.WriteLine("Failed to initialize PlaywrightServiceSettings: " + ex.Message); + Console.Error.WriteLine("Failed to initialize PlaywrightServiceSettings: " + ex); Environment.Exit(1); } @@ -686,75 +112,39 @@ private void InitializePlaywrightReporter(string xmlSettings) playwrightService.InitializeAsync().GetAwaiter().GetResult(); #pragma warning restore AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead. - RunId = Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID); - - try + var cloudRunId = Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId); + string baseUrl = Environment.GetEnvironmentVariable(ReporterConstants.s_pLAYWRIGHT_SERVICE_REPORTING_URL); + string accessToken = Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken); + if (string.IsNullOrEmpty(baseUrl)) { - ValidateArg.NotNullOrEmpty(BaseUrl, "Playwright Service URL"); - ValidateArg.NotNullOrEmpty(AccessToken, "Playwright Service Access Token"); + Console.Error.WriteLine(Constants.s_no_service_endpoint_error_message); + Environment.Exit(1); } - catch (Exception ex) + if (string.IsNullOrEmpty(accessToken)) { - Console.Error.WriteLine("Missing values : " + ex.Message); + Console.WriteLine(Constants.s_no_auth_error); Environment.Exit(1); } - TotalTestCount = 0; - PassedTestCount = 0; - FailedTestCount = 0; - SkippedTestCount = 0; - - TestRunStartTime = DateTime.UtcNow; - TokenDetails = ParseWorkspaceIdFromAccessToken(AccessToken); - WorkspaceId = TokenDetails.aid; - LogMessage("RunId: " + RunId); - LogMessage("BaseUrl: " + BaseUrl); - LogMessage("Workspace Id: " + WorkspaceId); - - PortalUrl = PlaywrightConstants.PortalBaseUrl + WorkspaceId + PlaywrightConstants.ReportingRoute + RunId; - - _httpClient = new HttpClient(); - var baseUri = new Uri(BaseUrl!); - _reportingTestRunsClient = new ReportingTestRunsClient(baseUri); - _reportingTestResultsClient = new ReportingTestResultsClient(baseUri); - - IsInitialized = true; - - LogMessage("Playwright Service Reporter Intialized"); - } + var baseUri = new Uri(baseUrl); + var reporterUtils = new ReporterUtils(); + TokenDetails tokenDetails = reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler: null, accessToken: accessToken); + var workspaceId = tokenDetails.aid; + var portalUri = ReporterConstants.s_portalBaseUrl + workspaceId + ReporterConstants.s_reportingRoute + cloudRunId; - internal void GenerateMarkdownSummary() - { - if (CiInfoProvider.GetCIProvider() == PlaywrightConstants.GITHUB_ACTIONS) + var cloudRunMetadata = new CloudRunMetadata { - string markdownContent = @$" -#### Results: - -![pass](https://img.shields.io/badge/status-passed-brightgreen) **Passed:** {TestRunShard!.ResultsSummary!.NumPassedTests} - -![fail](https://img.shields.io/badge/status-failed-red) **Failed:** {TestRunShard.ResultsSummary.NumFailedTests} - -![flaky](https://img.shields.io/badge/status-flaky-yellow) **Flaky:** {"0"} - -![skipped](https://img.shields.io/badge/status-skipped-lightgrey) **Skipped:** {TestRunShard.ResultsSummary.NumSkippedTests} - -#### For more details, visit the [service dashboard]({Uri.EscapeUriString(PortalUrl!)}). -"; - - string filePath = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY"); - try - { - File.WriteAllText(filePath, markdownContent); - } - catch (Exception ex) - { - LogErrorMessage($"Error writing Markdown summary: {ex}"); - } - } - } + RunId = cloudRunId, + WorkspaceId = workspaceId, + BaseUri = baseUri, + PortalUrl = portalUri, + EnableResultPublish = _enableResultPublish, + EnableGithubSummary = _enableGitHubSummary, + TestRunStartTime = DateTime.UtcNow, + AccessTokenDetails = tokenDetails, + }; - private void AddInformationalMessage(string message) - { - informationalMessages.Add(message); + _testProcessor = new TestProcessor(cloudRunMetadata, cIInfo); + _logger.Info("Playwright Service Reporter Initialized"); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs new file mode 100644 index 000000000000..79323a251f93 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System.Linq; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using System.Text.Json; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor +{ + internal class DataProcessor : IDataProcessor + { + private readonly ILogger _logger; + private readonly CIInfo _cIInfo; + private readonly CloudRunMetadata _cloudRunMetadata; + public DataProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? logger = null) + { + _cloudRunMetadata = cloudRunMetadata; + _cIInfo = cIInfo; + _logger = logger ?? new Logger(); + } + + public TestRunDtoV2 GetTestRun() + { + var startTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); + var gitBasedRunName = ReporterUtils.GetRunName(CiInfoProvider.GetCIInfo())?.Trim(); + string runName = string.IsNullOrEmpty(gitBasedRunName) ? _cloudRunMetadata.RunId! : gitBasedRunName!; + var run = new TestRunDtoV2 + { + TestRunId = _cloudRunMetadata.RunId!, + DisplayName = runName, + StartTime = startTime, + CreatorId = _cloudRunMetadata.AccessTokenDetails!.oid ?? "", + CreatorName = _cloudRunMetadata.AccessTokenDetails!.userName?.Trim() ?? "", + //CloudRunEnabled = "false", + CloudReportingEnabled = "true", + Summary = new TestRunSummary + { + Status = "RUNNING", + StartTime = startTime, + //Projects = ["playwright-dotnet"], + //Tags = ["Nunit", "dotnet"], + //Jobs = ["playwright-dotnet"], + }, + CiConfig = new CIConfig // TODO fetch dynamically + { + Branch = _cIInfo.Branch ?? "", + Author = _cIInfo.Author ?? "", + CommitId = _cIInfo.CommitId ?? "", + RevisionUrl = _cIInfo.RevisionUrl ?? "" + }, + TestRunConfig = new ClientConfig // TODO fetch some of these dynamically + { + Workers = 1, + PwVersion = "1.40", + Timeout = 60000, + TestType = "WebTest", + TestSdkLanguage = "Dotnet", + TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit/MSTest", Version = "3.1" }, // TODO fetch runner name MSTest/Nunit + ReporterPackageVersion = "0.0.1-dotnet", + Shards = new Shard() { Current = 0, Total = 1 } + } + }; + return run; + } + + public TestRunShardDto GetTestRunShard() + { + var startTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); + var shard = new TestRunShardDto + { + UploadCompleted = "false", + Summary = new TestRunShardSummary + { + Status = "RUNNING", + StartTime = startTime, + }, + TestRunConfig = new ClientConfig // TODO fetch some of these dynamically + { + Workers = 1, + PwVersion = "1.40", + Timeout = 60000, + TestType = "Functional", + TestSdkLanguage = "dotnet", + TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit", Version = "3.1" }, + ReporterPackageVersion = "0.0.1-dotnet", + Shards = new Shard() { Current = 0, Total = 1 }, + } + }; + return shard; + } + public TestResults GetTestCaseResultData(TestResult testResultSource) + { + if (testResultSource == null) + return new TestResults(); + + TestResults testCaseResultData = new() + { + ArtifactsPath = new List(), + AccountId = _cloudRunMetadata.WorkspaceId!, + RunId = _cloudRunMetadata.RunId!, + TestExecutionId = GetExecutionId(testResultSource).ToString() + }; + testCaseResultData.TestCombinationId = testCaseResultData.TestExecutionId; // TODO check + testCaseResultData.TestId = testResultSource.TestCase.Id.ToString(); + testCaseResultData.TestTitle = testResultSource.TestCase.DisplayName; + var className = FetchTestClassName(testResultSource.TestCase.FullyQualifiedName); + testCaseResultData.SuiteTitle = className; + testCaseResultData.SuiteId = className; + testCaseResultData.FileName = FetchFileName(testResultSource.TestCase.Source); + testCaseResultData.LineNumber = testResultSource.TestCase.LineNumber; + testCaseResultData.Retry = 0; // TODO Retry and PreviousRetries + testCaseResultData.WebTestConfig = new WebTestConfig + { + JobName = _cIInfo.JobId ?? "", + //ProjectName = "playwright-dotnet", // TODO no project concept NA?? + //BrowserName = "chromium", // TODO check if possible to get from test + Os = ReporterUtils.GetCurrentOS(), + }; + //testCaseResultData.Annotations = ["windows"]; // TODO MSTest/Nunit annotation ?? + //testCaseResultData.Tags = ["windows"]; // TODO NA ?? + + TimeSpan duration = testResultSource.Duration; + testCaseResultData.ResultsSummary = new TestResultsSummary + { + Duration = (long)duration.TotalMilliseconds, // TODO fallback get from End-Start + StartTime = testResultSource.StartTime.UtcDateTime.ToString(), + Status = TestCaseResultStatus.s_iNCONCLUSIVE + }; + TestOutcome outcome = testResultSource.Outcome; + switch (outcome) + { + case TestOutcome.Passed: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_pASSED; + testCaseResultData.Status = TestCaseResultStatus.s_pASSED; + break; + case TestOutcome.Failed: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_fAILED; + testCaseResultData.Status = TestCaseResultStatus.s_fAILED; + break; + case TestOutcome.Skipped: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_sKIPPED; + testCaseResultData.Status = TestCaseResultStatus.s_sKIPPED; + break; + default: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_iNCONCLUSIVE; + testCaseResultData.Status = TestCaseResultStatus.s_iNCONCLUSIVE; + break; + } + return testCaseResultData; + } + + public static RawTestResult GetRawResultObject(TestResult testResultSource) + { + List errors = new();//[testResultSource.ErrorMessage]; + if (testResultSource.ErrorMessage != null) + errors.Add(new MPTError() { message = testResultSource.ErrorMessage }); + var rawTestResult = new RawTestResult + { + errors = JsonSerializer.Serialize(errors), + stdErr = testResultSource?.ErrorStackTrace ?? string.Empty + }; + return rawTestResult; + } + + #region Data Processor Utility Methods + + private static Guid GetExecutionId(TestResult testResult) + { + TestProperty? executionIdProperty = testResult.Properties.FirstOrDefault( + property => property.Id.Equals(ReporterConstants.s_executionIdPropertyIdentifier)); + + Guid executionId = Guid.Empty; + if (executionIdProperty != null) + executionId = testResult.GetPropertyValue(executionIdProperty, Guid.Empty); + + return executionId.Equals(Guid.Empty) ? Guid.NewGuid() : executionId; + } + + private static string FetchTestClassName(string fullyQualifiedName) + { + string[] parts = fullyQualifiedName.Split('.'); + return string.Join(".", parts.Take(parts.Length - 1)); + } + + private static string FetchFileName(string fullFilePath) + { + char[] delimiters = { '\\', '/' }; + string[] parts = fullFilePath.Split(delimiters); + return parts.Last(); + } + #endregion + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs new file mode 100644 index 000000000000..831c793ae89c --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Json; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Storage.Blobs; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor +{ + internal class TestProcessor : ITestProcessor + { + // Dependency Injection + private readonly IDataProcessor _dataProcessor; + private readonly ILogger _logger; + private readonly ICloudRunErrorParser _cloudRunErrorParser; + private readonly IServiceClient _serviceClient; + private readonly IConsoleWriter _consoleWriter; + private readonly CIInfo _cIInfo; + private readonly CloudRunMetadata _cloudRunMetadata; + + // Test Metadata + private int TotalTestCount { get; set; } = 0; + private int PassedTestCount { get; set; } = 0; + private int FailedTestCount { get; set; } = 0; + private int SkippedTestCount { get; set; } = 0; + private List TestResults { get; set; } = new List(); + private ConcurrentDictionary RawTestResultsMap { get; set; } = new(); + private bool FatalTestExecution { get; set; } = false; + private TestRunShardDto? _testRunShard; + + public TestProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? logger = null, IDataProcessor? dataProcessor = null, ICloudRunErrorParser? cloudRunErrorParser = null, IServiceClient? serviceClient = null, IConsoleWriter? consoleWriter = null) + { + _cloudRunMetadata = cloudRunMetadata; + _cIInfo = cIInfo; + _logger = logger ?? new Logger(); + _dataProcessor = dataProcessor ?? new DataProcessor(_cloudRunMetadata, _cIInfo, _logger); + _cloudRunErrorParser = cloudRunErrorParser ?? new CloudRunErrorParser(_logger); + _serviceClient = serviceClient ?? new ServiceClient(_cloudRunMetadata); + _consoleWriter = consoleWriter ?? new ConsoleWriter(); + } + + public void TestRunStartHandler(object? sender, TestRunStartEventArgs e) + { + try + { + _logger.Info("Initialising test run"); + if (!_cloudRunMetadata.EnableResultPublish || FatalTestExecution) + { + return; + } + TestRunDtoV2 run = _dataProcessor.GetTestRun(); + TestRunShardDto shard = _dataProcessor.GetTestRunShard(); + TestRunDtoV2? testRun = _serviceClient.PatchTestRunInfo(run); + if (testRun == null) + { + _logger.Error("Failed to patch test run info"); + FatalTestExecution = true; + return; + } + _logger.Info("Successfully patched test run - init"); + TestRunShardDto? testShard = _serviceClient.PatchTestRunShardInfo(1, shard); + if (testShard == null) + { + _logger.Error("Failed to patch test run shard info"); + FatalTestExecution = true; + return; + } + _testRunShard = testShard; + _logger.Info("Successfully patched test run shard - init"); + _consoleWriter.WriteLine($"\nInitializing reporting for this test run. You can view the results at: {Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}"); + } + catch (Exception ex) + { + _logger.Error($"Failed to initialise test run: {ex}"); + FatalTestExecution = true; + } + } + public void TestCaseResultHandler(object? sender, TestResultEventArgs e) + { + try + { + if (FatalTestExecution) + { + return; + } + TestResult testResultSource = e.Result; + TestResults? testResult = _dataProcessor.GetTestCaseResultData(testResultSource); + RawTestResult rawResult = DataProcessor.GetRawResultObject(testResultSource); + RawTestResultsMap.TryAdd(testResult.TestExecutionId, rawResult); + + // TODO - Send error to blob + _cloudRunErrorParser.HandleScalableRunErrorMessage(testResultSource.ErrorMessage); + _cloudRunErrorParser.HandleScalableRunErrorMessage(testResultSource.ErrorStackTrace); + if (!_cloudRunMetadata.EnableResultPublish) + { + return; + } + if (testResult != null) + { + TotalTestCount++; + if (testResult.Status == TestCaseResultStatus.s_fAILED) + { + FailedTestCount++; + } + else if (testResult.Status == TestCaseResultStatus.s_pASSED) + { + PassedTestCount++; + } + else if (testResult.Status == TestCaseResultStatus.s_sKIPPED) + { + SkippedTestCount++; + } + } + if (testResult != null) + { + TestResults.Add(testResult); + } + } + catch (Exception ex) + { + // test case processing failures should not stop the test run + _logger.Error($"Failed to process test case result: {ex}"); + } + } + public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) + { + _logger.Info("Test run complete handler - start"); + if (_cloudRunMetadata.EnableResultPublish && !FatalTestExecution) + { + try + { + var body = new UploadTestResultsRequest() { Value = TestResults }; + _serviceClient.UploadBatchTestResults(body); + _logger.Info("Successfully uploaded test results"); + } + catch (Exception ex) + { + _logger.Error($"Failed to upload test results: {ex}"); + } + try + { + TestResultsUri? sasUri = _serviceClient.GetTestRunResultsUri(); + if (!string.IsNullOrEmpty(sasUri?.Uri)) + { + foreach (TestResults testResult in TestResults) + { + if (RawTestResultsMap.TryGetValue(testResult.TestExecutionId!, out RawTestResult? rawResult) && rawResult != null) + { + // Upload rawResult to blob storage using sasUri + var rawTestResultJson = JsonSerializer.Serialize(rawResult); + var filePath = $"{testResult.TestExecutionId}/rawTestResult.json"; + UploadBuffer(sasUri!.Uri!, rawTestResultJson, filePath); + } + else + { + _logger.Info("Couldnt find rawResult for Id: " + testResult.TestExecutionId); + } + } + _logger.Info("Successfully uploaded raw test results"); + } + else + { + _logger.Error("Sas Uri is empty"); + } + } + catch (Exception ex) + { + _logger.Error($"Failed to upload artifacts: {ex}"); + } + } + EndTestRun(e); + } + + #region Test Processor Helper Methods + private void EndTestRun(TestRunCompleteEventArgs e) + { + if (_cloudRunMetadata.EnableResultPublish && !FatalTestExecution) + { + try + { + _testRunShard = GetTestRunEndShard(e); + _serviceClient.PatchTestRunShardInfo(1, _testRunShard); + _logger.Info("Successfully ended test run shard"); + } + catch (Exception ex) + { + _logger.Error($"Failed to end test run shard: {ex}"); + } + _consoleWriter.WriteLine($"\nTest Report: {Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}"); + if (_cloudRunMetadata.EnableGithubSummary) + { + GenerateMarkdownSummary(); + } + } + _cloudRunErrorParser.DisplayMessages(); + } + private static string GetCloudFilePath(string uri, string fileRelativePath) + { + string[] parts = uri.Split(new string[] { ReporterConstants.s_sASUriSeparator }, StringSplitOptions.None); + string containerUri = parts[0]; + string sasToken = parts.Length > 1 ? parts[1] : string.Empty; + + return $"{containerUri}/{fileRelativePath}?{sasToken}"; + } + private void UploadBuffer(string uri, string buffer, string fileRelativePath) + { + string cloudFilePath = GetCloudFilePath(uri, fileRelativePath); + BlobClient blobClient = new(new Uri(cloudFilePath)); + byte[] bufferBytes = Encoding.UTF8.GetBytes(buffer); + blobClient.Upload(new BinaryData(bufferBytes), overwrite: true); + _logger.Info($"Uploaded buffer to {fileRelativePath}"); + } + private TestRunShardDto GetTestRunEndShard(TestRunCompleteEventArgs e) + { + DateTime testRunEndedOn = DateTime.UtcNow; + long durationInMs = 0; + + var result = FailedTestCount > 0 ? TestCaseResultStatus.s_fAILED : TestCaseResultStatus.s_pASSED; + + if (e.ElapsedTimeInRunningTests != null) + { + testRunEndedOn = _cloudRunMetadata.TestRunStartTime.Add(e.ElapsedTimeInRunningTests); + durationInMs = (long)e.ElapsedTimeInRunningTests.TotalMilliseconds; + } + TestRunShardDto? testRunShard = _testRunShard; + // Update Shard End + if (testRunShard!.Summary == null) + testRunShard.Summary = new TestRunShardSummary(); + testRunShard.Summary.Status = "CLIENT_COMPLETE"; + testRunShard.Summary.StartTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); + testRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ"); + testRunShard.Summary.TotalTime = durationInMs; + testRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 }; + testRunShard.ResultsSummary = new TestRunResultsSummary + { + NumTotalTests = TotalTestCount, + NumPassedTests = PassedTestCount, + NumFailedTests = FailedTestCount, + NumSkippedTests = SkippedTestCount, + NumFlakyTests = 0, // TODO: Implement flaky tests + Status = result + }; + testRunShard.UploadCompleted = "true"; + return testRunShard; + } + private void GenerateMarkdownSummary() + { + if (_cIInfo.Provider == CIConstants.s_gITHUB_ACTIONS) + { + string markdownContent = @$" +#### Results: + +![pass](https://img.shields.io/badge/status-passed-brightgreen) **Passed:** {_testRunShard!.ResultsSummary!.NumPassedTests} + +![fail](https://img.shields.io/badge/status-failed-red) **Failed:** {_testRunShard.ResultsSummary.NumFailedTests} + +![flaky](https://img.shields.io/badge/status-flaky-yellow) **Flaky:** {"0"} + +![skipped](https://img.shields.io/badge/status-skipped-lightgrey) **Skipped:** {_testRunShard.ResultsSummary.NumSkippedTests} + +#### For more details, visit the [service dashboard]({Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}). +"; + + string filePath = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY"); + try + { + File.WriteAllText(filePath, markdownContent); + } + catch (Exception ex) + { + _logger.Error($"Error writing Markdown summary: {ex}"); + } + } + } + #endregion + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs index 97e332b62505..fa229851e41d 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs @@ -3,7 +3,6 @@ using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; using System; -using PlaywrightConstants = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility.Constants; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; @@ -23,22 +22,22 @@ private static bool IsAzureDevOps() internal static string GetCIProvider() { if (IsGitHubActions()) - return PlaywrightConstants.GITHUB_ACTIONS; + return CIConstants.s_gITHUB_ACTIONS; else if (IsAzureDevOps()) - return PlaywrightConstants.AZURE_DEVOPS; + return CIConstants.s_aZURE_DEVOPS; else - return PlaywrightConstants.DEFAULT; + return CIConstants.s_dEFAULT; } internal static CIInfo GetCIInfo() { string ciProvider = GetCIProvider(); - if (ciProvider == PlaywrightConstants.GITHUB_ACTIONS) + if (ciProvider == CIConstants.s_gITHUB_ACTIONS) { // Logic to get GitHub Actions CIInfo return new CIInfo { - Provider = PlaywrightConstants.GITHUB_ACTIONS, + Provider = CIConstants.s_gITHUB_ACTIONS, Repo = Environment.GetEnvironmentVariable("GITHUB_REPOSITORY_ID"), Branch = GetGHBranchName(), Author = Environment.GetEnvironmentVariable("GITHUB_ACTOR"), @@ -53,12 +52,12 @@ internal static CIInfo GetCIInfo() JobId = Environment.GetEnvironmentVariable("GITHUB_JOB") }; } - else if (ciProvider == PlaywrightConstants.AZURE_DEVOPS) + else if (ciProvider == CIConstants.s_aZURE_DEVOPS) { // Logic to get Azure DevOps CIInfo return new CIInfo { - Provider = PlaywrightConstants.AZURE_DEVOPS, + Provider = CIConstants.s_aZURE_DEVOPS, Repo = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_ID"), Branch = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH"), Author = Environment.GetEnvironmentVariable("BUILD_REQUESTEDFOR"), @@ -78,7 +77,7 @@ internal static CIInfo GetCIInfo() // Handle unsupported CI provider return new CIInfo { - Provider = PlaywrightConstants.DEFAULT, + Provider = CIConstants.s_dEFAULT, Repo = Environment.GetEnvironmentVariable("REPO"), Branch = Environment.GetEnvironmentVariable("BRANCH"), Author = Environment.GetEnvironmentVariable("AUTHOR"), diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs deleted file mode 100644 index e04af7928a21..000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System.Collections.Generic; -using System.Text.RegularExpressions; - -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; - -internal static class Constants -{ - /// - /// Property Id storing the ExecutionId. - /// - internal const string ExecutionIdPropertyIdentifier = "ExecutionId"; - - /// - /// Property Id storing the ParentExecutionId. - /// - internal const string ParentExecutionIdPropertyIdentifier = "ParentExecId"; - - /// - /// Property If storing the TestType. - /// - internal const string TestTypePropertyIdentifier = "TestType"; - - internal const string SASUriSeparator = "?"; - - internal const string PortalBaseUrl = "https://playwright.microsoft.com/workspaces/"; - - internal const string ReportingRoute = "/runs/"; - - internal const string ReportingAPIVersion_2024_04_30_preview = "2024-04-30-preview"; - - internal const string ReportingAPIVersion_2024_05_20_preview = "2024-05-20-preview"; - - internal const string PLAYWRIGHT_SERVICE_REPORTING_URL = "PLAYWRIGHT_SERVICE_REPORTING_URL"; - - internal const string PLAYWRIGHT_SERVICE_WORKSPACE_ID = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; - - internal const string PLAYWRIGHT_SERVICE_ACCESS_TOKEN = "PLAYWRIGHT_SERVICE_ACCESS_TOKEN"; - - internal const string PLAYWRIGHT_SERVICE_DEBUG = "PLAYWRIGHT_SERVICE_DEBUG"; - - internal const string PLAYWRIGHT_SERVICE_RUN_ID = "PLAYWRIGHT_SERVICE_RUN_ID"; - - internal const string GITHUB_ACTIONS = "GitHub Actions"; - internal const string AZURE_DEVOPS = "Azure DevOps"; - internal const string DEFAULT = "Default"; -} - -internal enum TestErrorType -{ - Scalable -} - -internal class TestResultError -{ - internal string? Key { get; set; } = string.Empty; - internal string? Message { get; set; } = string.Empty; - internal Regex Pattern { get; set; } = new Regex(string.Empty); - internal TestErrorType Type { get; set; } -} - -internal static class TestResultErrorConstants -{ - public static List ErrorConstants = new() - { - new TestResultError - { - Key = "Unauthorized_Scalable", - Message = "The authentication token provided is invalid. Please check the token and try again.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*401 Unauthorized)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "NoPermissionOnWorkspace_Scalable", - Message = @"You do not have the required permissions to run tests. This could be because: - - a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator. - b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant '.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*CheckAccess API call with non successful response)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "InvalidWorkspace_Scalable", - Message = "The specified workspace does not exist. Please verify your workspace settings.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "AccessKeyBasedAuthNotSupported_Scalable", - Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "ServiceUnavailable_Scalable", - Message = "The service is currently unavailable. Please check the service status and try again.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*503 Service Unavailable)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "GatewayTimeout_Scalable", - Message = "The request to the service timed out. Please try again later.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*504 Gateway Timeout)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "QuotaLimitError_Scalable", - Message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded.", - Pattern = new Regex(@"(Timeout .* exceeded)(?=[\s\S]*ws connecting)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "BrowserConnectionError_Scalable", - Message = "The service is currently unavailable. Please try again after some time.", - Pattern = new Regex(@"Target page, context or browser has been closed", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - } - }; -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs deleted file mode 100644 index e01a38ca0396..000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; - -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; - -internal enum LogLevel -{ - Debug, - Info, - Warning, - Error -} - -internal static class Logger -{ - public static void Log(bool enableConsoleLog, LogLevel level, string message) - { - if (enableConsoleLog) - { - Console.WriteLine($"{DateTime.Now} [{level}]: {message}"); - } - } -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs index c25c8d466eb9..f880c2ee7c2c 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs @@ -3,18 +3,27 @@ using System; using System.Diagnostics; +using System.Linq; using System.Security.Cryptography; using System.Threading.Tasks; using Azure.Core.Pipeline; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Microsoft.IdentityModel.JsonWebTokens; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility { internal class ReporterUtils { + private readonly ILogger _logger; + public ReporterUtils(ILogger? logger = null) + { + _logger = logger ?? new Logger(); + } internal static string GetRunId(CIInfo cIInfo) { - if (cIInfo.Provider == Constants.DEFAULT) + if (cIInfo.Provider == CIConstants.s_dEFAULT) { return Guid.NewGuid().ToString(); } @@ -37,7 +46,7 @@ internal static string GetRunName(CIInfo ciInfo) string GIT_REV_PARSE = "git rev-parse --is-inside-work-tree"; string GIT_COMMIT_MESSAGE_COMMAND = "git log -1 --pretty=%B"; - if (ciInfo.Provider == Constants.GITHUB_ACTIONS && + if (ciInfo.Provider == CIConstants.s_gITHUB_ACTIONS && Environment.GetEnvironmentVariable("GITHUB_EVENT_NAME") == "pull_request") { var prNumber = Environment.GetEnvironmentVariable("GITHUB_REF_NAME")?.Split('/')[0]; @@ -94,5 +103,58 @@ internal static async Task RunCommandAsync(string command, bool async = return result; } } + + internal static string GetCurrentOS() + { + PlatformID platform = Environment.OSVersion.Platform; + if (platform == PlatformID.Unix) + return OSConstants.s_lINUX; + else if (platform == PlatformID.MacOSX) + return OSConstants.s_mACOS; + else + return OSConstants.s_wINDOWS; + } + + public TokenDetails ParseWorkspaceIdFromAccessToken(JsonWebTokenHandler? jsonWebTokenHandler, string? accessToken) + { + if (jsonWebTokenHandler == null) + { + jsonWebTokenHandler = new JsonWebTokenHandler(); + } + TokenDetails tokenDetails = new(); + if (string.IsNullOrEmpty(accessToken)) + { + throw new ArgumentNullException(nameof(accessToken), "AccessToken is null or empty"); + } + try + { + JsonWebToken inputToken = (JsonWebToken)jsonWebTokenHandler.ReadToken(accessToken); + var aid = inputToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value ?? string.Empty; + + if (!string.IsNullOrEmpty(aid)) // Custom Token + { + _logger.Info("Custom Token parsing"); + tokenDetails.aid = aid; + tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; + tokenDetails.id = inputToken.Claims.FirstOrDefault(c => c.Type == "id")?.Value ?? string.Empty; + tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; + } + else // Entra Token + { + _logger.Info("Entra Token parsing"); + tokenDetails.aid = Environment.GetEnvironmentVariable(ReporterConstants.s_pLAYWRIGHT_SERVICE_WORKSPACE_ID) ?? string.Empty; + tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; + tokenDetails.id = string.Empty; + tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; + // TODO add back suport for old claims https://devdiv.visualstudio.com/OnlineServices/_git/PlaywrightService?path=/src/Common/Authorization/JwtSecurityTokenValidator.cs&version=GBmain&line=200&lineEnd=200&lineStartColumn=30&lineEndColumn=52&lineStyle=plain&_a=contents + } + + return tokenDetails; + } + catch (Exception) + { + throw; + } + } } } From 1d5a12fb7cda0b2252b10b30fd691fed68aa77e7 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 01:43:51 +0530 Subject: [PATCH 02/18] chore(): additional scalable run failure handling --- .../src/Constants.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 5753ed87f524..8d39b2dcba4e 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -266,7 +266,7 @@ internal static class TestResultErrorConstants { new TestResultError { - Key = "Unauthorized_Scalable", + Key = "401", Message = "The authentication token provided is invalid. Please check the token and try again.", Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*401 Unauthorized)", RegexOptions.IgnoreCase), Type = TestErrorType.Scalable @@ -289,6 +289,20 @@ internal static class TestResultErrorConstants Type = TestErrorType.Scalable }, new TestResultError + { + Key = "InvalidAccessToken", + Message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccessToken)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "AccessTokenOrUserOrWorkspaceNotFound_Scalable", + Message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*404 Not Found)(?=.*NotFound)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError { Key = "AccessKeyBasedAuthNotSupported_Scalable", Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.", From f64347a1005d645eea45fce5ac04e2b95dddb86e Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 01:45:58 +0530 Subject: [PATCH 03/18] chore(): api retries and logging --- .../src/Constants.cs | 6 ++++++ .../src/Implementation/ServiceClient.cs | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 8d39b2dcba4e..02f385da4627 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -260,6 +260,12 @@ internal enum TestErrorType Scalable } +internal class ServiceClientConstants +{ + internal static readonly int s_mAX_RETRIES = 3; + internal static readonly int s_mAX_RETRY_DELAY_IN_SECONDS = 2000; +} + internal static class TestResultErrorConstants { public static List ErrorConstants = new() diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs index 1a459b574f8c..4d47a6b8f242 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -7,6 +7,8 @@ using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; using System.Text.Json; +using Azure.Core.Diagnostics; +using System.Diagnostics.Tracing; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation { @@ -15,14 +17,25 @@ internal class ServiceClient : IServiceClient private readonly ReportingTestResultsClient _reportingTestResultsClient; private readonly ReportingTestRunsClient _reportingTestRunsClient; private readonly CloudRunMetadata _cloudRunMetadata; + private readonly ILogger _logger; private static string AccessToken { get => $"Bearer {Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken)}"; set { } } private static string CorrelationId { get => Guid.NewGuid().ToString(); set { } } - public ServiceClient(CloudRunMetadata cloudRunMetadata, ReportingTestResultsClient? reportingTestResultsClient = null, ReportingTestRunsClient? reportingTestRunsClient = null) + public ServiceClient(CloudRunMetadata cloudRunMetadata, ReportingTestResultsClient? reportingTestResultsClient = null, ReportingTestRunsClient? reportingTestRunsClient = null, ILogger? logger = null) { _cloudRunMetadata = cloudRunMetadata; - _reportingTestResultsClient = reportingTestResultsClient ?? new ReportingTestResultsClient(_cloudRunMetadata.BaseUri); - _reportingTestRunsClient = reportingTestRunsClient ?? new ReportingTestRunsClient(_cloudRunMetadata.BaseUri); + _logger = logger ?? new Logger(); + AzureEventSourceListener listener = new(delegate (EventWrittenEventArgs eventData, string text) + { + _logger.Info($"[{eventData.Level}] {eventData.EventSource.Name}: {text}"); + }, EventLevel.Informational); + var clientOptions = new TestReportingClientOptions(); + clientOptions.Diagnostics.IsLoggingEnabled = true; + clientOptions.Diagnostics.IsTelemetryEnabled = true; + clientOptions.Retry.MaxRetries = ServiceClientConstants.s_mAX_RETRIES; + clientOptions.Retry.MaxDelay = TimeSpan.FromSeconds(ServiceClientConstants.s_mAX_RETRY_DELAY_IN_SECONDS); + _reportingTestResultsClient = reportingTestResultsClient ?? new ReportingTestResultsClient(_cloudRunMetadata.BaseUri, clientOptions); + _reportingTestRunsClient = reportingTestRunsClient ?? new ReportingTestRunsClient(_cloudRunMetadata.BaseUri, clientOptions); } public TestRunDtoV2? PatchTestRunInfo(TestRunDtoV2 run) From ea1a3f1a1b85261e108e1894d96d97e4fb4988be Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 02:55:19 +0530 Subject: [PATCH 04/18] chore(): reporting api failure error handling --- .../src/Constants.cs | 57 +++++++++++ .../src/Implementation/CloudRunErrorParser.cs | 5 + .../src/Implementation/ConsoleWriter.cs | 12 +++ .../src/Implementation/ServiceClient.cs | 97 ++++++++++++++++--- .../src/Interface/ICloudRunErrorParser.cs | 1 + .../src/Interface/IConsoleWriter.cs | 1 + .../src/Processor/TestProcessor.cs | 6 +- 7 files changed, 163 insertions(+), 16 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 02f385da4627..76beb50f6a0a 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -230,6 +230,12 @@ internal class ReporterConstants internal static readonly string s_pLAYWRIGHT_SERVICE_REPORTING_URL = "PLAYWRIGHT_SERVICE_REPORTING_URL"; internal static readonly string s_pLAYWRIGHT_SERVICE_WORKSPACE_ID = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; internal static readonly string s_aPPLICATION_JSON = "application/json"; + internal static readonly string s_cONFLICT_409_ERROR_MESSAGE = "Test run with id {runId} already exists. Provide a unique run id."; + internal static readonly string s_cONFLICT_409_ERROR_MESSAGE_KEY = "DuplicateRunId"; + + internal static readonly string s_fORBIDDEN_403_ERROR_MESSAGE = "Reporting is not enabled for your workspace {workspaceId}. Enable the Reporting feature under Feature management settings using the Playwright portal: https://playwright.microsoft.com/workspaces/{workspaceId}/settings/general"; + internal static readonly string s_fORBIDDEN_403_ERROR_MESSAGE_KEY = "ReportingNotEnabled"; + internal static readonly string s_uNKNOWN_ERROR_MESSAGE = "Unknown error occured."; } internal class CIConstants @@ -345,3 +351,54 @@ internal static class TestResultErrorConstants } }; } + +internal static class ApiErrorConstants +{ + private static Dictionary PatchTestRun { get; set; } = new Dictionary() { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + + private static Dictionary GetTestRun { get; set; } = new Dictionary() + { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + private static Dictionary PatchTestRunShard { get; set; } = new Dictionary() + { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + private static Dictionary GetStorageUri { get; set; } = new Dictionary() + { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + + internal static readonly Dictionary> s_errorOperationPair = new() + { + { "PatchTestRun", PatchTestRun }, + { "GetTestRun", GetTestRun }, + { "PatchTestRunShard", PatchTestRunShard }, + { "GetStorageUri", GetStorageUri } + }; +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs index bc0dae518d9f..12ad11bef2b9 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs @@ -51,6 +51,11 @@ public void DisplayMessages() } } + public void PrintErrorToConsole(string message) + { + _consoleWriter.WriteError(message); + } + public void HandleScalableRunErrorMessage(string? message) { if (string.IsNullOrEmpty(message)) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs index 5948545c6593..bfbfe7766cb0 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs @@ -19,5 +19,17 @@ public void WriteLine(string? message = null) Console.WriteLine(message); } } + + public void WriteError(string? message = null) + { + if (message == null) + { + Console.Error.WriteLine(); + } + else + { + Console.Error.WriteLine(message); + } + } } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs index 4d47a6b8f242..87c247dce894 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -9,6 +9,7 @@ using System.Text.Json; using Azure.Core.Diagnostics; using System.Diagnostics.Tracing; +using System.Net; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation { @@ -17,13 +18,15 @@ internal class ServiceClient : IServiceClient private readonly ReportingTestResultsClient _reportingTestResultsClient; private readonly ReportingTestRunsClient _reportingTestRunsClient; private readonly CloudRunMetadata _cloudRunMetadata; + private readonly ICloudRunErrorParser _cloudRunErrorParser; private readonly ILogger _logger; private static string AccessToken { get => $"Bearer {Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken)}"; set { } } private static string CorrelationId { get => Guid.NewGuid().ToString(); set { } } - public ServiceClient(CloudRunMetadata cloudRunMetadata, ReportingTestResultsClient? reportingTestResultsClient = null, ReportingTestRunsClient? reportingTestRunsClient = null, ILogger? logger = null) + public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser cloudRunErrorParser, ReportingTestResultsClient? reportingTestResultsClient = null, ReportingTestRunsClient? reportingTestRunsClient = null, ILogger? logger = null) { _cloudRunMetadata = cloudRunMetadata; + _cloudRunErrorParser = cloudRunErrorParser; _logger = logger ?? new Logger(); AzureEventSourceListener listener = new(delegate (EventWrittenEventArgs eventData, string text) { @@ -31,6 +34,7 @@ public ServiceClient(CloudRunMetadata cloudRunMetadata, ReportingTestResultsClie }, EventLevel.Informational); var clientOptions = new TestReportingClientOptions(); clientOptions.Diagnostics.IsLoggingEnabled = true; + clientOptions.Diagnostics.IsLoggingContentEnabled = false; clientOptions.Diagnostics.IsTelemetryEnabled = true; clientOptions.Retry.MaxRetries = ServiceClientConstants.s_mAX_RETRIES; clientOptions.Retry.MaxDelay = TimeSpan.FromSeconds(ServiceClientConstants.s_mAX_RETRY_DELAY_IN_SECONDS); @@ -40,37 +44,108 @@ public ServiceClient(CloudRunMetadata cloudRunMetadata, ReportingTestResultsClie public TestRunDtoV2? PatchTestRunInfo(TestRunDtoV2 run) { - Response apiResponse = _reportingTestRunsClient.PatchTestRunInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(run), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); - if (apiResponse.Content != null) + int statusCode; + try { - return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + Response? apiResponse = _reportingTestRunsClient.PatchTestRunInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(run), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + if (apiResponse.Status == (int)HttpStatusCode.OK) + { + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + } + statusCode = apiResponse.Status; } + catch (RequestFailedException ex) + { + if (ex.Status == (int)HttpStatusCode.Conflict) + { + var errorMessage = ReporterConstants.s_cONFLICT_409_ERROR_MESSAGE.Replace("{runId}", _cloudRunMetadata.RunId!); + _cloudRunErrorParser.PrintErrorToConsole(errorMessage); + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, ReporterConstants.s_cONFLICT_409_ERROR_MESSAGE_KEY); + throw new Exception(errorMessage); + } + else if (ex.Status == (int)HttpStatusCode.Forbidden) + { + var errorMessage = ReporterConstants.s_fORBIDDEN_403_ERROR_MESSAGE.Replace("{workspaceId}", _cloudRunMetadata.WorkspaceId!); + _cloudRunErrorParser.PrintErrorToConsole(errorMessage); + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, ReporterConstants.s_fORBIDDEN_403_ERROR_MESSAGE_KEY); + throw new Exception(errorMessage); + } + statusCode = ex.Status; + } + HandleAPIFailure(statusCode, "PatchTestRun"); return null; } public TestRunShardDto? PatchTestRunShardInfo(int shardId, TestRunShardDto runShard) { - Response apiResponse = _reportingTestRunsClient.PatchTestRunShardInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, shardId.ToString(), RequestContent.Create(runShard), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); - if (apiResponse.Content != null) + int statusCode; + try + { + Response apiResponse = _reportingTestRunsClient.PatchTestRunShardInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, shardId.ToString(), RequestContent.Create(runShard), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + if (apiResponse.Status == (int)HttpStatusCode.OK) + { + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + } + statusCode = apiResponse.Status; + } + catch (RequestFailedException ex) { - return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + statusCode = ex.Status; } + HandleAPIFailure(statusCode, "PatchTestRunShard"); return null; } public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsRequest) { - _reportingTestResultsClient.UploadBatchTestResults(_cloudRunMetadata.WorkspaceId!, RequestContent.Create(JsonSerializer.Serialize(uploadTestResultsRequest)), AccessToken, CorrelationId, null); + int statusCode; + try + { + Response apiResponse = _reportingTestResultsClient.UploadBatchTestResults(_cloudRunMetadata.WorkspaceId!, RequestContent.Create(JsonSerializer.Serialize(uploadTestResultsRequest)), AccessToken, CorrelationId, null); + if (apiResponse.Status == (int)HttpStatusCode.OK) + { + return; + } + statusCode = apiResponse.Status; + } + catch (RequestFailedException ex) + { + statusCode = ex.Status; + } + HandleAPIFailure(statusCode, "PostTestResults"); } public TestResultsUri? GetTestRunResultsUri() { - Response response = _reportingTestRunsClient.GetTestRunResultsUri(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, AccessToken, CorrelationId, null); - if (response.Content != null) + int statusCode; + try + { + Response response = _reportingTestRunsClient.GetTestRunResultsUri(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, AccessToken, CorrelationId, null); + if (response.Status == (int)HttpStatusCode.OK) + { + return response.Content!.ToObject(new JsonObjectSerializer()); + } + statusCode = response.Status; + } + catch (RequestFailedException ex) { - return response.Content!.ToObject(new JsonObjectSerializer()); + statusCode = ex.Status; } + HandleAPIFailure(statusCode, "GetStorageUri"); return null; } + + private void HandleAPIFailure(int? statusCode, string operationName) + { + if (statusCode == null) + return; + ApiErrorConstants.s_errorOperationPair.TryGetValue(operationName, out System.Collections.Generic.Dictionary? errorObject); + if (errorObject == null) + return; + errorObject.TryGetValue((int)statusCode, out string? errorMessage); + if (errorMessage == null) + errorMessage = ReporterConstants.s_uNKNOWN_ERROR_MESSAGE; + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, operationName); + } } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs index f2c507b3bc50..7dbbecf68c14 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs @@ -9,5 +9,6 @@ internal interface ICloudRunErrorParser bool TryPushMessageAndKey(string? message, string? key); void PushMessage(string message); void DisplayMessages(); + void PrintErrorToConsole(string message); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs index 30a051fda807..41939a5d6429 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs @@ -6,5 +6,6 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface internal interface IConsoleWriter { void WriteLine(string? message = null); + void WriteError(string? message = null); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs index 831c793ae89c..6410272b0043 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -45,7 +45,7 @@ public TestProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? _logger = logger ?? new Logger(); _dataProcessor = dataProcessor ?? new DataProcessor(_cloudRunMetadata, _cIInfo, _logger); _cloudRunErrorParser = cloudRunErrorParser ?? new CloudRunErrorParser(_logger); - _serviceClient = serviceClient ?? new ServiceClient(_cloudRunMetadata); + _serviceClient = serviceClient ?? new ServiceClient(_cloudRunMetadata, _cloudRunErrorParser); _consoleWriter = consoleWriter ?? new ConsoleWriter(); } @@ -89,10 +89,6 @@ public void TestCaseResultHandler(object? sender, TestResultEventArgs e) { try { - if (FatalTestExecution) - { - return; - } TestResult testResultSource = e.Result; TestResults? testResult = _dataProcessor.GetTestCaseResultData(testResultSource); RawTestResult rawResult = DataProcessor.GetRawResultObject(testResultSource); From 4eff9d8a3d1cca7e7650aa31db7016ba6fb0412b Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 11:09:18 +0530 Subject: [PATCH 05/18] fix(): no access token error message logged with error level --- .../src/PlaywrightReporter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs index 3f5d660ee0b3..1e23395d07d1 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs @@ -122,7 +122,7 @@ private void InitializePlaywrightReporter(string xmlSettings) } if (string.IsNullOrEmpty(accessToken)) { - Console.WriteLine(Constants.s_no_auth_error); + Console.Error.WriteLine(Constants.s_no_auth_error); Environment.Exit(1); } From ca57afcc2b1880f6b1a7b5a7d4faebb3369b4427 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 11:18:13 +0530 Subject: [PATCH 06/18] chore(): remove null check condition in console writer --- .../src/Implementation/ConsoleWriter.cs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs index bfbfe7766cb0..c0092b80de03 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs @@ -10,26 +10,12 @@ internal class ConsoleWriter : IConsoleWriter { public void WriteLine(string? message = null) { - if (message == null) - { - Console.WriteLine(); - } - else - { - Console.WriteLine(message); - } + Console.WriteLine(message); } public void WriteError(string? message = null) { - if (message == null) - { - Console.Error.WriteLine(); - } - else - { - Console.Error.WriteLine(message); - } + Console.Error.WriteLine(message); } } } From 00e6f595fdf9dc8789376e21b3b0c12dbc28a4b0 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 11:42:24 +0530 Subject: [PATCH 07/18] chore(): add different logging levels --- .../src/Constants.cs | 2 +- .../src/Implementation/Logger.cs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 76beb50f6a0a..e381ee7dfa4e 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -187,7 +187,7 @@ internal class Constants // Default constants internal static readonly string s_default_os = ServiceOs.Linux; internal static readonly string s_default_expose_network = ""; - internal static readonly string s_pLAYWRIGHT_SERVICE_DEBUG = "PLAYWRIGHT_SERVICE_DEBUG"; + internal static readonly string s_pLAYWRIGHT_SERVICE_DEBUG = "Logging__LogLevel__MicrosoftPlaywrightTesting"; // Entra id access token constants internal static readonly int s_entra_access_token_lifetime_left_threshold_in_minutes_for_rotation = 15; diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs index b1d508840637..3251ea8584d7 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs @@ -16,15 +16,16 @@ internal enum LogLevel internal class Logger : ILogger { - internal static bool EnableDebug { get { return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.s_pLAYWRIGHT_SERVICE_DEBUG)); } set { } } + internal static string SdkLogLevel => Environment.GetEnvironmentVariable(Constants.s_pLAYWRIGHT_SERVICE_DEBUG); #pragma warning disable CA1822 // Mark members as static private void Log(LogLevel level, string message) #pragma warning restore CA1822 // Mark members as static { - if (EnableDebug) + if (Enum.TryParse(SdkLogLevel, out LogLevel configuredLevel) && level >= configuredLevel) { - Console.WriteLine($"{DateTime.Now} [{level}]: {message}"); + System.IO.TextWriter writer = level == LogLevel.Error || level == LogLevel.Warning ? Console.Error : Console.Out; + writer.WriteLine($"{DateTime.Now} [{level}]: {message}"); } } From 3a92c39e4de5dca53d01b8932cf4945564573d11 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 13:42:35 +0530 Subject: [PATCH 08/18] refactor(): rename logger to microsoft-playwright-testing --- .../Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md | 2 +- .../src/PlaywrightReporter.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md index bbca4b74ad9d..5f181658a03e 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md @@ -81,7 +81,7 @@ Ensure that the `PLAYWRIGHT_SERVICE_URL` that you obtained in previous step is a Run Playwright tests against browsers managed by the service using the configuration you created above. ```dotnetcli -dotnet test --logger "ms-playwright-service" +dotnet test --logger "microsoft-playwright-testing" ``` ## Key concepts diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs index 1e23395d07d1..d59b4ea4f921 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs @@ -15,7 +15,7 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; -[FriendlyName("ms-playwright-service")] +[FriendlyName("microsoft-playwright-testing")] [ExtensionUri("logger://Microsoft/Playwright/ServiceLogger/v1")] internal class PlaywrightReporter : ITestLoggerWithParameters { From cf885696d767e145ec6735facd3faecbc2130ca9 Mon Sep 17 00:00:00 2001 From: Kashish Gupta <90824921+kashish2508@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:45:36 +0530 Subject: [PATCH 09/18] Update TestProcessor.cs --- .../src/Processor/TestProcessor.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs index 6410272b0043..8f97f2220938 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -10,6 +10,7 @@ using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; using Azure.Storage.Blobs; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -152,6 +153,14 @@ public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) { if (RawTestResultsMap.TryGetValue(testResult.TestExecutionId!, out RawTestResult? rawResult) && rawResult != null) { + // Renew the SAS URI if needed + var reporterUtils = new ReporterUtils(); + if (sasUri == null || !reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri.Uri)) + { + sasUri = _serviceClient.GetTestRunResultsUri(); // Create new SAS URI + _logger.Info($"Fetched SAS URI with validity: {sasUri?.ExpiresAt} and access: {sasUri?.AccessLevel}."); + } + // Upload rawResult to blob storage using sasUri var rawTestResultJson = JsonSerializer.Serialize(rawResult); var filePath = $"{testResult.TestExecutionId}/rawTestResult.json"; @@ -159,14 +168,14 @@ public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) } else { - _logger.Info("Couldnt find rawResult for Id: " + testResult.TestExecutionId); + _logger.Info("Couldn't find rawResult for Id: " + testResult.TestExecutionId); } } _logger.Info("Successfully uploaded raw test results"); } else { - _logger.Error("Sas Uri is empty"); + _logger.Error("SAS URI is empty"); } } catch (Exception ex) @@ -177,7 +186,7 @@ public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) EndTestRun(e); } - #region Test Processor Helper Methods + #region Test Processor Helper Methods private void EndTestRun(TestRunCompleteEventArgs e) { if (_cloudRunMetadata.EnableResultPublish && !FatalTestExecution) From 9bb77717afae29cf36a4fcba6df26f6bc396128b Mon Sep 17 00:00:00 2001 From: Kashish Gupta <90824921+kashish2508@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:46:37 +0530 Subject: [PATCH 10/18] Update ReporterUtils.cs --- .../src/Utility/ReporterUtils.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs index f880c2ee7c2c..6a57757b491b 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs @@ -156,5 +156,46 @@ public TokenDetails ParseWorkspaceIdFromAccessToken(JsonWebTokenHandler? jsonWeb throw; } } + public bool IsTimeGreaterThanCurrentPlus10Minutes(string sasUri) + { + try + { + // Parse the SAS URI + Uri url = new Uri(sasUri); + string query = url.Query; + var queryParams = System.Web.HttpUtility.ParseQueryString(query); + string expiryTime = queryParams["se"]; // 'se' is the query parameter for the expiry time + + if (!string.IsNullOrEmpty(expiryTime)) + { + // Convert expiry time to a timestamp + DateTime expiryDateTime = DateTime.Parse(expiryTime, null, System.Globalization.DateTimeStyles.RoundtripKind); + long timestampFromIsoString = ((DateTimeOffset)expiryDateTime).ToUnixTimeMilliseconds(); + + // Get current time + 10 minutes in milliseconds + long currentTimestampPlus10Minutes = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + (10 * 60 * 1000); + + bool isSasValidityGreaterThanCurrentTimePlus10Minutes = timestampFromIsoString > currentTimestampPlus10Minutes; + + if (!isSasValidityGreaterThanCurrentTimePlus10Minutes) + { + // Log if SAS is close to expiry + _logger.Info( + $"Sas rotation required because close to expiry, SasUriValidTillTime: {timestampFromIsoString}, CurrentTime: {currentTimestampPlus10Minutes}" + ); + } + + return isSasValidityGreaterThanCurrentTimePlus10Minutes; + } + + _logger.Info("Sas rotation required because expiry param not found."); + return false; + } + catch (Exception ex) + { + _logger.Info($"Sas rotation required because of {ex.Message}."); + return false; + } + } } } From 094014d386c2b5ce191e22ec5dd96272730a667b Mon Sep 17 00:00:00 2001 From: Kashish Gupta <90824921+kashish2508@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:47:20 +0530 Subject: [PATCH 11/18] Create ReporterUtilsTests.cs --- .../tests/ReporterUtilsTests.cs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs new file mode 100644 index 000000000000..1f4c5c23e4c2 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests; + +[TestFixture] +[Parallelizable(ParallelScope.Self)] +public class ReporterUtilsTests +{ + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_ValidFutureSasUri_ReturnsTrue() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(15).ToString("o"); // 15 minutes in the future + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsTrue(result); + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_ExpiredSasUri_ReturnsFalse() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(-5).ToString("o"); // 5 minutes in the past + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsFalse(result); + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_InvalidSasUri_ReturnsFalse() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "not_a_valid_sas_uri"; // Invalid SAS URI + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsFalse(result); + } +} From 51fe5874ffbe403b5d7e9dfe5762695376c4d5f6 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Thu, 17 Oct 2024 22:01:30 +0530 Subject: [PATCH 12/18] fix(): uri escape data string for workspace and run id --- .../src/Model/CloudRunMetadata.cs | 5 ++++- .../src/PlaywrightReporter.cs | 12 +++++------- .../src/Processor/TestProcessor.cs | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs index d80d56b4cb98..124eed6ab0ce 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs @@ -10,7 +10,10 @@ internal class CloudRunMetadata internal string? WorkspaceId { get; set; } internal string? RunId { get; set; } internal Uri? BaseUri { get; set; } - internal string? PortalUrl { get; set; } + internal string? PortalUrl + { + get { return ReporterConstants.s_portalBaseUrl + Uri.EscapeDataString(WorkspaceId) + ReporterConstants.s_reportingRoute + Uri.EscapeDataString(RunId); } + } internal bool EnableResultPublish { get; set; } = true; internal bool EnableGithubSummary { get; set; } = true; internal DateTime TestRunStartTime { get; set; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs index d59b4ea4f921..bfb6d6fdccb3 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs @@ -16,11 +16,11 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; [FriendlyName("microsoft-playwright-testing")] -[ExtensionUri("logger://Microsoft/Playwright/ServiceLogger/v1")] +[ExtensionUri("logger://MicrosoftPlaywrightTesting/Logger/v1")] internal class PlaywrightReporter : ITestLoggerWithParameters { private Dictionary? _parametersDictionary; - private PlaywrightService? playwrightService; + private PlaywrightService? _playwrightService; private readonly ILogger _logger; private TestProcessor? _testProcessor; @@ -62,7 +62,7 @@ internal void TestResultHandler(object? sender, TestResultEventArgs e) internal void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) { _testProcessor?.TestRunCompleteHandler(sender, e); - playwrightService?.Cleanup(); + _playwrightService?.Cleanup(); } #endregion @@ -107,9 +107,9 @@ private void InitializePlaywrightReporter(string xmlSettings) } // setup entra rotation handlers - playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, playwrightServiceSettings.AzureTokenCredential); + _playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, playwrightServiceSettings.AzureTokenCredential); #pragma warning disable AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead. - playwrightService.InitializeAsync().GetAwaiter().GetResult(); + _playwrightService.InitializeAsync().GetAwaiter().GetResult(); #pragma warning restore AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead. var cloudRunId = Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId); @@ -130,14 +130,12 @@ private void InitializePlaywrightReporter(string xmlSettings) var reporterUtils = new ReporterUtils(); TokenDetails tokenDetails = reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler: null, accessToken: accessToken); var workspaceId = tokenDetails.aid; - var portalUri = ReporterConstants.s_portalBaseUrl + workspaceId + ReporterConstants.s_reportingRoute + cloudRunId; var cloudRunMetadata = new CloudRunMetadata { RunId = cloudRunId, WorkspaceId = workspaceId, BaseUri = baseUri, - PortalUrl = portalUri, EnableResultPublish = _enableResultPublish, EnableGithubSummary = _enableGitHubSummary, TestRunStartTime = DateTime.UtcNow, diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs index 8f97f2220938..06303fa8ed2f 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -78,7 +78,7 @@ public void TestRunStartHandler(object? sender, TestRunStartEventArgs e) } _testRunShard = testShard; _logger.Info("Successfully patched test run shard - init"); - _consoleWriter.WriteLine($"\nInitializing reporting for this test run. You can view the results at: {Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}"); + _consoleWriter.WriteLine($"\nInitializing reporting for this test run. You can view the results at: {_cloudRunMetadata.PortalUrl!}"); } catch (Exception ex) { @@ -201,7 +201,7 @@ private void EndTestRun(TestRunCompleteEventArgs e) { _logger.Error($"Failed to end test run shard: {ex}"); } - _consoleWriter.WriteLine($"\nTest Report: {Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}"); + _consoleWriter.WriteLine($"\nTest Report: {_cloudRunMetadata.PortalUrl!}"); if (_cloudRunMetadata.EnableGithubSummary) { GenerateMarkdownSummary(); From 6cdba5a5215821a84d5d3e163319ce7d031c1170 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Fri, 18 Oct 2024 01:27:06 +0530 Subject: [PATCH 13/18] fix(): ad hoc fixes --- .../samples/Sample1_CustomisingServiceParameters.md | 4 ++-- .../samples/Sample2_SetDefaultAuthenticationMechanism.md | 2 +- .../src/Constants.cs | 8 ++++---- .../src/Implementation/CloudRunErrorParser.cs | 2 +- .../src/Implementation/ServiceClient.cs | 2 +- .../src/Utility/ReporterUtils.cs | 5 +++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md index ef62a3929fe6..a2eeefe04b61 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md @@ -16,7 +16,7 @@ This guide explains the different options available to you in the Azure.Develope - + @@ -79,7 +79,7 @@ public class PlaywrightServiceSetup : PlaywrightServiceNUnit 3. **`ExposeNetwork`**: - **Description**: This settings exposes network available on the connecting client to the browser being connected to. -4. **`ServiceAuth`** +4. **`ServiceAuthType`** - **Description**: This setting allows you to specify the default authentication mechanism to be used for sending requests to the service. - **Available Options**: - `ServiceAuthType.EntraId` for Microsoft Entra ID authentication. diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md index 13c3ba68314c..d8fd28f8d7eb 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md @@ -24,7 +24,7 @@ public class PlaywrightServiceSetup : PlaywrightServiceNUnit {}; - + ``` diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index e381ee7dfa4e..3dc2180bdc32 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -297,28 +297,28 @@ internal static class TestResultErrorConstants { Key = "InvalidWorkspace_Scalable", Message = "The specified workspace does not exist. Please verify your workspace settings.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase), + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase), Type = TestErrorType.Scalable }, new TestResultError { Key = "InvalidAccessToken", Message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccessToken)", RegexOptions.IgnoreCase), + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*InvalidAccessToken)", RegexOptions.IgnoreCase), Type = TestErrorType.Scalable }, new TestResultError { Key = "AccessTokenOrUserOrWorkspaceNotFound_Scalable", Message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*404 Not Found)(?=.*NotFound)", RegexOptions.IgnoreCase), + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*404 Not Found)(?=[\s\S]*NotFound)", RegexOptions.IgnoreCase), Type = TestErrorType.Scalable }, new TestResultError { Key = "AccessKeyBasedAuthNotSupported_Scalable", Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase), + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase), Type = TestErrorType.Scalable }, new TestResultError diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs index 12ad11bef2b9..2410754187a5 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs @@ -8,7 +8,7 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation { internal class CloudRunErrorParser : ICloudRunErrorParser { - private List InformationalMessages { get; set; } = new(); + internal List InformationalMessages { get; private set; } = new(); private List ProcessedErrorMessageKeys { get; set; } = new(); private readonly ILogger _logger; private readonly IConsoleWriter _consoleWriter; diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs index 87c247dce894..0db462621b35 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -145,7 +145,7 @@ private void HandleAPIFailure(int? statusCode, string operationName) errorObject.TryGetValue((int)statusCode, out string? errorMessage); if (errorMessage == null) errorMessage = ReporterConstants.s_uNKNOWN_ERROR_MESSAGE; - _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, operationName); + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, statusCode.ToString()); } } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs index 6a57757b491b..9923a81ab1f3 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs @@ -115,7 +115,7 @@ internal static string GetCurrentOS() return OSConstants.s_wINDOWS; } - public TokenDetails ParseWorkspaceIdFromAccessToken(JsonWebTokenHandler? jsonWebTokenHandler, string? accessToken) + internal TokenDetails ParseWorkspaceIdFromAccessToken(JsonWebTokenHandler? jsonWebTokenHandler, string? accessToken) { if (jsonWebTokenHandler == null) { @@ -156,7 +156,8 @@ public TokenDetails ParseWorkspaceIdFromAccessToken(JsonWebTokenHandler? jsonWeb throw; } } - public bool IsTimeGreaterThanCurrentPlus10Minutes(string sasUri) + + internal bool IsTimeGreaterThanCurrentPlus10Minutes(string sasUri) { try { From cc202fa3e648d824254fa1ab6f98f7f2219846cd Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Fri, 18 Oct 2024 01:28:35 +0530 Subject: [PATCH 14/18] fix(): handle null sas uri post refresh --- .../src/Processor/TestProcessor.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs index 06303fa8ed2f..650f7e0e4e09 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -160,6 +160,11 @@ public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) sasUri = _serviceClient.GetTestRunResultsUri(); // Create new SAS URI _logger.Info($"Fetched SAS URI with validity: {sasUri?.ExpiresAt} and access: {sasUri?.AccessLevel}."); } + if (sasUri == null) + { + _logger.Warning("SAS URI is empty"); + continue; // allow recovery from temporary reporter API failures. In the future, we might consider shortciruiting the upload process. + } // Upload rawResult to blob storage using sasUri var rawTestResultJson = JsonSerializer.Serialize(rawResult); @@ -186,7 +191,7 @@ public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) EndTestRun(e); } - #region Test Processor Helper Methods + #region Test Processor Helper Methods private void EndTestRun(TestRunCompleteEventArgs e) { if (_cloudRunMetadata.EnableResultPublish && !FatalTestExecution) From 8a40eb6a5e962c46b1ba51b6665e7037274ea76e Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Fri, 18 Oct 2024 14:26:12 +0530 Subject: [PATCH 15/18] chore(): change reporting api version 2024-09-01-preview --- .../src/Client/ReportingTestResultsClient.cs | 159 ----------------- ...stRunsClient.cs => TestReportingClient.cs} | 163 +++++++++--------- .../src/Client/TestReportingClientOptions.cs | 10 +- .../src/Constants.cs | 32 ++-- .../src/Implementation/ServiceClient.cs | 30 ++-- .../src/Interface/IDataProcessor.cs | 2 +- .../src/Interface/IServiceClient.cs | 4 +- .../src/Model/TestReporting.cs | 44 ++--- .../src/Processor/DataProcessor.cs | 52 ++---- .../src/Processor/TestProcessor.cs | 34 ++-- 10 files changed, 165 insertions(+), 365 deletions(-) delete mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs rename sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/{ReportingTestRunsClient.cs => TestReportingClient.cs} (74%) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs deleted file mode 100644 index ed2f8c76c3fe..000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// - -#nullable disable - -using System; -using System.Threading.Tasks; -using Azure.Core; -using Azure.Core.Pipeline; - -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client -{ - // Data plane generated client. - /// The TestResults service client. - internal partial class ReportingTestResultsClient - { - private readonly HttpPipeline _pipeline; - private readonly Uri _endpoint; - private readonly string _apiVersion; - - /// The ClientDiagnostics is used to provide tracing support for the client library. - internal ClientDiagnostics ClientDiagnostics { get; } - - /// The HTTP pipeline for sending and receiving REST requests and responses. - public virtual HttpPipeline Pipeline => _pipeline; - - /// Initializes a new instance of ReportingTestResultsClient for mocking. - protected ReportingTestResultsClient() - { - } - - /// Initializes a new instance of ReportingTestResultsClient. - /// server parameter. - /// is null. - public ReportingTestResultsClient(Uri endpoint) : this(endpoint, new TestReportingClientOptions()) - { - } - - /// Initializes a new instance of ReportingTestResultsClient. - /// server parameter. - /// The options for configuring the client. - /// is null. - public ReportingTestResultsClient(Uri endpoint, TestReportingClientOptions options) - { - Argument.AssertNotNull(endpoint, nameof(endpoint)); - options ??= new TestReportingClientOptions(); - - ClientDiagnostics = new ClientDiagnostics(options, true); - _pipeline = HttpPipelineBuilder.Build(options, Array.Empty(), Array.Empty(), new ResponseClassifier()); - _endpoint = endpoint; - _apiVersion = options.Version; - } - - /// - /// [Protocol Method] Uploads a batch of test results to the test run - /// - /// - /// - /// This protocol method allows explicit creation of the request and processing of the response for advanced scenarios. - /// - /// - /// - /// - /// The to use. - /// The content to send as the body of the request. - /// access token. - /// Correlation-id used for tracing and debugging. - /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// is null. - /// is an empty string, and was expected to be non-empty. - /// Service returned a non-success status code. - /// The response returned from the service. - public virtual async Task UploadBatchTestResultsAsync(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) - { - Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - - using var scope = ClientDiagnostics.CreateScope("ReportingTestResultsClient.UploadBatchTestResults"); - scope.Start(); - try - { - using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); - return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); - } - catch (Exception e) - { - scope.Failed(e); - throw; - } - } - - /// - /// [Protocol Method] Uploads a batch of test results to the test run - /// - /// - /// - /// This protocol method allows explicit creation of the request and processing of the response for advanced scenarios. - /// - /// - /// - /// - /// The to use. - /// The content to send as the body of the request. - /// access token. - /// Correlation-id used for tracing and debugging. - /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// is null. - /// is an empty string, and was expected to be non-empty. - /// Service returned a non-success status code. - /// The response returned from the service. - public virtual Response UploadBatchTestResults(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) - { - Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - - using var scope = ClientDiagnostics.CreateScope("ReportingTestResultsClient.UploadBatchTestResults"); - scope.Start(); - try - { - using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); - return _pipeline.ProcessMessage(message, context); - } - catch (Exception e) - { - scope.Failed(e); - throw; - } - } - - internal HttpMessage CreateUploadBatchTestResultsRequest(string workspaceId, RequestContent content, string authorization, string xCorrelationId, RequestContext context) - { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400500); - var request = message.Request; - request.Method = RequestMethod.Post; - var uri = new RawRequestUriBuilder(); - uri.Reset(_endpoint); - uri.AppendPath("/workspaces/", false); - uri.AppendPath(workspaceId, true); - uri.AppendPath("/test-results/upload-batch", false); - uri.AppendQuery("api-version", _apiVersion, true); - request.Uri = uri; - request.Headers.Add("Accept", "application/json"); - if (authorization != null) - { - request.Headers.Add("Authorization", authorization); - } - if (xCorrelationId != null) - { - request.Headers.Add("x-correlation-id", xCorrelationId); - } - request.Headers.Add("Content-Type", "application/json"); - request.Content = content; - return message; - } - - private static ResponseClassifier _responseClassifier200400500; - private static ResponseClassifier ResponseClassifier200400500 => _responseClassifier200400500 ??= new StatusCodeClassifier(stackalloc ushort[] { 200, 400, 500 }); - } -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestRunsClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClient.cs similarity index 74% rename from sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestRunsClient.cs rename to sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClient.cs index 4bdb0cee6e0a..12944db1f0f6 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestRunsClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClient.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // @@ -13,8 +13,8 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client { // Data plane generated client. - /// The TestRuns service client. - internal partial class ReportingTestRunsClient + /// The TestReporting service client. + internal partial class TestReportingClient { private readonly HttpPipeline _pipeline; private readonly Uri _endpoint; @@ -26,23 +26,23 @@ internal partial class ReportingTestRunsClient /// The HTTP pipeline for sending and receiving REST requests and responses. public virtual HttpPipeline Pipeline => _pipeline; - /// Initializes a new instance of ReportingTestRunsClient for mocking. - protected ReportingTestRunsClient() + /// Initializes a new instance of TestReportingClient for mocking. + protected TestReportingClient() { } - /// Initializes a new instance of ReportingTestRunsClient. + /// Initializes a new instance of TestReportingClient. /// server parameter. /// is null. - public ReportingTestRunsClient(Uri endpoint) : this(endpoint, new TestReportingClientOptions()) + public TestReportingClient(Uri endpoint) : this(endpoint, new TestReportingClientOptions()) { } - /// Initializes a new instance of ReportingTestRunsClient. + /// Initializes a new instance of TestReportingClient. /// server parameter. /// The options for configuring the client. /// is null. - public ReportingTestRunsClient(Uri endpoint, TestReportingClientOptions options) + public TestReportingClient(Uri endpoint, TestReportingClientOptions options) { Argument.AssertNotNull(endpoint, nameof(endpoint)); options ??= new TestReportingClientOptions(); @@ -54,7 +54,7 @@ public ReportingTestRunsClient(Uri endpoint, TestReportingClientOptions options) } /// - /// [Protocol Method] Patch Test Run Info + /// [Protocol Method] /// /// /// @@ -64,26 +64,23 @@ public ReportingTestRunsClient(Uri endpoint, TestReportingClientOptions options) /// /// /// The to use. - /// The to use. /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// or is null. - /// or is an empty string, and was expected to be non-empty. + /// is null. + /// is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task PatchTestRunInfoAsync(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual async Task UploadBatchTestResultsAsync(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.UploadBatchTestResults"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunInfoAsyncRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -94,7 +91,7 @@ public virtual async Task PatchTestRunInfoAsync(string workspaceId, st } /// - /// [Protocol Method] Patch Test Run Info + /// [Protocol Method] /// /// /// @@ -104,26 +101,23 @@ public virtual async Task PatchTestRunInfoAsync(string workspaceId, st /// /// /// The to use. - /// The to use. /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// or is null. - /// or is an empty string, and was expected to be non-empty. + /// is null. + /// is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual Response UploadBatchTestResults(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.UploadBatchTestResults"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunInfoAsyncRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -134,7 +128,7 @@ public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, R } /// - /// [Protocol Method] Get Test Run Info + /// [Protocol Method] /// /// /// @@ -145,6 +139,7 @@ public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, R /// /// The to use. /// The to use. + /// The content to send as the body of the request. /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -152,16 +147,16 @@ public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, R /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task GetTestRunAsync(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual async Task PatchTestRunInfoAsync(string workspaceId, string testRunId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PatchTestRunInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePatchTestRunInfoRequest(workspaceId, testRunId, content, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -172,7 +167,7 @@ public virtual async Task GetTestRunAsync(string workspaceId, string t } /// - /// [Protocol Method] Get Test Run Info + /// [Protocol Method] /// /// /// @@ -183,6 +178,7 @@ public virtual async Task GetTestRunAsync(string workspaceId, string t /// /// The to use. /// The to use. + /// The content to send as the body of the request. /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -190,16 +186,16 @@ public virtual async Task GetTestRunAsync(string workspaceId, string t /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response GetTestRun(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PatchTestRunInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePatchTestRunInfoRequest(workspaceId, testRunId, content, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -210,7 +206,7 @@ public virtual Response GetTestRun(string workspaceId, string testRunId, string } /// - /// [Protocol Method] Patch Test Run Shard Info + /// [Protocol Method] /// /// /// @@ -221,27 +217,23 @@ public virtual Response GetTestRun(string workspaceId, string testRunId, string /// /// The to use. /// The to use. - /// The to use. - /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// , or is null. - /// , or is an empty string, and was expected to be non-empty. + /// or is null. + /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task PatchTestRunShardInfoAsync(string workspaceId, string testRunId, string shardId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual async Task GetTestRunResultsUriAsync(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - Argument.AssertNotNullOrEmpty(shardId, nameof(shardId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunShardInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.GetTestRunResultsUri"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunShardInfoAsyncRequest(workspaceId, testRunId, shardId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateGetTestRunResultsUriRequest(workspaceId, testRunId, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -252,7 +244,7 @@ public virtual async Task PatchTestRunShardInfoAsync(string workspaceI } /// - /// [Protocol Method] Patch Test Run Shard Info + /// [Protocol Method] /// /// /// @@ -263,27 +255,23 @@ public virtual async Task PatchTestRunShardInfoAsync(string workspaceI /// /// The to use. /// The to use. - /// The to use. - /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// , or is null. - /// , or is an empty string, and was expected to be non-empty. + /// or is null. + /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response PatchTestRunShardInfo(string workspaceId, string testRunId, string shardId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual Response GetTestRunResultsUri(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - Argument.AssertNotNullOrEmpty(shardId, nameof(shardId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunShardInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.GetTestRunResultsUri"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunShardInfoAsyncRequest(workspaceId, testRunId, shardId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateGetTestRunResultsUriRequest(workspaceId, testRunId, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -294,7 +282,7 @@ public virtual Response PatchTestRunShardInfo(string workspaceId, string testRun } /// - /// [Protocol Method] Get Test Run Results Uri + /// [Protocol Method] /// /// /// @@ -305,6 +293,8 @@ public virtual Response PatchTestRunShardInfo(string workspaceId, string testRun /// /// The to use. /// The to use. + /// The content to send as the body of the request. + /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -312,16 +302,16 @@ public virtual Response PatchTestRunShardInfo(string workspaceId, string testRun /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task GetTestRunResultsUriAsync(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual async Task PostTestRunShardInfoAsync(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunResultsUriAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PostTestRunShardInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunResultsUriAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePostTestRunShardInfoRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -332,7 +322,7 @@ public virtual async Task GetTestRunResultsUriAsync(string workspaceId } /// - /// [Protocol Method] Get Test Run Results Uri + /// [Protocol Method] /// /// /// @@ -343,6 +333,8 @@ public virtual async Task GetTestRunResultsUriAsync(string workspaceId /// /// The to use. /// The to use. + /// The content to send as the body of the request. + /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -350,16 +342,16 @@ public virtual async Task GetTestRunResultsUriAsync(string workspaceId /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response GetTestRunResultsUri(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual Response PostTestRunShardInfo(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunResultsUriAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PostTestRunShardInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunResultsUriAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePostTestRunShardInfoRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -369,20 +361,18 @@ public virtual Response GetTestRunResultsUri(string workspaceId, string testRunI } } - internal HttpMessage CreatePatchTestRunInfoAsyncRequest(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreateUploadBatchTestResultsRequest(string workspaceId, RequestContent content, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Patch; + request.Method = RequestMethod.Post; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); uri.AppendPath(workspaceId, true); - uri.AppendPath("/test-runs/", false); - uri.AppendPath(testRunId, true); + uri.AppendPath("/test-results/upload-batch", false); uri.AppendQuery("api-version", _apiVersion, true); request.Uri = uri; - request.Headers.Add("Accept", "application/json"); if (authorization != null) { request.Headers.Add("Authorization", authorization); @@ -391,16 +381,16 @@ internal HttpMessage CreatePatchTestRunInfoAsyncRequest(string workspaceId, stri { request.Headers.Add("x-correlation-id", xCorrelationId); } - request.Headers.Add("Content-Type", contentType.ToString()); + request.Headers.Add("Content-Type", "application/json"); request.Content = content; return message; } - internal HttpMessage CreateGetTestRunAsyncRequest(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreatePatchTestRunInfoRequest(string workspaceId, string testRunId, RequestContent content, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Get; + request.Method = RequestMethod.Patch; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); @@ -418,22 +408,23 @@ internal HttpMessage CreateGetTestRunAsyncRequest(string workspaceId, string tes { request.Headers.Add("x-correlation-id", xCorrelationId); } + request.Headers.Add("Content-Type", "application/merge-patch+json"); + request.Content = content; return message; } - internal HttpMessage CreatePatchTestRunShardInfoAsyncRequest(string workspaceId, string testRunId, string shardId, RequestContent content, ContentType contentType, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreateGetTestRunResultsUriRequest(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Patch; + request.Method = RequestMethod.Post; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); uri.AppendPath(workspaceId, true); uri.AppendPath("/test-runs/", false); uri.AppendPath(testRunId, true); - uri.AppendPath("/shards/", false); - uri.AppendPath(shardId, true); + uri.AppendPath(":createartifactsuploadbaseuri", false); uri.AppendQuery("api-version", _apiVersion, true); request.Uri = uri; request.Headers.Add("Accept", "application/json"); @@ -445,23 +436,21 @@ internal HttpMessage CreatePatchTestRunShardInfoAsyncRequest(string workspaceId, { request.Headers.Add("x-correlation-id", xCorrelationId); } - request.Headers.Add("Content-Type", contentType.ToString()); - request.Content = content; return message; } - internal HttpMessage CreateGetTestRunResultsUriAsyncRequest(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreatePostTestRunShardInfoRequest(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Get; + request.Method = RequestMethod.Post; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); uri.AppendPath(workspaceId, true); uri.AppendPath("/test-runs/", false); uri.AppendPath(testRunId, true); - uri.AppendPath("/resulturi", false); + uri.AppendPath(":updateshardexecutionstatus", false); uri.AppendQuery("api-version", _apiVersion, true); request.Uri = uri; request.Headers.Add("Accept", "application/json"); @@ -473,10 +462,12 @@ internal HttpMessage CreateGetTestRunResultsUriAsyncRequest(string workspaceId, { request.Headers.Add("x-correlation-id", xCorrelationId); } + request.Headers.Add("Content-Type", contentType.ToString()); + request.Content = content; return message; } - private static ResponseClassifier _responseClassifier200400401500; - private static ResponseClassifier ResponseClassifier200400401500 => _responseClassifier200400401500 ??= new StatusCodeClassifier(stackalloc ushort[] { 200, 400, 401, 500 }); + private static ResponseClassifier _responseClassifier200; + private static ResponseClassifier ResponseClassifier200 => _responseClassifier200 ??= new StatusCodeClassifier(stackalloc ushort[] { 200 }); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs index cf4492767fe7..8416fd53cc7b 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs @@ -10,16 +10,16 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client { - /// Client options for TestReporting library clients. + /// Client options for TestReportingClient. public partial class TestReportingClientOptions : ClientOptions { - private const ServiceVersion LatestVersion = ServiceVersion.V2024_05_20_Preview; + private const ServiceVersion LatestVersion = ServiceVersion.V2024_09_01_Preview; /// The version of the service to use. public enum ServiceVersion { - /// Service version "2024-05-20-preview". - V2024_05_20_Preview = 1, + /// Service version "2024-09-01-preview". + V2024_09_01_Preview = 1, } internal string Version { get; } @@ -29,7 +29,7 @@ public TestReportingClientOptions(ServiceVersion version = LatestVersion) { Version = version switch { - ServiceVersion.V2024_05_20_Preview => "2024-05-20-preview", + ServiceVersion.V2024_09_01_Preview => "2024-09-01-preview", _ => throw new NotSupportedException() }; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 3dc2180bdc32..f5bca5359440 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -212,9 +212,9 @@ internal class Constants internal class OSConstants { - internal static readonly string s_lINUX = "Linux"; - internal static readonly string s_wINDOWS = "Windows"; - internal static readonly string s_mACOS = "MacOS"; + internal static readonly string s_lINUX = "LINUX"; + internal static readonly string s_wINDOWS = "WINDOWS"; + internal static readonly string s_mACOS = "MACOS"; } internal class ReporterConstants @@ -240,17 +240,17 @@ internal class ReporterConstants internal class CIConstants { - internal static readonly string s_gITHUB_ACTIONS = "GitHub Actions"; - internal static readonly string s_aZURE_DEVOPS = "Azure DevOps"; - internal static readonly string s_dEFAULT = "Default"; + internal static readonly string s_gITHUB_ACTIONS = "GITHUB"; + internal static readonly string s_aZURE_DEVOPS = "ADO"; + internal static readonly string s_dEFAULT = "DEFAULT"; } internal class TestCaseResultStatus { - internal static readonly string s_pASSED = "passed"; - internal static readonly string s_fAILED = "failed"; - internal static readonly string s_sKIPPED = "skipped"; - internal static readonly string s_iNCONCLUSIVE = "inconclusive"; + internal static readonly string s_pASSED = "PASSED"; + internal static readonly string s_fAILED = "FAILED"; + internal static readonly string s_sKIPPED = "SKIPPED"; + internal static readonly string s_iNCONCLUSIVE = "INCONCLUSIVE"; } internal class TestResultError @@ -363,7 +363,7 @@ internal static class ApiErrorConstants { 503, "The service is currently unavailable. Please check the service status and try again." } }; - private static Dictionary GetTestRun { get; set; } = new Dictionary() + private static Dictionary UploadBatchTestResults { get; set; } = new Dictionary() { { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, { 401, "The authentication token provided is invalid. Please check the token and try again." }, @@ -373,7 +373,7 @@ internal static class ApiErrorConstants { 504, "The request to the service timed out. Please try again later." }, { 503, "The service is currently unavailable. Please check the service status and try again." } }; - private static Dictionary PatchTestRunShard { get; set; } = new Dictionary() + private static Dictionary PostTestRunShardInfo { get; set; } = new Dictionary() { { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, { 401, "The authentication token provided is invalid. Please check the token and try again." }, @@ -383,7 +383,7 @@ internal static class ApiErrorConstants { 504, "The request to the service timed out. Please try again later." }, { 503, "The service is currently unavailable. Please check the service status and try again." } }; - private static Dictionary GetStorageUri { get; set; } = new Dictionary() + private static Dictionary GetTestRunResultsUri { get; set; } = new Dictionary() { { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, { 401, "The authentication token provided is invalid. Please check the token and try again." }, @@ -397,8 +397,8 @@ internal static class ApiErrorConstants internal static readonly Dictionary> s_errorOperationPair = new() { { "PatchTestRun", PatchTestRun }, - { "GetTestRun", GetTestRun }, - { "PatchTestRunShard", PatchTestRunShard }, - { "GetStorageUri", GetStorageUri } + { "UploadBatchTestResults", UploadBatchTestResults }, + { "PostTestRunShardInfo", PostTestRunShardInfo }, + { "GetTestRunResultsUri", GetTestRunResultsUri } }; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs index 0db462621b35..b6329e05675e 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -15,15 +15,15 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation { internal class ServiceClient : IServiceClient { - private readonly ReportingTestResultsClient _reportingTestResultsClient; - private readonly ReportingTestRunsClient _reportingTestRunsClient; + private readonly TestReportingClient _testReportingClient; private readonly CloudRunMetadata _cloudRunMetadata; private readonly ICloudRunErrorParser _cloudRunErrorParser; private readonly ILogger _logger; private static string AccessToken { get => $"Bearer {Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken)}"; set { } } private static string CorrelationId { get => Guid.NewGuid().ToString(); set { } } + private static readonly JsonSerializerOptions _jsonSerializerOptions = new() { DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; - public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser cloudRunErrorParser, ReportingTestResultsClient? reportingTestResultsClient = null, ReportingTestRunsClient? reportingTestRunsClient = null, ILogger? logger = null) + public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser cloudRunErrorParser, TestReportingClient? testReportingClient = null, ILogger? logger = null) { _cloudRunMetadata = cloudRunMetadata; _cloudRunErrorParser = cloudRunErrorParser; @@ -34,23 +34,21 @@ public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser clo }, EventLevel.Informational); var clientOptions = new TestReportingClientOptions(); clientOptions.Diagnostics.IsLoggingEnabled = true; - clientOptions.Diagnostics.IsLoggingContentEnabled = false; clientOptions.Diagnostics.IsTelemetryEnabled = true; clientOptions.Retry.MaxRetries = ServiceClientConstants.s_mAX_RETRIES; clientOptions.Retry.MaxDelay = TimeSpan.FromSeconds(ServiceClientConstants.s_mAX_RETRY_DELAY_IN_SECONDS); - _reportingTestResultsClient = reportingTestResultsClient ?? new ReportingTestResultsClient(_cloudRunMetadata.BaseUri, clientOptions); - _reportingTestRunsClient = reportingTestRunsClient ?? new ReportingTestRunsClient(_cloudRunMetadata.BaseUri, clientOptions); + _testReportingClient = testReportingClient ?? new TestReportingClient(_cloudRunMetadata.BaseUri, clientOptions); } - public TestRunDtoV2? PatchTestRunInfo(TestRunDtoV2 run) + public TestRunDto? PatchTestRunInfo(TestRunDto run) { int statusCode; try { - Response? apiResponse = _reportingTestRunsClient.PatchTestRunInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(run), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + Response? apiResponse = _testReportingClient.PatchTestRunInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(JsonSerializer.Serialize(run)), AccessToken, CorrelationId); if (apiResponse.Status == (int)HttpStatusCode.OK) { - return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); } statusCode = apiResponse.Status; } @@ -76,12 +74,12 @@ public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser clo return null; } - public TestRunShardDto? PatchTestRunShardInfo(int shardId, TestRunShardDto runShard) + public TestRunShardDto? PostTestRunShardInfo(TestRunShardDto runShard) { int statusCode; try { - Response apiResponse = _reportingTestRunsClient.PatchTestRunShardInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, shardId.ToString(), RequestContent.Create(runShard), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + Response apiResponse = _testReportingClient.PostTestRunShardInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(runShard), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); if (apiResponse.Status == (int)HttpStatusCode.OK) { return apiResponse.Content!.ToObject(new JsonObjectSerializer()); @@ -92,7 +90,7 @@ public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser clo { statusCode = ex.Status; } - HandleAPIFailure(statusCode, "PatchTestRunShard"); + HandleAPIFailure(statusCode, "PostTestRunShardInfo"); return null; } @@ -101,7 +99,7 @@ public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsReq int statusCode; try { - Response apiResponse = _reportingTestResultsClient.UploadBatchTestResults(_cloudRunMetadata.WorkspaceId!, RequestContent.Create(JsonSerializer.Serialize(uploadTestResultsRequest)), AccessToken, CorrelationId, null); + Response apiResponse = _testReportingClient.UploadBatchTestResults(_cloudRunMetadata.WorkspaceId!, RequestContent.Create(JsonSerializer.Serialize(uploadTestResultsRequest)), AccessToken, CorrelationId, null); if (apiResponse.Status == (int)HttpStatusCode.OK) { return; @@ -112,7 +110,7 @@ public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsReq { statusCode = ex.Status; } - HandleAPIFailure(statusCode, "PostTestResults"); + HandleAPIFailure(statusCode, "UploadBatchTestResults"); } public TestResultsUri? GetTestRunResultsUri() @@ -120,7 +118,7 @@ public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsReq int statusCode; try { - Response response = _reportingTestRunsClient.GetTestRunResultsUri(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, AccessToken, CorrelationId, null); + Response response = _testReportingClient.GetTestRunResultsUri(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, AccessToken, CorrelationId, null); if (response.Status == (int)HttpStatusCode.OK) { return response.Content!.ToObject(new JsonObjectSerializer()); @@ -131,7 +129,7 @@ public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsReq { statusCode = ex.Status; } - HandleAPIFailure(statusCode, "GetStorageUri"); + HandleAPIFailure(statusCode, "GetTestRunResultsUri"); return null; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs index 37f49cbd1ccf..d40e9d97cc97 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs @@ -8,7 +8,7 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface { internal interface IDataProcessor { - TestRunDtoV2 GetTestRun(); + TestRunDto GetTestRun(); TestRunShardDto GetTestRunShard(); TestResults GetTestCaseResultData(TestResult testResultSource); } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs index a9e14963acb8..07edddb64230 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs @@ -7,8 +7,8 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface { internal interface IServiceClient { - TestRunDtoV2? PatchTestRunInfo(TestRunDtoV2 run); - TestRunShardDto? PatchTestRunShardInfo(int shardId, TestRunShardDto runShard); + TestRunDto? PatchTestRunInfo(TestRunDto run); + TestRunShardDto? PostTestRunShardInfo(TestRunShardDto runShard); void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsRequest); TestResultsUri? GetTestRunResultsUri(); } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs index 08cff08b222f..4692883feaa0 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs @@ -16,15 +16,15 @@ internal enum AccessLevel internal partial class CIConfig { - [JsonPropertyName("ciProviderName")] public string CiProviderName { get; set; } = ""; + [JsonPropertyName("ciProviderName")] public string? CiProviderName { get; set; } = ""; - [JsonPropertyName("branch")] public string Branch { get; set; } = ""; + [JsonPropertyName("branch")] public string? Branch { get; set; } - [JsonPropertyName("author")] public string Author { get; set; } = ""; + [JsonPropertyName("author")] public string? Author { get; set; } - [JsonPropertyName("commitId")] public string CommitId { get; set; } = ""; + [JsonPropertyName("commitId")] public string? CommitId { get; set; } - [JsonPropertyName("revisionUrl")] public string RevisionUrl { get; set; } = ""; + [JsonPropertyName("revisionUrl")] public string? RevisionUrl { get; set; } } internal partial class ClientConfig @@ -81,7 +81,7 @@ internal partial class Shard public int Total { get; set; } [JsonPropertyName("current")] - public int Current { get; set; } + public int? Current { get; set; } } internal partial class TestFramework @@ -103,6 +103,8 @@ internal partial class TestResults [JsonPropertyName("runId")] public string RunId { get; set; } = ""; + [JsonPropertyName("shardId")] public string ShardId { get; set; } = ""; + [JsonPropertyName("accountId")] public string AccountId { get; set; } = ""; [JsonPropertyName("suiteId")] public string SuiteId { get; set; } = ""; @@ -167,7 +169,7 @@ internal partial class TestResultsUri public AccessLevel? AccessLevel { get; set; } } -internal partial class TestRunDtoV2 +internal partial class TestRunDto { [JsonPropertyName("testRunId")] public string TestRunId { get; set; } = ""; @@ -199,9 +201,9 @@ internal partial class TestRunDtoV2 [JsonPropertyName("testResultsUri")] public TestResultsUri? TestResultsUri { get; set; } - [JsonPropertyName("cloudRunEnabled")] public string CloudRunEnabled { get; set; } = ""; + [JsonPropertyName("cloudRunEnabled")] public bool? CloudRunEnabled { get; set; } - [JsonPropertyName("cloudReportingEnabled")] public string CloudReportingEnabled { get; set; } = ""; + [JsonPropertyName("cloudReportingEnabled")] public bool? CloudReportingEnabled { get; set; } } internal partial class TestRunResultsSummary @@ -227,22 +229,18 @@ internal partial class TestRunResultsSummary internal partial class TestRunShardDto { - [JsonPropertyName("uploadCompleted")] public string UploadCompleted { get; set; } = ""; + [JsonPropertyName("shardId")] public string ShardId { get; set; } = ""; + [JsonPropertyName("uploadCompleted")] public bool UploadCompleted { get; set; } = false; [JsonPropertyName("summary")] public TestRunShardSummary? Summary { get; set; } - [JsonPropertyName("testRunConfig")] - public ClientConfig? TestRunConfig { get; set; } - - [JsonPropertyName("resultsSummary")] - public TestRunResultsSummary? ResultsSummary { get; set; } + [JsonPropertyName("workers")] public int? Workers { get; set; } } internal partial class TestRunShardSummary { [JsonPropertyName("status")] public string Status { get; set; } = ""; - [JsonPropertyName("startTime")] public string StartTime { get; set; } = ""; [JsonPropertyName("endTime")] public string EndTime { get; set; } = ""; @@ -260,14 +258,8 @@ internal partial class TestRunSummary { [JsonPropertyName("status")] public string Status { get; set; } = ""; - [JsonPropertyName("startTime")] public string StartTime { get; set; } = ""; - - [JsonPropertyName("endTime")] public string EndTime { get; set; } = ""; - [JsonPropertyName("billableTime")] public long BillableTime { get; set; } - [JsonPropertyName("totalTime")] public long TotalTime { get; set; } - [JsonPropertyName("numBrowserSessions")] public long NumBrowserSessions { get; set; } [JsonPropertyName("jobs")] @@ -278,12 +270,6 @@ internal partial class TestRunSummary [JsonPropertyName("tags")] public ICollection Tags { get; set; } = new List(); - - [JsonPropertyName("errorMessages")] - public ICollection ErrorMessages { get; set; } = new List(); - - [JsonPropertyName("uploadMetadata")] - public UploadMetadata? UploadMetadata { get; set; } } internal partial class UploadMetadata @@ -307,7 +293,7 @@ internal partial class WebTestConfig [JsonPropertyName("projectName")] public string ProjectName { get; set; } = ""; - [JsonPropertyName("browserName")] public string BrowserName { get; set; } = ""; + [JsonPropertyName("browserType")] public string BrowserName { get; set; } = ""; [JsonPropertyName("os")] public string Os { get; set; } = ""; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs index 79323a251f93..bbca90edf96a 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs @@ -25,34 +25,27 @@ public DataProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? _logger = logger ?? new Logger(); } - public TestRunDtoV2 GetTestRun() + public TestRunDto GetTestRun() { var startTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); var gitBasedRunName = ReporterUtils.GetRunName(CiInfoProvider.GetCIInfo())?.Trim(); string runName = string.IsNullOrEmpty(gitBasedRunName) ? _cloudRunMetadata.RunId! : gitBasedRunName!; - var run = new TestRunDtoV2 + var run = new TestRunDto { TestRunId = _cloudRunMetadata.RunId!, DisplayName = runName, StartTime = startTime, CreatorId = _cloudRunMetadata.AccessTokenDetails!.oid ?? "", CreatorName = _cloudRunMetadata.AccessTokenDetails!.userName?.Trim() ?? "", - //CloudRunEnabled = "false", - CloudReportingEnabled = "true", - Summary = new TestRunSummary + CloudReportingEnabled = true, + CloudRunEnabled = false, + CiConfig = new CIConfig { - Status = "RUNNING", - StartTime = startTime, - //Projects = ["playwright-dotnet"], - //Tags = ["Nunit", "dotnet"], - //Jobs = ["playwright-dotnet"], - }, - CiConfig = new CIConfig // TODO fetch dynamically - { - Branch = _cIInfo.Branch ?? "", - Author = _cIInfo.Author ?? "", - CommitId = _cIInfo.CommitId ?? "", - RevisionUrl = _cIInfo.RevisionUrl ?? "" + Branch = _cIInfo.Branch, + Author = _cIInfo.Author, + CommitId = _cIInfo.CommitId, + RevisionUrl = _cIInfo.RevisionUrl, + CiProviderName = _cIInfo.Provider ?? CIConstants.s_dEFAULT }, TestRunConfig = new ClientConfig // TODO fetch some of these dynamically { @@ -60,10 +53,10 @@ public TestRunDtoV2 GetTestRun() PwVersion = "1.40", Timeout = 60000, TestType = "WebTest", - TestSdkLanguage = "Dotnet", - TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit/MSTest", Version = "3.1" }, // TODO fetch runner name MSTest/Nunit - ReporterPackageVersion = "0.0.1-dotnet", - Shards = new Shard() { Current = 0, Total = 1 } + TestSdkLanguage = "CSHARP", + TestFramework = new TestFramework() { Name = "PLAYWRIGHT", RunnerName = "NUNIT", Version = "3.1" }, // TODO fetch runner name MSTest/Nunit + ReporterPackageVersion = "1.0.0-beta.1", + Shards = new Shard() { Total = 1 } } }; return run; @@ -74,23 +67,14 @@ public TestRunShardDto GetTestRunShard() var startTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); var shard = new TestRunShardDto { - UploadCompleted = "false", + UploadCompleted = false, + ShardId = "1", Summary = new TestRunShardSummary { Status = "RUNNING", StartTime = startTime, }, - TestRunConfig = new ClientConfig // TODO fetch some of these dynamically - { - Workers = 1, - PwVersion = "1.40", - Timeout = 60000, - TestType = "Functional", - TestSdkLanguage = "dotnet", - TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit", Version = "3.1" }, - ReporterPackageVersion = "0.0.1-dotnet", - Shards = new Shard() { Current = 0, Total = 1 }, - } + Workers = 1 }; return shard; } @@ -129,7 +113,7 @@ public TestResults GetTestCaseResultData(TestResult testResultSource) testCaseResultData.ResultsSummary = new TestResultsSummary { Duration = (long)duration.TotalMilliseconds, // TODO fallback get from End-Start - StartTime = testResultSource.StartTime.UtcDateTime.ToString(), + StartTime = testResultSource.StartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"), Status = TestCaseResultStatus.s_iNCONCLUSIVE }; TestOutcome outcome = testResultSource.Outcome; diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs index 650f7e0e4e09..01c7e2667fc9 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -59,9 +59,9 @@ public void TestRunStartHandler(object? sender, TestRunStartEventArgs e) { return; } - TestRunDtoV2 run = _dataProcessor.GetTestRun(); + TestRunDto run = _dataProcessor.GetTestRun(); TestRunShardDto shard = _dataProcessor.GetTestRunShard(); - TestRunDtoV2? testRun = _serviceClient.PatchTestRunInfo(run); + TestRunDto? testRun = _serviceClient.PatchTestRunInfo(run); if (testRun == null) { _logger.Error("Failed to patch test run info"); @@ -69,7 +69,7 @@ public void TestRunStartHandler(object? sender, TestRunStartEventArgs e) return; } _logger.Info("Successfully patched test run - init"); - TestRunShardDto? testShard = _serviceClient.PatchTestRunShardInfo(1, shard); + TestRunShardDto? testShard = _serviceClient.PostTestRunShardInfo(shard); if (testShard == null) { _logger.Error("Failed to patch test run shard info"); @@ -199,7 +199,7 @@ private void EndTestRun(TestRunCompleteEventArgs e) try { _testRunShard = GetTestRunEndShard(e); - _serviceClient.PatchTestRunShardInfo(1, _testRunShard); + _serviceClient.PostTestRunShardInfo(_testRunShard); _logger.Info("Successfully ended test run shard"); } catch (Exception ex) @@ -251,16 +251,16 @@ private TestRunShardDto GetTestRunEndShard(TestRunCompleteEventArgs e) testRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ"); testRunShard.Summary.TotalTime = durationInMs; testRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 }; - testRunShard.ResultsSummary = new TestRunResultsSummary - { - NumTotalTests = TotalTestCount, - NumPassedTests = PassedTestCount, - NumFailedTests = FailedTestCount, - NumSkippedTests = SkippedTestCount, - NumFlakyTests = 0, // TODO: Implement flaky tests - Status = result - }; - testRunShard.UploadCompleted = "true"; + //testRunShard.Summary = new TestRunResultsSummary + //{ + // NumTotalTests = TotalTestCount, + // NumPassedTests = PassedTestCount, + // NumFailedTests = FailedTestCount, + // NumSkippedTests = SkippedTestCount, + // NumFlakyTests = 0, // TODO: Implement flaky tests + // Status = result + //}; + testRunShard.UploadCompleted = true; return testRunShard; } private void GenerateMarkdownSummary() @@ -270,13 +270,13 @@ private void GenerateMarkdownSummary() string markdownContent = @$" #### Results: -![pass](https://img.shields.io/badge/status-passed-brightgreen) **Passed:** {_testRunShard!.ResultsSummary!.NumPassedTests} +![pass](https://img.shields.io/badge/status-passed-brightgreen) **Passed:** {PassedTestCount} -![fail](https://img.shields.io/badge/status-failed-red) **Failed:** {_testRunShard.ResultsSummary.NumFailedTests} +![fail](https://img.shields.io/badge/status-failed-red) **Failed:** {FailedTestCount} ![flaky](https://img.shields.io/badge/status-flaky-yellow) **Flaky:** {"0"} -![skipped](https://img.shields.io/badge/status-skipped-lightgrey) **Skipped:** {_testRunShard.ResultsSummary.NumSkippedTests} +![skipped](https://img.shields.io/badge/status-skipped-lightgrey) **Skipped:** {SkippedTestCount} #### For more details, visit the [service dashboard]({Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}). "; From dde053b3383380cde841460ea3da2805be8910ee Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Fri, 18 Oct 2024 22:03:09 +0530 Subject: [PATCH 16/18] chore(): add tests for reporter --- .../src/Implementation/ServiceClient.cs | 26 +- .../src/Interface/IDataProcessor.cs | 2 +- .../src/Processor/DataProcessor.cs | 8 +- .../src/Processor/TestProcessor.cs | 28 +- .../tests/EntraLifecycleTests.cs | 1 + .../CloudRunErrorParserTests.cs | 236 +++++++ .../Implementation/ServiceClientTests.cs | 371 +++++++++++ .../tests/Model/CloudRunMetadataTests.cs | 48 ++ .../tests/Model/MPTResultTests.cs | 21 + .../tests/PlaywrightServiceTests.cs | 1 - .../tests/Processor/DataProcessorTests.cs | 179 ++++++ .../tests/Processor/TestProcessorTests.cs | 584 ++++++++++++++++++ .../tests/ReporterUtilsTests.cs | 38 -- .../tests/Utility/CiInfoProviderTests.cs | 257 ++++++++ .../tests/Utility/ReporterUtilsTests.cs | 145 +++++ 15 files changed, 1872 insertions(+), 73 deletions(-) create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs delete mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs create mode 100644 sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs index b6329e05675e..1d4f423da5f4 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -133,17 +133,23 @@ public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsReq return null; } - private void HandleAPIFailure(int? statusCode, string operationName) + internal void HandleAPIFailure(int? statusCode, string operationName) { - if (statusCode == null) - return; - ApiErrorConstants.s_errorOperationPair.TryGetValue(operationName, out System.Collections.Generic.Dictionary? errorObject); - if (errorObject == null) - return; - errorObject.TryGetValue((int)statusCode, out string? errorMessage); - if (errorMessage == null) - errorMessage = ReporterConstants.s_uNKNOWN_ERROR_MESSAGE; - _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, statusCode.ToString()); + try + { + if (statusCode == null) + return; + ApiErrorConstants.s_errorOperationPair.TryGetValue(operationName, out System.Collections.Generic.Dictionary? errorObject); + if (errorObject == null) + return; + errorObject.TryGetValue((int)statusCode, out string? errorMessage); + errorMessage ??= ReporterConstants.s_uNKNOWN_ERROR_MESSAGE; + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, statusCode.ToString()); + } + catch (Exception ex) + { + _logger.Error(ex.Message); + } } } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs index d40e9d97cc97..280a6fe08f84 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs @@ -10,6 +10,6 @@ internal interface IDataProcessor { TestRunDto GetTestRun(); TestRunShardDto GetTestRunShard(); - TestResults GetTestCaseResultData(TestResult testResultSource); + TestResults GetTestCaseResultData(TestResult? testResultSource); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs index bbca90edf96a..c5a4665c09a2 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs @@ -78,7 +78,7 @@ public TestRunShardDto GetTestRunShard() }; return shard; } - public TestResults GetTestCaseResultData(TestResult testResultSource) + public TestResults GetTestCaseResultData(TestResult? testResultSource) { if (testResultSource == null) return new TestResults(); @@ -139,9 +139,11 @@ public TestResults GetTestCaseResultData(TestResult testResultSource) return testCaseResultData; } - public static RawTestResult GetRawResultObject(TestResult testResultSource) + public static RawTestResult GetRawResultObject(TestResult? testResultSource) { - List errors = new();//[testResultSource.ErrorMessage]; + if (testResultSource == null) + return new RawTestResult(); + List errors = new();//[testResultSource.ErrorMessage]; if (testResultSource.ErrorMessage != null) errors.Add(new MPTError() { message = testResultSource.ErrorMessage }); var rawTestResult = new RawTestResult diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs index 01c7e2667fc9..db8142ca2a40 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -30,14 +30,14 @@ internal class TestProcessor : ITestProcessor private readonly CloudRunMetadata _cloudRunMetadata; // Test Metadata - private int TotalTestCount { get; set; } = 0; - private int PassedTestCount { get; set; } = 0; - private int FailedTestCount { get; set; } = 0; - private int SkippedTestCount { get; set; } = 0; - private List TestResults { get; set; } = new List(); - private ConcurrentDictionary RawTestResultsMap { get; set; } = new(); - private bool FatalTestExecution { get; set; } = false; - private TestRunShardDto? _testRunShard; + internal int TotalTestCount { get; set; } = 0; + internal int PassedTestCount { get; set; } = 0; + internal int FailedTestCount { get; set; } = 0; + internal int SkippedTestCount { get; set; } = 0; + internal List TestResults { get; set; } = new List(); + internal ConcurrentDictionary RawTestResultsMap { get; set; } = new(); + internal bool FatalTestExecution { get; set; } = false; + internal TestRunShardDto? _testRunShard; public TestProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? logger = null, IDataProcessor? dataProcessor = null, ICloudRunErrorParser? cloudRunErrorParser = null, IServiceClient? serviceClient = null, IConsoleWriter? consoleWriter = null) { @@ -117,9 +117,6 @@ public void TestCaseResultHandler(object? sender, TestResultEventArgs e) { SkippedTestCount++; } - } - if (testResult != null) - { TestResults.Add(testResult); } } @@ -251,15 +248,6 @@ private TestRunShardDto GetTestRunEndShard(TestRunCompleteEventArgs e) testRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ"); testRunShard.Summary.TotalTime = durationInMs; testRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 }; - //testRunShard.Summary = new TestRunResultsSummary - //{ - // NumTotalTests = TotalTestCount, - // NumPassedTests = PassedTestCount, - // NumFailedTests = FailedTestCount, - // NumSkippedTests = SkippedTestCount, - // NumFlakyTests = 0, // TODO: Implement flaky tests - // Status = result - //}; testRunShard.UploadCompleted = true; return testRunShard; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs index 62930afb0fff..4899545fd75c 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs @@ -14,6 +14,7 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests; [TestFixture] +[Parallelizable(ParallelScope.Self)] public class EntraLifecycleTests { private static string GetToken(Dictionary claims, DateTime? expires = null) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs new file mode 100644 index 000000000000..f9235f972283 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Text.RegularExpressions; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Microsoft.Extensions.FileSystemGlobbing.Internal; +using Moq; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Implementation +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class CloudRunErrorParserTests + { + private Mock? _loggerMock; + private Mock? _consoleWriterMock; + private CloudRunErrorParser? _errorParser; + + [SetUp] + public void Setup() + { + _loggerMock = new Mock(); + _consoleWriterMock = new Mock(); + _errorParser = new CloudRunErrorParser(_loggerMock.Object, _consoleWriterMock.Object); + } + + [Test] + public void TryPushMessageAndKey_WithValidMessageAndKey_ReturnsTrue() + { + string message = "Test message"; + string key = "Test key"; + + bool result = _errorParser!.TryPushMessageAndKey(message, key); + + Assert.IsTrue(result); + } + + [Test] + public void TryPushMessageAndKey_WithNullOrEmptyMessage_ReturnsFalse() + { + string? message = null; + string key = "Test key"; + + bool result = _errorParser!.TryPushMessageAndKey(message, key); + + Assert.IsFalse(result); + } + + [Test] + public void TryPushMessageAndKey_WithNullOrEmptyKey_ReturnsFalse() + { + string message = "Test message"; + string? key = null; + + bool result = _errorParser!.TryPushMessageAndKey(message, key); + + Assert.IsFalse(result); + } + + [Test] + public void TryPushMessageAndKey_WithExistingKey_ReturnsFalse() + { + string message = "Test message"; + string key = "Existing key"; + _errorParser!.TryPushMessageAndKey(message, key); + + bool result = _errorParser.TryPushMessageAndKey(message, key); + + Assert.IsFalse(result); + } + + [Test] + public void PushMessage_AddsMessageToList() + { + string message = "Test message"; + + _errorParser!.PushMessage(message); + + CollectionAssert.Contains(_errorParser!.InformationalMessages, message); + } + + [Test] + public void DisplayMessages_WithMessages_WritesMessagesToConsole() + { + _errorParser!.PushMessage("Message 1"); + _errorParser.PushMessage("Message 2"); + + _errorParser.DisplayMessages(); + + _consoleWriterMock!.Verify(cw => cw.WriteLine(null), Times.Once); + _consoleWriterMock.Verify(cw => cw.WriteLine("1) Message 1"), Times.Once); + _consoleWriterMock.Verify(cw => cw.WriteLine("2) Message 2"), Times.Once); + } + + [Test] + public void DisplayMessages_WithoutMessages_DoesNotWriteToConsole() + { + _errorParser!.DisplayMessages(); + + _consoleWriterMock!.Verify(cw => cw.WriteLine(null), Times.Never); + _consoleWriterMock.Verify(cw => cw.WriteLine(It.IsAny()), Times.Never); + } + + [Test] + public void PrintErrorToConsole_WritesErrorMessageToConsole() + { + string errorMessage = "Test error message"; + + _errorParser!.PrintErrorToConsole(errorMessage); + + _consoleWriterMock!.Verify(cw => cw.WriteError(errorMessage), Times.Once); + } + + [Test] + public void HandleScalableRunErrorMessage_WithNullMessage_DoesNotPushMessage() + { + _errorParser!.HandleScalableRunErrorMessage(null); + + Assert.IsEmpty(_errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessage_WithoutMatchingPattern_DoesNotPushMessage() + { + string message = "Unknown error"; + + _errorParser!.HandleScalableRunErrorMessage(message); + + Assert.IsEmpty(_errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessage401_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 401 Unauthorized"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The authentication token provided is invalid. Please check the token and try again."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageNoPermissionOnWorkspaceScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nCheckAccess API call with non successful response."; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = @"You do not have the required permissions to run tests. This could be because: + + a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator. + b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant '."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageInvalidWorkspaceScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nInvalidAccountOrSubscriptionState"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The specified workspace does not exist. Please verify your workspace settings."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageInvalidAccessToken_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nInvalidAccessToken"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageAccessTokenOrUserOrWorkspaceNotFoundScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 404 Not Found\r\nNotFound"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageAccessKeyBasedAuthNotSupportedScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nAccessKeyBasedAuthNotSupported"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageServiceUnavailableScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers 503 Service Unavailable"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The service is currently unavailable. Please check the service status and try again."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageGatewayTimeoutScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers 504 Gateway Timeout"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The request to the service timed out. Please try again later."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageQuotaLimitErrorScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = "Timeout 60000s exceeded,\r\nws connecting wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageBrowserConnectionErrorScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = "Target page, context or browser has been closed"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The service is currently unavailable. Please try again after some time."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs new file mode 100644 index 000000000000..30f34150a1f5 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Text.Json; +using Azure.Core; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Moq; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Implementation +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class ServiceClientTests + { + private ServiceClient? _serviceClient; + private Mock? _mockTestReportingClient; + private Mock? _mockCloudRunErrorParser; + private Mock? _mockLogger; + private CloudRunMetadata? _cloudRunMetadata; + + [SetUp] + public void Setup() + { + _mockTestReportingClient = new Mock(); + _mockCloudRunErrorParser = new Mock(); + _mockLogger = new Mock(); + _cloudRunMetadata = new CloudRunMetadata + { + BaseUri = new Uri("https://example.com"), + WorkspaceId = "workspaceId", + RunId = "runId" + }; + + _serviceClient = new ServiceClient(_cloudRunMetadata, _mockCloudRunErrorParser.Object, _mockTestReportingClient.Object, _mockLogger.Object); + } + + [Test] + public void PatchTestRunInfo_ReturnsTestRunDto() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(200); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunDto? result = _serviceClient!.PatchTestRunInfo(run); + + Assert.IsNotNull(result); + } + + [Test] + public void PatchTestRunInfo_On409Conflict_Throws() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(409); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + Assert.Throws(() => _serviceClient!.PatchTestRunInfo(run)); + + _mockCloudRunErrorParser!.Verify(x => x.PrintErrorToConsole(It.IsAny()), Times.Once); + _mockCloudRunErrorParser.Verify(x => x.TryPushMessageAndKey(It.IsAny(), ReporterConstants.s_cONFLICT_409_ERROR_MESSAGE_KEY), Times.Once); + } + + [Test] + public void PatchTestRunInfo_On403Forbidden_Throws() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(403); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + Assert.Throws(() => _serviceClient!.PatchTestRunInfo(run)); + + _mockCloudRunErrorParser!.Verify(x => x.PrintErrorToConsole(It.IsAny()), Times.Once); + _mockCloudRunErrorParser.Verify(x => x.TryPushMessageAndKey(It.IsAny(), ReporterConstants.s_fORBIDDEN_403_ERROR_MESSAGE_KEY), Times.Once); + } + + [Test] + public void PatchTestRunInfo_OnAPIError_ReturnsNull() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(401); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + TestRunDto? result = _serviceClient!.PatchTestRunInfo(run); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void PatchTestRunInfo_OnSuccessButNot200_ReturnsNull() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(201); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunDto? result = _serviceClient!.PatchTestRunInfo(run); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + + [Test] + public void PostTestRunShardInfo_ReturnsTestRunShardDto() + { + var shard = new TestRunShardDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(shard)); + + responseMock.SetupGet(r => r.Status).Returns(200); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PostTestRunShardInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunShardDto? result = _serviceClient!.PostTestRunShardInfo(shard); + + Assert.IsNotNull(result); + } + + [Test] + public void PostTestRunShardInfo_OnAPIError_ReturnsNull() + { + var shard = new TestRunShardDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(shard)); + + responseMock.SetupGet(r => r.Status).Returns(401); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PostTestRunShardInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + TestRunShardDto? result = _serviceClient!.PostTestRunShardInfo(shard); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void PostTestRunShardInfo_OnSuccessButNot200_ReturnsNull() + { + var shard = new TestRunShardDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(shard)); + + responseMock.SetupGet(r => r.Status).Returns(201); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PostTestRunShardInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunShardDto? result = _serviceClient!.PostTestRunShardInfo(shard); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + + [Test] + public void GetTestRunResultsUri_ReturnsTestResultsUri() + { + var testResultsUri = new TestResultsUri(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(testResultsUri)); + + responseMock.SetupGet(r => r.Status).Returns(200); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.GetTestRunResultsUri( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestResultsUri? result = _serviceClient!.GetTestRunResultsUri(); + + Assert.IsNotNull(result); + } + + [Test] + public void GetTestRunResultsUri_OnAPIError_ReturnsNull() + { + var testResultsUri = new TestResultsUri(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(testResultsUri)); + + responseMock.SetupGet(r => r.Status).Returns(401); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.GetTestRunResultsUri( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + TestResultsUri? result = _serviceClient!.GetTestRunResultsUri(); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void GetTestRunResultsUri_OnSuccessButNot200_ReturnsNull() + { + var testResultsUri = new TestResultsUri(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(testResultsUri)); + + responseMock.SetupGet(r => r.Status).Returns(201); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.GetTestRunResultsUri( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestResultsUri? result = _serviceClient!.GetTestRunResultsUri(); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + + [Test] + public void UploadBatchTestResults_Returns() + { + var responseMock = new Mock(); + + responseMock.SetupGet(r => r.Status).Returns(200); + + _mockTestReportingClient!.Setup(x => x.UploadBatchTestResults( + _cloudRunMetadata!.WorkspaceId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + _serviceClient!.UploadBatchTestResults(new UploadTestResultsRequest()); + } + + [Test] + public void UploadBatchTestResults_OnAPIError_Returns() + { + var responseMock = new Mock(); + + responseMock.SetupGet(r => r.Status).Returns(401); + + _mockTestReportingClient!.Setup(x => x.UploadBatchTestResults( + _cloudRunMetadata!.WorkspaceId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + _serviceClient!.UploadBatchTestResults(new UploadTestResultsRequest()); + + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void UploadBatchTestResults_OnSuccessButNot200_Returns() + { + var responseMock = new Mock(); + + responseMock.SetupGet(r => r.Status).Returns(201); + + _mockTestReportingClient!.Setup(x => x.UploadBatchTestResults( + _cloudRunMetadata!.WorkspaceId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + _serviceClient!.UploadBatchTestResults(new UploadTestResultsRequest()); + + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs new file mode 100644 index 000000000000..dfd7b5481853 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Model +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class CloudRunMetadataTests + { + [Test] + public void TestPortalUrl() + { + var metadata = new CloudRunMetadata + { + WorkspaceId = "eastus_2e8c076a-b67c-4984-b861-8d22d7b525c6", + RunId = "#run456^1" + }; + + string portalUrl = metadata.PortalUrl!; + + string expectedPortalUrl = "https://playwright.microsoft.com/workspaces/eastus_2e8c076a-b67c-4984-b861-8d22d7b525c6/runs/%23run456%5E1"; + Assert.AreEqual(expectedPortalUrl, portalUrl); + } + + [Test] + public void TestEnableResultPublish() + { + var metadata = new CloudRunMetadata(); + + bool enableResultPublish = metadata.EnableResultPublish; + + Assert.IsTrue(enableResultPublish); + } + + [Test] + public void TestEnableGithubSummary() + { + var metadata = new CloudRunMetadata(); + + bool enableGithubSummary = metadata.EnableGithubSummary; + + Assert.IsTrue(enableGithubSummary); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs new file mode 100644 index 000000000000..30cb8f3297fe --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Model +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class MPTResultTests + { + [Test] + public void RawTestResult_Errors_Initialized() + { + var rawTestResult = new RawTestResult(); + Assert.AreEqual("[]", rawTestResult.errors); + Assert.AreEqual("[]", rawTestResult.stdOut); + Assert.AreEqual("[]", rawTestResult.stdErr); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs index 8e1ab574e980..d725bea4df98 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs @@ -16,7 +16,6 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests; [TestFixture] -[Parallelizable(ParallelScope.Self)] public class PlaywrightServiceTests { private static string GetToken(Dictionary claims, DateTime? expires = null) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs new file mode 100644 index 000000000000..42b423e35767 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Processor +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class DataProcessorTests + { + [Test] + public void GetTestRun_ReturnsTestRunDto() + { + var cloudRunMetadata = new CloudRunMetadata + { + WorkspaceId = "workspaceId", + RunId = "runId", + AccessTokenDetails = new() + { + oid = "oid", + userName = " userName " + } + }; + var cIInfo = new CIInfo + { + Branch = "branch_name", + Author = "author", + CommitId = "commitId", + RevisionUrl = "revisionUrl", + Provider = CIConstants.s_gITHUB_ACTIONS + }; + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + + TestRunDto result = dataProcessor.GetTestRun(); + + Assert.IsNotNull(result); + Assert.IsInstanceOf(result); + + Assert.AreEqual(cloudRunMetadata.RunId, result.TestRunId); + Assert.IsNotNull(result.DisplayName); + Assert.IsNotNull(result.StartTime); + Assert.AreEqual(cloudRunMetadata.AccessTokenDetails.oid, result.CreatorId); + Assert.AreEqual("userName", result.CreatorName); + Assert.IsTrue(result.CloudReportingEnabled); + Assert.IsFalse(result.CloudRunEnabled); + Assert.IsNotNull(result.CiConfig); + Assert.AreEqual(cIInfo.Branch, result.CiConfig!.Branch); + Assert.AreEqual(cIInfo.Author, result.CiConfig!.Author); + Assert.AreEqual(cIInfo.CommitId, result.CiConfig!.CommitId); + Assert.AreEqual(cIInfo.RevisionUrl, result.CiConfig!.RevisionUrl); + Assert.AreEqual(cIInfo.Provider, result.CiConfig!.CiProviderName); + Assert.IsNotNull(result.TestRunConfig); + Assert.AreEqual(1, result.TestRunConfig!.Workers); + Assert.AreEqual("1.40", result.TestRunConfig!.PwVersion); + Assert.AreEqual(60000, result.TestRunConfig!.Timeout); + Assert.AreEqual("WebTest", result.TestRunConfig!.TestType); + Assert.AreEqual("CSHARP", result.TestRunConfig!.TestSdkLanguage); + Assert.IsNotNull(result.TestRunConfig!.TestFramework); + Assert.AreEqual("PLAYWRIGHT", result.TestRunConfig!.TestFramework!.Name); + Assert.AreEqual("NUNIT", result.TestRunConfig!.TestFramework!.RunnerName); + Assert.AreEqual("3.1", result.TestRunConfig!.TestFramework!.Version); + Assert.AreEqual("1.0.0-beta.1", result.TestRunConfig!.ReporterPackageVersion); + Assert.IsNotNull(result.TestRunConfig!.Shards); + Assert.AreEqual(1, result.TestRunConfig!.Shards!.Total); + } + + [Test] + public void GetTestRunShard_ReturnsTestRunShardDto() + { + var cloudRunMetadata = new CloudRunMetadata(); + var cIInfo = new CIInfo(); + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + + TestRunShardDto result = dataProcessor.GetTestRunShard(); + + Assert.IsNotNull(result); + Assert.IsInstanceOf(result); + Assert.IsFalse(result.UploadCompleted); + Assert.AreEqual("1", result.ShardId); + Assert.IsNotNull(result.Summary); + Assert.AreEqual("RUNNING", result.Summary!.Status); + Assert.IsNotNull(result.Summary!.StartTime); + Assert.AreEqual(1, result.Workers); + } + + [Test] + public void GetTestCaseResultData_WithNullTestResult_ReturnsEmptyTestResults() + { + var cloudRunMetadata = new CloudRunMetadata(); + var cIInfo = new CIInfo(); + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + TestResult? testResult = null; + + TestResults result = dataProcessor.GetTestCaseResultData(testResult); + + Assert.IsNotNull(result); + } + + [Test] + public void GetTestCaseResultData_WithNonNullTestResult_ReturnsTestResults() + { + var cloudRunMetadata = new CloudRunMetadata + { + WorkspaceId = "workspaceId", + RunId = "runId", + AccessTokenDetails = new() + { + oid = "oid", + userName = " userName " + } + }; + var cIInfo = new CIInfo + { + Branch = "branch_name", + Author = "author", + CommitId = "commitId", + RevisionUrl = "revisionUrl", + Provider = CIConstants.s_gITHUB_ACTIONS, + JobId = "jobId" + }; + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + var testResult = new TestResult(new TestCase("Test.Reporting", new System.Uri("file:///test.cs"), "TestNamespace.TestClass")); + + TestResults result = dataProcessor.GetTestCaseResultData(testResult); + + Assert.IsNotNull(result); + Assert.IsEmpty(result.ArtifactsPath); + Assert.AreEqual(cloudRunMetadata.WorkspaceId, result.AccountId); + Assert.AreEqual(cloudRunMetadata.RunId, result.RunId); + Assert.IsNotNull(result.TestExecutionId); + Assert.IsNotNull(result.TestCombinationId); + Assert.IsNotNull(result.TestId); + Assert.AreEqual(testResult.TestCase.DisplayName, result.TestTitle); + Assert.AreEqual("Test", result.SuiteTitle); + Assert.AreEqual("Test", result.SuiteId); + Assert.AreEqual("TestNamespace.TestClass", result.FileName); + Assert.AreEqual(testResult.TestCase.LineNumber, result.LineNumber); + Assert.AreEqual(0, result.Retry); + Assert.IsNotNull(result.WebTestConfig); + Assert.AreEqual(cIInfo.JobId, result.WebTestConfig!.JobName); + Assert.AreEqual(ReporterUtils.GetCurrentOS(), result.WebTestConfig.Os); + Assert.IsNotNull(result.ResultsSummary); + Assert.AreEqual((long)testResult.Duration.TotalMilliseconds, result.ResultsSummary!.Duration); + Assert.AreEqual(testResult.StartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"), result.ResultsSummary.StartTime); + Assert.AreEqual(TestCaseResultStatus.s_iNCONCLUSIVE, result.ResultsSummary.Status); + Assert.AreEqual(TestCaseResultStatus.s_iNCONCLUSIVE, result.Status); + } + + [Test] + public void GetRawResultObject_WithNullTestResult_ReturnsRawTestResultWithEmptyErrorsAndStdErr() + { + RawTestResult result = DataProcessor.GetRawResultObject(null); + + Assert.IsNotNull(result); + Assert.AreEqual("[]", result.errors); + Assert.AreEqual("[]", result.stdErr); + } + + [Test] + public void GetRawResultObject_WithNonNullTestResult_ReturnsRawTestResultWithErrorsAndStdErr() + { + var testResult = new TestResult(new TestCase("Test", new System.Uri("file:///test.cs"), "TestNamespace.TestClass")) + { + ErrorMessage = "An error occurred", + ErrorStackTrace = "Error stack trace" + }; + + RawTestResult result = DataProcessor.GetRawResultObject(testResult); + + Assert.IsNotNull(result); + Assert.AreEqual("[{\"message\":\"An error occurred\"}]", result.errors); + Assert.AreEqual("Error stack trace", result.stdErr); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs new file mode 100644 index 000000000000..2ebe866cc5f2 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs @@ -0,0 +1,584 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using Moq; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Processor +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class TestProcessorTests + { + private CIInfo _cIInfo = new(); + private CloudRunMetadata _cloudRunMetadata = new(); + + [SetUp] + public void Setup() + { + _cloudRunMetadata = new CloudRunMetadata + { + WorkspaceId = "workspaceId", + RunId = "runId", + AccessTokenDetails = new() + { + oid = "oid", + userName = " userName " + }, + EnableGithubSummary = false + }; + _cIInfo = new CIInfo + { + Branch = "branch_name", + Author = "author", + CommitId = "commitId", + RevisionUrl = "revisionUrl", + Provider = CIConstants.s_gITHUB_ACTIONS + }; + } + + [Test] + public void TestRunStartHandler_CreatesTestRunAndShardInfo() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testRunShardDto = new TestRunShardDto(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns(new TestRunDto()); + serviceClientMock.Setup(sc => sc.PostTestRunShardInfo(It.IsAny())).Returns(testRunShardDto); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.AreEqual(testRunShardDto, testProcessor._testRunShard); + Assert.IsFalse(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PatchTestRunReturnsNull_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns((TestRunDto?)null); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Never); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PatchTestRunThrowsError_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Throws(new System.Exception()); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Never); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PostTestRunShardReturnsNull_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns(new TestRunDto()); + serviceClientMock.Setup(sc => sc.PostTestRunShardInfo(It.IsAny())).Returns((TestRunShardDto?)null); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PostTestRunShardThrowsError_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns(new TestRunDto()); + serviceClientMock.Setup(sc => sc.PostTestRunShardInfo(It.IsAny())).Throws(new System.Exception()); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_EnableResultPublishIsFalse_ShouldBeNoOp() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + _cloudRunMetadata.EnableResultPublish = false; + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Never); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Never); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Never); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Never); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsFalse(testProcessor.FatalTestExecution); + } + + [Test] + public void TestCaseResultHandler_TestPassed_AddsTestResultToTestResultsList() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_pASSED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(1, testProcessor.TestResults.Count); + Assert.AreEqual(testResults, testProcessor.TestResults[0]); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 1); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_TestFailed_AddsTestResultToTestResultsList() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_fAILED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(1, testProcessor.TestResults.Count); + Assert.AreEqual(testResults, testProcessor.TestResults[0]); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 1); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_TestSkipped_AddsTestResultToTestResultsList() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(1, testProcessor.TestResults.Count); + Assert.AreEqual(testResults, testProcessor.TestResults[0]); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 1); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_ShouldPassErrorMessageAndStackTraceForScalableErrorParsing() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult) + { + Result = + { + ErrorMessage = "Error message", + ErrorStackTrace = "Error stack trace" + } + }); + + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage("Error message"), Times.Once); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage("Error stack trace"), Times.Once); + } + + [Test] + public void TestCaseResultHandler_EnableResultPublishFalse_OnlyParseScalableErrorMessage() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + _cloudRunMetadata.EnableResultPublish = false; + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(0, testProcessor.TestResults.Count); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_ExceptionThrown_ShouldBeNoOp() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Throws(new System.Exception()); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(0, testProcessor.TestResults.Count); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 0); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Never); + } + + [Test] + public void TestRunCompleteHandler_UploadsTestResults() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + var testResults = new List + { + new() { Status = TestCaseResultStatus.s_pASSED }, + new() { Status = TestCaseResultStatus.s_fAILED }, + new() { Status = TestCaseResultStatus.s_sKIPPED } + }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + } + + [Test] + public void TestRunCompleteHandler_UploadsTestResultsThrows_IgnoresException() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(c => c.UploadBatchTestResults(It.IsAny())).Throws(new System.Exception()); + + var testResults = new List + { + new() { Status = TestCaseResultStatus.s_pASSED }, + new() { Status = TestCaseResultStatus.s_fAILED }, + new() { Status = TestCaseResultStatus.s_sKIPPED } + }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + } + + [Test] + public void TestRunCompleteHandler_PatchesTestRunShardInfo() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + testProcessor._testRunShard = new TestRunShardDto(); + testProcessor.TotalTestCount = 100; + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.AreEqual("CLIENT_COMPLETE", testProcessor._testRunShard.Summary!.Status); + Assert.IsNotNull(testProcessor._testRunShard.Summary!.EndTime); + Assert.IsNotNull(testProcessor._testRunShard.Summary!.TotalTime); + Assert.AreEqual(100, testProcessor._testRunShard.Summary!.UploadMetadata!.NumTestResults); + Assert.AreEqual(0, testProcessor._testRunShard.Summary!.UploadMetadata!.NumTotalAttachments); + Assert.AreEqual(0, testProcessor._testRunShard.Summary!.UploadMetadata!.SizeTotalAttachments); + Assert.IsTrue(testProcessor._testRunShard.UploadCompleted); + } + + [Test] + public void TestRunCompleteHandler_PatchesTestRunShardInfoThrows_DisplaysInformationMessagesAndPortalUrl() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(c => c.PostTestRunShardInfo(It.IsAny())).Throws(new System.Exception()); + + testProcessor._testRunShard = new TestRunShardDto(); + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + consoleWriterMock.Verify(c => c.WriteLine("\nTest Report: " + _cloudRunMetadata.PortalUrl), Times.Once); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + + [Test] + public void TestRunCompleteHandler_DisplaysTestRunUrl() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + testProcessor._testRunShard = new TestRunShardDto(); + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + consoleWriterMock.Verify(c => c.WriteLine("\nTest Report: " + _cloudRunMetadata.PortalUrl), Times.Once); + } + + [Test] + public void TestRunCompleteHandler_DisplaysMessagesOnEnd() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + testProcessor._testRunShard = new TestRunShardDto(); + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + consoleWriterMock.Verify(c => c.WriteLine("\nTest Report: " + _cloudRunMetadata.PortalUrl), Times.Once); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + + [Test] + public void TestRunCompleteHandler_FatalExecutionSetToTrue_DisplaysMessagesOnEnd() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + testProcessor.FatalTestExecution = true; + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Never); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + + [Test] + public void TestRunCompleteHandler_EnableResultPublishSetToFalse_DisplaysMessagesOnEnd() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + _cloudRunMetadata.EnableResultPublish = false; + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Never); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs deleted file mode 100644 index 1f4c5c23e4c2..000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/ReporterUtilsTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests; - -[TestFixture] -[Parallelizable(ParallelScope.Self)] -public class ReporterUtilsTests -{ - [Test] - public void IsTimeGreaterThanCurrentPlus10Minutes_ValidFutureSasUri_ReturnsTrue() - { - var reporterUtils = new ReporterUtils(); - string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(15).ToString("o"); // 15 minutes in the future - bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); - Assert.IsTrue(result); - } - - [Test] - public void IsTimeGreaterThanCurrentPlus10Minutes_ExpiredSasUri_ReturnsFalse() - { - var reporterUtils = new ReporterUtils(); - string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(-5).ToString("o"); // 5 minutes in the past - bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); - Assert.IsFalse(result); - } - - [Test] - public void IsTimeGreaterThanCurrentPlus10Minutes_InvalidSasUri_ReturnsFalse() - { - var reporterUtils = new ReporterUtils(); - string sasUri = "not_a_valid_sas_uri"; // Invalid SAS URI - bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); - Assert.IsFalse(result); - } -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs new file mode 100644 index 000000000000..5c18556d0b39 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs @@ -0,0 +1,257 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Utility +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class CiInfoProviderTests + { + private Dictionary _originalEnvironmentVariables = new(); + + [OneTimeSetUp] + public void Init() + { + _originalEnvironmentVariables = Environment.GetEnvironmentVariables() + .Cast() + .ToDictionary(entry => (string)entry.Key, entry => (string)entry.Value!)!; + } + + [SetUp] + public void Setup() + { + foreach (KeyValuePair kvp in _originalEnvironmentVariables) + { + Environment.SetEnvironmentVariable(kvp.Key, kvp.Value); + } + + var keysToRemove = Environment.GetEnvironmentVariables() + .Cast() + .Select(entry => (string)entry.Key) + .Where(key => key.StartsWith("github", StringComparison.OrdinalIgnoreCase) || key.StartsWith("gh", StringComparison.OrdinalIgnoreCase) || key.StartsWith("ado", StringComparison.OrdinalIgnoreCase) || key.StartsWith("azure", StringComparison.OrdinalIgnoreCase) || key.StartsWith("tf", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + foreach (var key in keysToRemove) + { + Environment.SetEnvironmentVariable(key, null); // Remove environment variable + } + } + + [Test] + public void GetCIProvider_GitHubActions_ReturnsGitHubActions() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + + string ciProvider = CiInfoProvider.GetCIProvider(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciProvider); + } + + [Test] + public void GetCIProvider_AzureDevOps_ReturnsAzureDevOps() + { + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", "some_value"); + Environment.SetEnvironmentVariable("TF_BUILD", "some_value"); + + string ciProvider = CiInfoProvider.GetCIProvider(); + + Assert.AreEqual(CIConstants.s_aZURE_DEVOPS, ciProvider); + } + + [Test] + public void GetCIProvider_Default_ReturnsDefault() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", null); + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", null); + Environment.SetEnvironmentVariable("TF_BUILD", null); + + string ciProvider = CiInfoProvider.GetCIProvider(); + + Assert.AreEqual(CIConstants.s_dEFAULT, ciProvider); + } + [Test] + public void GetCIInfo_GitHubActions_ReturnsGitHubActionsCIInfo() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("GITHUB_ACTOR", "actor"); + Environment.SetEnvironmentVariable("GITHUB_SHA", "commit_sha"); + Environment.SetEnvironmentVariable("GITHUB_SERVER_URL", "server_url"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "repository"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("GITHUB_JOB", "job_id"); + Environment.SetEnvironmentVariable("GITHUB_REF_NAME", "refs/heads/branch_name"); + Environment.SetEnvironmentVariable("GITHUB_HEAD_REF", "refs/heads/head_branch_name"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("refs/heads/branch_name", ciInfo.Branch); + Assert.AreEqual("actor", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("server_url/repository/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_GitHubActions_ReturnsGitHubActionsCIInfo_WithRefPrefixWhenEventNameIsPullRequest() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("GITHUB_ACTOR", "actor"); + Environment.SetEnvironmentVariable("GITHUB_SHA", "commit_sha"); + Environment.SetEnvironmentVariable("GITHUB_SERVER_URL", "server_url"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "repository"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("GITHUB_JOB", "job_id"); + Environment.SetEnvironmentVariable("GITHUB_REF", "refs/heads/feature/branch_name"); + Environment.SetEnvironmentVariable("GITHUB_HEAD_REF", "refs/heads/head_branch_name"); + Environment.SetEnvironmentVariable("GITHUB_EVENT_NAME", "pull_request"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("refs/heads/head_branch_name", ciInfo.Branch); + Assert.AreEqual("actor", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("server_url/repository/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_GitHubActions_ReturnsGitHubActionsCIInfo_WithRefPrefixWhenEventNameIsPullRequestTarget() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("GITHUB_ACTOR", "actor"); + Environment.SetEnvironmentVariable("GITHUB_SHA", "commit_sha"); + Environment.SetEnvironmentVariable("GITHUB_SERVER_URL", "server_url"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "repository"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("GITHUB_JOB", "job_id"); + Environment.SetEnvironmentVariable("GITHUB_REF", "refs/heads/feature/branch_name"); + Environment.SetEnvironmentVariable("GITHUB_HEAD_REF", "refs/heads/head_branch_name"); + Environment.SetEnvironmentVariable("GITHUB_EVENT_NAME", "pull_request_target"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("refs/heads/head_branch_name", ciInfo.Branch); + Assert.AreEqual("actor", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("server_url/repository/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_AzureDevOps_ReturnsAzureDevOpsCIInfo() + { + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", "some_value"); + Environment.SetEnvironmentVariable("TF_BUILD", "some_value"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", "branch_name"); + Environment.SetEnvironmentVariable("BUILD_REQUESTEDFOR", "author"); + Environment.SetEnvironmentVariable("BUILD_SOURCEVERSION", "commit_sha"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", "collection_uri/"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "team_project"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "repository_name"); + Environment.SetEnvironmentVariable("RELEASE_ATTEMPTNUMBER", "1"); + Environment.SetEnvironmentVariable("SYSTEM_JOBATTEMPT", "2"); + Environment.SetEnvironmentVariable("RELEASE_DEPLOYMENTID", "deployment_id"); + Environment.SetEnvironmentVariable("SYSTEM_DEFINITIONID", "definition_id"); + Environment.SetEnvironmentVariable("SYSTEM_JOBID", "job_id"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_aZURE_DEVOPS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("branch_name", ciInfo.Branch); + Assert.AreEqual("author", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("collection_uri/team_project/_git/repository_name/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("definition_id-job_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("deployment_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_AzureDevOpsWithReleaseInfo_ReturnsAzureDevOpsCIInfo() + { + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", "some_value"); + Environment.SetEnvironmentVariable("TF_BUILD", "some_value"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", "branch_name"); + Environment.SetEnvironmentVariable("BUILD_REQUESTEDFOR", "author"); + Environment.SetEnvironmentVariable("BUILD_SOURCEVERSION", "commit_sha"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", "collection_uri/"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "team_project"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "repository_name"); + Environment.SetEnvironmentVariable("RELEASE_ATTEMPTNUMBER", "1"); + Environment.SetEnvironmentVariable("SYSTEM_JOBATTEMPT", "2"); + Environment.SetEnvironmentVariable("RELEASE_DEPLOYMENTID", "deployment_id"); + Environment.SetEnvironmentVariable("SYSTEM_DEFINITIONID", "definition_id"); + Environment.SetEnvironmentVariable("SYSTEM_JOBID", "job_id"); + Environment.SetEnvironmentVariable("RELEASE_DEFINITIONID", "release-def"); + Environment.SetEnvironmentVariable("RELEASE_DEPLOYMENTID", "release-dep"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_aZURE_DEVOPS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("branch_name", ciInfo.Branch); + Assert.AreEqual("author", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("collection_uri/team_project/_git/repository_name/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("release-def-release-dep", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("release-dep", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_Default_ReturnsDefaultCIInfo() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", null); + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", null); + Environment.SetEnvironmentVariable("TF_BUILD", null); + Environment.SetEnvironmentVariable("REPO", "repo"); + Environment.SetEnvironmentVariable("BRANCH", "branch"); + Environment.SetEnvironmentVariable("AUTHOR", "author"); + Environment.SetEnvironmentVariable("COMMIT_ID", "commit_sha"); + Environment.SetEnvironmentVariable("REVISION_URL", "revision_url"); + Environment.SetEnvironmentVariable("RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("JOB_ID", "job_id"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_dEFAULT, ciInfo.Provider); + Assert.AreEqual("repo", ciInfo.Repo); + Assert.AreEqual("branch", ciInfo.Branch); + Assert.AreEqual("author", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("revision_url", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs new file mode 100644 index 000000000000..2456c11617d4 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Microsoft.IdentityModel.JsonWebTokens; +using Microsoft.IdentityModel.Tokens; +using Moq; +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Utility; + +[TestFixture] +[Parallelizable(ParallelScope.Self)] +public class ReporterUtilsTests +{ + private static string GetToken(Dictionary claims, DateTime? expires = null) + { + var tokenHandler = new JsonWebTokenHandler(); + var token = tokenHandler.CreateToken(new SecurityTokenDescriptor + { + Claims = claims, + Expires = expires ?? DateTime.UtcNow.AddMinutes(10), + }); + return token!; + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_ValidFutureSasUri_ReturnsTrue() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(15).ToString("o"); // 15 minutes in the future + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsTrue(result); + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_ExpiredSasUri_ReturnsFalse() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(-5).ToString("o"); // 5 minutes in the past + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsFalse(result); + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_InvalidSasUri_ReturnsFalse() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "not_a_valid_sas_uri"; // Invalid SAS URI + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsFalse(result); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_CustomToken_ReturnsTokenDetails() + { + var reporterUtils = new ReporterUtils(); + var accessToken = GetToken(new Dictionary + { + { "aid", "custom_aid" }, + { "oid", "custom_oid" }, + { "id", "custom_id" }, + { "name", "custom_username" }, + }); + + TokenDetails result = reporterUtils.ParseWorkspaceIdFromAccessToken(null, accessToken); + + Assert.AreEqual("custom_aid", result.aid); + Assert.AreEqual("custom_oid", result.oid); + Assert.AreEqual("custom_id", result.id); + Assert.AreEqual("custom_username", result.userName); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_EntraToken_ReturnsTokenDetails() + { + var reporterUtils = new ReporterUtils(); + var jsonWebTokenHandler = new JsonWebTokenHandler(); + var accessToken = GetToken(new Dictionary + { + { "oid", "entra_oid" }, + { "name", "entra_username" }, + }); + + TokenDetails result = reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler, accessToken); + + Assert.AreEqual("entra_oid", result.oid); + Assert.AreEqual(string.Empty, result.id); + Assert.AreEqual("entra_username", result.userName); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_NullToken_ThrowsArgumentNullException() + { + var reporterUtils = new ReporterUtils(); + var jsonWebTokenHandler = new JsonWebTokenHandler(); + string? accessToken = null; + + Assert.Throws(() => reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler, accessToken)); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_EmptyToken_ThrowsArgumentNullException() + { + var reporterUtils = new ReporterUtils(); + var jsonWebTokenHandler = new JsonWebTokenHandler(); + string accessToken = string.Empty; + + Assert.Throws(() => reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler, accessToken)); + } + [Test] + public void GetRunId_DefaultProvider_ReturnsNewGuid() + { + var cIInfo = new CIInfo { Provider = CIConstants.s_dEFAULT }; + var result = ReporterUtils.GetRunId(cIInfo); + Assert.IsNotNull(result); + Assert.IsTrue(Guid.TryParse(result, out _)); + } + + [Test] + public void GetRunId_NonDefaultProvider_ReturnsSha1Hash() + { + var cIInfo = new CIInfo { Provider = "NonDefaultProvider", Repo = "Repo", RunId = "RunId", RunAttempt = 1 }; + var expectedRunIdBeforeHash = $"{cIInfo.Provider}-{cIInfo.Repo}-{cIInfo.RunId}-{cIInfo.RunAttempt}"; + var result = ReporterUtils.GetRunId(cIInfo); + Assert.IsNotNull(result); + Assert.AreEqual(40, result.Length); + Assert.AreEqual(ReporterUtils.CalculateSha1Hash(expectedRunIdBeforeHash), result); + } + + [Test] + public void GetRunName_GitHubActionsPullRequest_ReturnsExpectedValue() + { + var ciInfo = new CIInfo { Provider = CIConstants.s_gITHUB_ACTIONS }; + Environment.SetEnvironmentVariable("GITHUB_EVENT_NAME", "pull_request"); + Environment.SetEnvironmentVariable("GITHUB_REF_NAME", "543/refs/merge"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "owner/repo"); + + var result = ReporterUtils.GetRunName(ciInfo); + + var expected = "PR# 543 on Repo: owner/repo (owner/repo/pull/543)"; + Assert.AreEqual(expected, result); + } +} From b462813d2455e05eacbd1a116451338cfc18feeb Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Sun, 20 Oct 2024 17:53:10 +0530 Subject: [PATCH 17/18] chore(): add new api info --- ...er.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs index 0880153e7dc9..b961eccd0a9b 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs @@ -55,10 +55,10 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client { public partial class TestReportingClientOptions : Azure.Core.ClientOptions { - public TestReportingClientOptions(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion version = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion.V2024_05_20_Preview) { } + public TestReportingClientOptions(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion version = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion.V2024_09_01_Preview) { } public enum ServiceVersion { - V2024_05_20_Preview = 1, + V2024_09_01_Preview = 1, } } } From e96d865e3fd4bd5264bd570b500b98dd3bc8f2a1 Mon Sep 17 00:00:00 2001 From: Siddharth Singha Roy Date: Sun, 20 Oct 2024 19:48:27 +0530 Subject: [PATCH 18/18] fix(): remove parallelization from entra tests --- .../tests/EntraLifecycleTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs index 4899545fd75c..62930afb0fff 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs @@ -14,7 +14,6 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests; [TestFixture] -[Parallelizable(ParallelScope.Self)] public class EntraLifecycleTests { private static string GetToken(Dictionary claims, DateTime? expires = null)