chore(): add tests for reporter

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs

@@ -133,17 +133,23 @@ public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsReq
             return null;
         }
 
-        private void HandleAPIFailure(int? statusCode, string operationName)
+        internal void HandleAPIFailure(int? statusCode, string operationName)
         {
-            if (statusCode == null)
-                return;
-            ApiErrorConstants.s_errorOperationPair.TryGetValue(operationName, out System.Collections.Generic.Dictionary<int, string>? errorObject);
-            if (errorObject == null)
-                return;
-            errorObject.TryGetValue((int)statusCode, out string? errorMessage);
-            if (errorMessage == null)
-                errorMessage = ReporterConstants.s_uNKNOWN_ERROR_MESSAGE;
-            _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, statusCode.ToString());
+            try
+            {
+                if (statusCode == null)
+                    return;
+                ApiErrorConstants.s_errorOperationPair.TryGetValue(operationName, out System.Collections.Generic.Dictionary<int, string>? errorObject);
+                if (errorObject == null)
+                    return;
+                errorObject.TryGetValue((int)statusCode, out string? errorMessage);
+                errorMessage ??= ReporterConstants.s_uNKNOWN_ERROR_MESSAGE;
+                _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, statusCode.ToString());
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex.Message);
+            }
         }
     }
 }

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs

@@ -10,6 +10,6 @@ internal interface IDataProcessor
     {
         TestRunDto GetTestRun();
         TestRunShardDto GetTestRunShard();
-        TestResults GetTestCaseResultData(TestResult testResultSource);
+        TestResults GetTestCaseResultData(TestResult? testResultSource);
     }
 }

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs

@@ -78,7 +78,7 @@ public TestRunShardDto GetTestRunShard()
             };
             return shard;
         }
-        public TestResults GetTestCaseResultData(TestResult testResultSource)
+        public TestResults GetTestCaseResultData(TestResult? testResultSource)
         {
             if (testResultSource == null)
                 return new TestResults();
@@ -139,9 +139,11 @@ public TestResults GetTestCaseResultData(TestResult testResultSource)
             return testCaseResultData;
         }
 
-        public static RawTestResult GetRawResultObject(TestResult testResultSource)
+        public static RawTestResult GetRawResultObject(TestResult? testResultSource)
         {
-            List<MPTError> errors = new();//[testResultSource.ErrorMessage];
+            if (testResultSource == null)
+                return new RawTestResult();
+            List <MPTError> errors = new();//[testResultSource.ErrorMessage];
             if (testResultSource.ErrorMessage != null)
                 errors.Add(new MPTError() { message = testResultSource.ErrorMessage });
             var rawTestResult = new RawTestResult

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs

@@ -30,14 +30,14 @@ internal class TestProcessor : ITestProcessor
         private readonly CloudRunMetadata _cloudRunMetadata;
 
         // Test Metadata
-        private int TotalTestCount { get; set; } = 0;
-        private int PassedTestCount { get; set; } = 0;
-        private int FailedTestCount { get; set; } = 0;
-        private int SkippedTestCount { get; set; } = 0;
-        private List<TestResults> TestResults { get; set; } = new List<TestResults>();
-        private ConcurrentDictionary<string, RawTestResult?> RawTestResultsMap { get; set; } = new();
-        private bool FatalTestExecution { get; set; } = false;
-        private TestRunShardDto? _testRunShard;
+        internal int TotalTestCount { get; set; } = 0;
+        internal int PassedTestCount { get; set; } = 0;
+        internal int FailedTestCount { get; set; } = 0;
+        internal int SkippedTestCount { get; set; } = 0;
+        internal List<TestResults> TestResults { get; set; } = new List<TestResults>();
+        internal ConcurrentDictionary<string, RawTestResult?> RawTestResultsMap { get; set; } = new();
+        internal bool FatalTestExecution { get; set; } = false;
+        internal TestRunShardDto? _testRunShard;
 
         public TestProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? logger = null, IDataProcessor? dataProcessor = null, ICloudRunErrorParser? cloudRunErrorParser = null, IServiceClient? serviceClient = null, IConsoleWriter? consoleWriter = null)
         {
@@ -117,9 +117,6 @@ public void TestCaseResultHandler(object? sender, TestResultEventArgs e)
                     {
                         SkippedTestCount++;
                     }
-                }
-                if (testResult != null)
-                {
                     TestResults.Add(testResult);
                 }
             }
@@ -251,15 +248,6 @@ private TestRunShardDto GetTestRunEndShard(TestRunCompleteEventArgs e)
             testRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ");
             testRunShard.Summary.TotalTime = durationInMs;
             testRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 };
-            //testRunShard.Summary = new TestRunResultsSummary
-            //{
-            //    NumTotalTests = TotalTestCount,
-            //    NumPassedTests = PassedTestCount,
-            //    NumFailedTests = FailedTestCount,
-            //    NumSkippedTests = SkippedTestCount,
-            //    NumFlakyTests = 0, // TODO: Implement flaky tests
-            //    Status = result
-            //};
             testRunShard.UploadCompleted = true;
             return testRunShard;
         }

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/EntraLifecycleTests.cs

@@ -14,6 +14,7 @@
 namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests;
 
 [TestFixture]
+[Parallelizable(ParallelScope.Self)]
 public class EntraLifecycleTests
 {
     private static string GetToken(Dictionary<string, object> claims, DateTime? expires = null)

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs

@@ -0,0 +1,236 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Text.RegularExpressions;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
+using Microsoft.Extensions.FileSystemGlobbing.Internal;
+using Moq;
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Implementation
+{
+    [TestFixture]
+    [Parallelizable(ParallelScope.Self)]
+    public class CloudRunErrorParserTests
+    {
+        private Mock<ILogger>? _loggerMock;
+        private Mock<IConsoleWriter>? _consoleWriterMock;
+        private CloudRunErrorParser? _errorParser;
+
+        [SetUp]
+        public void Setup()
+        {
+            _loggerMock = new Mock<ILogger>();
+            _consoleWriterMock = new Mock<IConsoleWriter>();
+            _errorParser = new CloudRunErrorParser(_loggerMock.Object, _consoleWriterMock.Object);
+        }
+
+        [Test]
+        public void TryPushMessageAndKey_WithValidMessageAndKey_ReturnsTrue()
+        {
+            string message = "Test message";
+            string key = "Test key";
+
+            bool result = _errorParser!.TryPushMessageAndKey(message, key);
+
+            Assert.IsTrue(result);
+        }
+
+        [Test]
+        public void TryPushMessageAndKey_WithNullOrEmptyMessage_ReturnsFalse()
+        {
+            string? message = null;
+            string key = "Test key";
+
+            bool result = _errorParser!.TryPushMessageAndKey(message, key);
+
+            Assert.IsFalse(result);
+        }
+
+        [Test]
+        public void TryPushMessageAndKey_WithNullOrEmptyKey_ReturnsFalse()
+        {
+            string message = "Test message";
+            string? key = null;
+
+            bool result = _errorParser!.TryPushMessageAndKey(message, key);
+
+            Assert.IsFalse(result);
+        }
+
+        [Test]
+        public void TryPushMessageAndKey_WithExistingKey_ReturnsFalse()
+        {
+            string message = "Test message";
+            string key = "Existing key";
+            _errorParser!.TryPushMessageAndKey(message, key);
+
+            bool result = _errorParser.TryPushMessageAndKey(message, key);
+
+            Assert.IsFalse(result);
+        }
+
+        [Test]
+        public void PushMessage_AddsMessageToList()
+        {
+            string message = "Test message";
+
+            _errorParser!.PushMessage(message);
+
+            CollectionAssert.Contains(_errorParser!.InformationalMessages, message);
+        }
+
+        [Test]
+        public void DisplayMessages_WithMessages_WritesMessagesToConsole()
+        {
+            _errorParser!.PushMessage("Message 1");
+            _errorParser.PushMessage("Message 2");
+
+            _errorParser.DisplayMessages();
+
+            _consoleWriterMock!.Verify(cw => cw.WriteLine(null), Times.Once);
+            _consoleWriterMock.Verify(cw => cw.WriteLine("1) Message 1"), Times.Once);
+            _consoleWriterMock.Verify(cw => cw.WriteLine("2) Message 2"), Times.Once);
+        }
+
+        [Test]
+        public void DisplayMessages_WithoutMessages_DoesNotWriteToConsole()
+        {
+            _errorParser!.DisplayMessages();
+
+            _consoleWriterMock!.Verify(cw => cw.WriteLine(null), Times.Never);
+            _consoleWriterMock.Verify(cw => cw.WriteLine(It.IsAny<string>()), Times.Never);
+        }
+
+        [Test]
+        public void PrintErrorToConsole_WritesErrorMessageToConsole()
+        {
+            string errorMessage = "Test error message";
+
+            _errorParser!.PrintErrorToConsole(errorMessage);
+
+            _consoleWriterMock!.Verify(cw => cw.WriteError(errorMessage), Times.Once);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessage_WithNullMessage_DoesNotPushMessage()
+        {
+            _errorParser!.HandleScalableRunErrorMessage(null);
+
+            Assert.IsEmpty(_errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessage_WithoutMatchingPattern_DoesNotPushMessage()
+        {
+            string message = "Unknown error";
+
+            _errorParser!.HandleScalableRunErrorMessage(message);
+
+            Assert.IsEmpty(_errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessage401_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 401 Unauthorized";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The authentication token provided is invalid. Please check the token and try again.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageNoPermissionOnWorkspaceScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nCheckAccess API call with non successful response.";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = @"You do not have the required permissions to run tests. This could be because:
+
+    a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator.
+    b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant <TENANT_ID>'.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageInvalidWorkspaceScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nInvalidAccountOrSubscriptionState";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The specified workspace does not exist. Please verify your workspace settings.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageInvalidAccessToken_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nInvalidAccessToken";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageAccessTokenOrUserOrWorkspaceNotFoundScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 404 Not Found\r\nNotFound";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageAccessKeyBasedAuthNotSupportedScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nAccessKeyBasedAuthNotSupported";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageServiceUnavailableScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers 503 Service Unavailable";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The service is currently unavailable. Please check the service status and try again.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageGatewayTimeoutScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers 504 Gateway Timeout";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The request to the service timed out. Please try again later.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageQuotaLimitErrorScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = "Timeout 60000s exceeded,\r\nws connecting wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+
+        [Test]
+        public void HandleScalableRunErrorMessageBrowserConnectionErrorScalable_WithMatchingPattern_PushesMessage()
+        {
+            string errorMessage = "Target page, context or browser has been closed";
+
+            _errorParser!.HandleScalableRunErrorMessage(errorMessage);
+            var message = "The service is currently unavailable. Please try again after some time.";
+            Assert.Contains(message, _errorParser.InformationalMessages);
+        }
+    }
+}