feat(playwrighttesting): separate api, test data processing and utility layers

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs

@@ -1,6 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
 namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger;
 
 /// <summary>
@@ -183,6 +187,7 @@ internal class Constants
     // Default constants
     internal static readonly string s_default_os = ServiceOs.Linux;
     internal static readonly string s_default_expose_network = "<loopback>";
+    internal static readonly string s_pLAYWRIGHT_SERVICE_DEBUG = "PLAYWRIGHT_SERVICE_DEBUG";
 
     // Entra id access token constants
     internal static readonly int s_entra_access_token_lifetime_left_threshold_in_minutes_for_rotation = 15;
@@ -204,3 +209,196 @@ internal class Constants
     internal static readonly string s_playwright_service_reporting_url_environment_variable = "PLAYWRIGHT_SERVICE_REPORTING_URL";
     internal static readonly string s_playwright_service_workspace_id_environment_variable = "PLAYWRIGHT_SERVICE_WORKSPACE_ID";
 }
+
+internal class OSConstants
+{
+    internal static readonly string s_lINUX = "Linux";
+    internal static readonly string s_wINDOWS = "Windows";
+    internal static readonly string s_mACOS = "MacOS";
+}
+
+internal class ReporterConstants
+{
+    internal static readonly string s_executionIdPropertyIdentifier = "ExecutionId";
+    internal static readonly string s_parentExecutionIdPropertyIdentifier = "ParentExecId";
+    internal static readonly string s_testTypePropertyIdentifier = "TestType";
+    internal static readonly string s_sASUriSeparator = "?";
+    internal static readonly string s_portalBaseUrl = "https://playwright.microsoft.com/workspaces/";
+    internal static readonly string s_reportingRoute = "/runs/";
+    internal static readonly string s_reportingAPIVersion_2024_04_30_preview = "2024-04-30-preview";
+    internal static readonly string s_reportingAPIVersion_2024_05_20_preview = "2024-05-20-preview";
+    internal static readonly string s_pLAYWRIGHT_SERVICE_REPORTING_URL = "PLAYWRIGHT_SERVICE_REPORTING_URL";
+    internal static readonly string s_pLAYWRIGHT_SERVICE_WORKSPACE_ID = "PLAYWRIGHT_SERVICE_WORKSPACE_ID";
+    internal static readonly string s_aPPLICATION_JSON = "application/json";
+    internal static readonly string s_cONFLICT_409_ERROR_MESSAGE = "Test run with id {runId} already exists. Provide a unique run id.";
+    internal static readonly string s_cONFLICT_409_ERROR_MESSAGE_KEY = "DuplicateRunId";
+
+    internal static readonly string s_fORBIDDEN_403_ERROR_MESSAGE = "Reporting is not enabled for your workspace {workspaceId}. Enable the Reporting feature under Feature management settings using the Playwright portal: https://playwright.microsoft.com/workspaces/{workspaceId}/settings/general";
+    internal static readonly string s_fORBIDDEN_403_ERROR_MESSAGE_KEY = "ReportingNotEnabled";
+    internal static readonly string s_uNKNOWN_ERROR_MESSAGE = "Unknown error occured.";
+}
+
+internal class CIConstants
+{
+    internal static readonly string s_gITHUB_ACTIONS = "GitHub Actions";
+    internal static readonly string s_aZURE_DEVOPS = "Azure DevOps";
+    internal static readonly string s_dEFAULT = "Default";
+}
+
+internal class TestCaseResultStatus
+{
+    internal static readonly string s_pASSED = "passed";
+    internal static readonly string s_fAILED = "failed";
+    internal static readonly string s_sKIPPED = "skipped";
+    internal static readonly string s_iNCONCLUSIVE = "inconclusive";
+}
+
+internal class TestResultError
+{
+    internal string? Key { get; set; } = string.Empty;
+    internal string? Message { get; set; } = string.Empty;
+    internal Regex Pattern { get; set; } = new Regex(string.Empty);
+    internal TestErrorType Type { get; set; }
+}
+
+internal enum TestErrorType
+{
+    Scalable
+}
+
+internal class ServiceClientConstants
+{
+    internal static readonly int s_mAX_RETRIES = 3;
+    internal static readonly int s_mAX_RETRY_DELAY_IN_SECONDS = 2000;
+}
+
+internal static class TestResultErrorConstants
+{
+    public static List<TestResultError> ErrorConstants = new()
+    {
+        new TestResultError
+        {
+            Key = "401",
+            Message = "The authentication token provided is invalid. Please check the token and try again.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*401 Unauthorized)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "NoPermissionOnWorkspace_Scalable",
+            Message = @"You do not have the required permissions to run tests. This could be because:
+
+    a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator.
+    b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant <TENANT_ID>'.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*CheckAccess API call with non successful response)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "InvalidWorkspace_Scalable",
+            Message = "The specified workspace does not exist. Please verify your workspace settings.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "InvalidAccessToken",
+            Message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccessToken)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "AccessTokenOrUserOrWorkspaceNotFound_Scalable",
+            Message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*404 Not Found)(?=.*NotFound)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "AccessKeyBasedAuthNotSupported_Scalable",
+            Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "ServiceUnavailable_Scalable",
+            Message = "The service is currently unavailable. Please check the service status and try again.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*503 Service Unavailable)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "GatewayTimeout_Scalable",
+            Message = "The request to the service timed out. Please try again later.",
+            Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*504 Gateway Timeout)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "QuotaLimitError_Scalable",
+            Message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded.",
+            Pattern = new Regex(@"(Timeout .* exceeded)(?=[\s\S]*ws connecting)", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        },
+        new TestResultError
+        {
+            Key = "BrowserConnectionError_Scalable",
+            Message = "The service is currently unavailable. Please try again after some time.",
+            Pattern = new Regex(@"Target page, context or browser has been closed", RegexOptions.IgnoreCase),
+            Type = TestErrorType.Scalable
+        }
+    };
+}
+
+internal static class ApiErrorConstants
+{
+    private static Dictionary<int, string> PatchTestRun { get; set; } = new Dictionary<int, string>() {
+        { 400, "The request made to the server is invalid. Please check the request parameters and try again." },
+        { 401, "The authentication token provided is invalid. Please check the token and try again." },
+        { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." },
+        { 429, "You have exceeded the rate limit for the API. Please wait and try again later." },
+        { 504, "The request to the service timed out. Please try again later." },
+        { 503, "The service is currently unavailable. Please check the service status and try again." }
+    };
+
+    private static Dictionary<int, string> GetTestRun { get; set; } = new Dictionary<int, string>()
+    {
+        { 400, "The request made to the server is invalid. Please check the request parameters and try again." },
+        { 401, "The authentication token provided is invalid. Please check the token and try again." },
+        { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." },
+        { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." },
+        { 429, "You have exceeded the rate limit for the API. Please wait and try again later." },
+        { 504, "The request to the service timed out. Please try again later." },
+        { 503, "The service is currently unavailable. Please check the service status and try again." }
+    };
+    private static Dictionary<int, string> PatchTestRunShard { get; set; } = new Dictionary<int, string>()
+    {
+        { 400, "The request made to the server is invalid. Please check the request parameters and try again." },
+        { 401, "The authentication token provided is invalid. Please check the token and try again." },
+        { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." },
+        { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." },
+        { 429, "You have exceeded the rate limit for the API. Please wait and try again later." },
+        { 504, "The request to the service timed out. Please try again later." },
+        { 503, "The service is currently unavailable. Please check the service status and try again." }
+    };
+    private static Dictionary<int, string> GetStorageUri { get; set; } = new Dictionary<int, string>()
+    {
+        { 400, "The request made to the server is invalid. Please check the request parameters and try again." },
+        { 401, "The authentication token provided is invalid. Please check the token and try again." },
+        { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." },
+        { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." },
+        { 429, "You have exceeded the rate limit for the API. Please wait and try again later." },
+        { 504, "The request to the service timed out. Please try again later." },
+        { 503, "The service is currently unavailable. Please check the service status and try again." }
+    };
+
+    internal static readonly Dictionary<string, Dictionary<int, string>> s_errorOperationPair = new()
+    {
+        { "PatchTestRun", PatchTestRun },
+        { "GetTestRun", GetTestRun },
+        { "PatchTestRunShard", PatchTestRunShard },
+        { "GetStorageUri", GetStorageUri }
+    };
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Collections.Generic;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation
+{
+    internal class CloudRunErrorParser : ICloudRunErrorParser
+    {
+        private List<string> InformationalMessages { get; set; } = new();
+        private List<string> ProcessedErrorMessageKeys { get; set; } = new();
+        private readonly ILogger _logger;
+        private readonly IConsoleWriter _consoleWriter;
+        public CloudRunErrorParser(ILogger? logger = null, IConsoleWriter? consoleWriter = null)
+        {
+            _logger = logger ?? new Logger();
+            _consoleWriter = consoleWriter ?? new ConsoleWriter();
+        }
+
+        public bool TryPushMessageAndKey(string? message, string? key)
+        {
+            if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(message))
+            {
+                return false;
+            }
+            if (ProcessedErrorMessageKeys.Contains(key!))
+            {
+                return false;
+            }
+            _logger.Info($"Adding message with key: {key}");
+
+            ProcessedErrorMessageKeys.Add(key!);
+            InformationalMessages.Add(message!);
+            return true;
+        }
+
+        public void PushMessage(string message)
+        {
+            InformationalMessages.Add(message);
+        }
+
+        public void DisplayMessages()
+        {
+            if (InformationalMessages.Count > 0)
+                _consoleWriter.WriteLine();
+            int index = 1;
+            foreach (string message in InformationalMessages)
+            {
+                _consoleWriter.WriteLine($"{index++}) {message}");
+            }
+        }
+
+        public void PrintErrorToConsole(string message)
+        {
+            _consoleWriter.WriteError(message);
+        }
+
+        public void HandleScalableRunErrorMessage(string? message)
+        {
+            if (string.IsNullOrEmpty(message))
+            {
+                return;
+            }
+            foreach (TestResultError testResultErrorObj in TestResultErrorConstants.ErrorConstants)
+            {
+                if (testResultErrorObj.Pattern.IsMatch(message))
+                {
+                    TryPushMessageAndKey(testResultErrorObj.Message, testResultErrorObj.Key);
+                }
+            }
+        }
+    }
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation
+{
+    internal class ConsoleWriter : IConsoleWriter
+    {
+        public void WriteLine(string? message = null)
+        {
+            if (message == null)
+            {
+                Console.WriteLine();
+            }
+            else
+            {
+                Console.WriteLine(message);
+            }
+        }
+
+        public void WriteError(string? message = null)
+        {
+            if (message == null)
+            {
+                Console.Error.WriteLine();
+            }
+            else
+            {
+                Console.Error.WriteLine(message);
+            }
+        }
+    }
+}

sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
+
+namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation;
+
+internal enum LogLevel
+{
+    Debug,
+    Info,
+    Warning,
+    Error
+}
+
+internal class Logger : ILogger
+{
+    internal static bool EnableDebug { get { return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.s_pLAYWRIGHT_SERVICE_DEBUG)); } set { } }
+
+#pragma warning disable CA1822 // Mark members as static
+    private void Log(LogLevel level, string message)
+#pragma warning restore CA1822 // Mark members as static
+    {
+        if (EnableDebug)
+        {
+            Console.WriteLine($"{DateTime.Now} [{level}]: {message}");
+        }
+    }
+
+    public void Debug(string message)
+    {
+        Log(LogLevel.Debug, message);
+    }
+
+    public void Error(string message)
+    {
+        Log(LogLevel.Error, message);
+    }
+
+    public void Info(string message)
+    {
+        Log(LogLevel.Info, message);
+    }
+
+    public void Warning(string message)
+    {
+        Log(LogLevel.Warning, message);
+    }
+};