From dc7ff9190fb014650110059f3bbb4d4c60f0255f Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Fri, 5 Dec 2025 18:27:44 +0000 Subject: [PATCH 1/3] feat: add JUnit reporter and command provider for test results output --- TUnit.Engine.Tests/JUnitReporterTests.cs | 154 ++++++++ .../JUnitReporterCommandProvider.cs | 50 +++ .../TestApplicationBuilderExtensions.cs | 18 + TUnit.Engine/Reporters/JUnitReporter.cs | 161 +++++++++ TUnit.Engine/Xml/JUnitXmlWriter.cs | 339 ++++++++++++++++++ 5 files changed, 722 insertions(+) create mode 100644 TUnit.Engine.Tests/JUnitReporterTests.cs create mode 100644 TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs create mode 100644 TUnit.Engine/Reporters/JUnitReporter.cs create mode 100644 TUnit.Engine/Xml/JUnitXmlWriter.cs diff --git a/TUnit.Engine.Tests/JUnitReporterTests.cs b/TUnit.Engine.Tests/JUnitReporterTests.cs new file mode 100644 index 0000000000..a25ebd3c5a --- /dev/null +++ b/TUnit.Engine.Tests/JUnitReporterTests.cs @@ -0,0 +1,154 @@ +using Microsoft.Testing.Platform.Extensions; +using TUnit.Core; +using TUnit.Engine.Reporters; + +namespace TUnit.Engine.Tests; + +[NotInParallel] +public class JUnitReporterTests +{ + private sealed class MockExtension : IExtension + { + public string Uid => "MockExtension"; + public string DisplayName => "Mock"; + public string Version => "1.0.0"; + public string Description => "Mock Extension"; + public Task IsEnabledAsync() => Task.FromResult(true); + } + + [After(Test)] + public void Cleanup() + { + // Clean up environment variables after each test + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER", null); + Environment.SetEnvironmentVariable("DISABLE_JUNIT_REPORTER", null); + Environment.SetEnvironmentVariable("TUNIT_ENABLE_JUNIT_REPORTER", null); + Environment.SetEnvironmentVariable("GITLAB_CI", null); + Environment.SetEnvironmentVariable("CI_SERVER", null); + Environment.SetEnvironmentVariable("JUNIT_XML_OUTPUT_PATH", null); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_False_When_TUNIT_DISABLE_JUNIT_REPORTER_Is_Set() + { + // Arrange + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER", "true"); + Environment.SetEnvironmentVariable("GITLAB_CI", "true"); // Even with GitLab CI, should be disabled + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsFalse(); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_False_When_DISABLE_JUNIT_REPORTER_Is_Set() + { + // Arrange + Environment.SetEnvironmentVariable("DISABLE_JUNIT_REPORTER", "true"); + Environment.SetEnvironmentVariable("GITLAB_CI", "true"); // Even with GitLab CI, should be disabled + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsFalse(); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_True_When_GITLAB_CI_Is_Set() + { + // Arrange + Environment.SetEnvironmentVariable("GITLAB_CI", "true"); + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsTrue(); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_True_When_CI_SERVER_Is_Set() + { + // Arrange + Environment.SetEnvironmentVariable("CI_SERVER", "yes"); + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsTrue(); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_True_When_TUNIT_ENABLE_JUNIT_REPORTER_Is_Set() + { + // Arrange + Environment.SetEnvironmentVariable("TUNIT_ENABLE_JUNIT_REPORTER", "true"); + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsTrue(); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_False_When_No_Environment_Variables_Are_Set() + { + // Arrange + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsFalse(); + } + + [Test] + public async Task IsEnabledAsync_Should_Return_False_When_Both_Disable_Variables_Are_Set() + { + // Arrange + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER", "true"); + Environment.SetEnvironmentVariable("DISABLE_JUNIT_REPORTER", "true"); + Environment.SetEnvironmentVariable("GITLAB_CI", "true"); // Even with GitLab CI, should be disabled + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsFalse(); + } + + [Test] + public async Task IsEnabledAsync_Should_Prefer_Disable_Over_Enable() + { + // Arrange + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER", "true"); + Environment.SetEnvironmentVariable("TUNIT_ENABLE_JUNIT_REPORTER", "true"); + var extension = new MockExtension(); + var reporter = new JUnitReporter(extension); + + // Act + var isEnabled = await reporter.IsEnabledAsync(); + + // Assert + await Assert.That(isEnabled).IsFalse(); + } +} diff --git a/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs b/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs new file mode 100644 index 0000000000..ec88c59215 --- /dev/null +++ b/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs @@ -0,0 +1,50 @@ +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; + +namespace TUnit.Engine.CommandLineProviders; + +internal class JUnitReporterCommandProvider(IExtension extension) : ICommandLineOptionsProvider +{ + public const string JUnitOutputPathOption = "junit-output-path"; + + public Task IsEnabledAsync() => extension.IsEnabledAsync(); + + public string Uid => extension.Uid; + + public string Version => extension.Version; + + public string DisplayName => extension.DisplayName; + + public string Description => extension.Description; + + public IReadOnlyCollection GetCommandLineOptions() + { + return + [ + new CommandLineOption( + JUnitOutputPathOption, + "Path to output JUnit XML file (default: TestResults/{AssemblyName}-junit.xml)", + ArgumentArity.ExactlyOne, + false) + ]; + } + + public Task ValidateOptionArgumentsAsync( + CommandLineOption commandOption, + string[] arguments) + { + if (commandOption.Name == JUnitOutputPathOption && arguments.Length != 1) + { + return ValidationResult.InvalidTask("A single output path must be provided for --junit-output-path"); + } + + return ValidationResult.ValidTask; + } + + public Task ValidateCommandLineOptionsAsync( + ICommandLineOptions commandLineOptions) + { + return ValidationResult.ValidTask; + } +} diff --git a/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs b/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs index 83b7f0c98e..435894225f 100644 --- a/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs +++ b/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs @@ -21,6 +21,9 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder) var githubReporter = new GitHubReporter(extension); var githubReporterCommandProvider = new GitHubReporterCommandProvider(extension); + var junitReporter = new JUnitReporter(extension); + var junitReporterCommandProvider = new JUnitReporterCommandProvider(extension); + testApplicationBuilder.RegisterTestFramework( serviceProvider => new TestFrameworkCapabilities(CreateCapabilities(serviceProvider)), (capabilities, serviceProvider) => new TUnitTestFramework(extension, serviceProvider, capabilities)); @@ -46,6 +49,9 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder) // GitHub reporter configuration testApplicationBuilder.CommandLine.AddProvider(() => githubReporterCommandProvider); + // JUnit reporter configuration + testApplicationBuilder.CommandLine.AddProvider(() => junitReporterCommandProvider); + testApplicationBuilder.TestHost.AddDataConsumer(serviceProvider => { // Apply command-line configuration if provided @@ -58,6 +64,18 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder) return githubReporter; }); testApplicationBuilder.TestHost.AddTestHostApplicationLifetime(_ => githubReporter); + + testApplicationBuilder.TestHost.AddDataConsumer(serviceProvider => + { + // Apply command-line configuration if provided + var commandLineOptions = serviceProvider.GetRequiredService(); + if (commandLineOptions.TryGetOptionArgumentList(JUnitReporterCommandProvider.JUnitOutputPathOption, out var pathArgs)) + { + junitReporter.SetOutputPath(pathArgs[0]); + } + return junitReporter; + }); + testApplicationBuilder.TestHost.AddTestHostApplicationLifetime(_ => junitReporter); } private static IReadOnlyCollection CreateCapabilities(IServiceProvider serviceProvider) diff --git a/TUnit.Engine/Reporters/JUnitReporter.cs b/TUnit.Engine/Reporters/JUnitReporter.cs new file mode 100644 index 0000000000..359152fff7 --- /dev/null +++ b/TUnit.Engine/Reporters/JUnitReporter.cs @@ -0,0 +1,161 @@ +using System.Collections.Concurrent; +using System.Reflection; +using System.Text; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestHost; +using TUnit.Engine.Framework; +using TUnit.Engine.Helpers; +using TUnit.Engine.Xml; + +namespace TUnit.Engine.Reporters; + +public class JUnitReporter(IExtension extension) : IDataConsumer, ITestHostApplicationLifetime, IFilterReceiver +{ + private string _outputPath = null!; + private bool _isEnabled; + + public async Task IsEnabledAsync() + { + // Check if explicitly disabled + if (EnvironmentVariableCache.Get("TUNIT_DISABLE_JUNIT_REPORTER") is not null || + EnvironmentVariableCache.Get("DISABLE_JUNIT_REPORTER") is not null) + { + return false; + } + + // Check if explicitly enabled OR running in GitLab CI + var explicitlyEnabled = EnvironmentVariableCache.Get("TUNIT_ENABLE_JUNIT_REPORTER") is not null; + var runningInGitLab = EnvironmentVariableCache.Get("GITLAB_CI") is not null || + EnvironmentVariableCache.Get("CI_SERVER") is not null; + + if (!explicitlyEnabled && !runningInGitLab) + { + return false; + } + + // Determine output path + _outputPath = EnvironmentVariableCache.Get("JUNIT_XML_OUTPUT_PATH") + ?? GetDefaultOutputPath(); + + _isEnabled = true; + return await extension.IsEnabledAsync(); + } + + public string Uid { get; } = $"{extension.Uid}JUnitReporter"; + + public string Version => extension.Version; + + public string DisplayName => extension.DisplayName; + + public string Description => extension.Description; + + private readonly ConcurrentDictionary> _updates = []; + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + var testNodeUpdateMessage = (TestNodeUpdateMessage)value; + + _updates.GetOrAdd(testNodeUpdateMessage.TestNode.Uid.Value, []).Add(testNodeUpdateMessage); + + return Task.CompletedTask; + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + public Task BeforeRunAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async Task AfterRunAsync(int exitCode, CancellationToken cancellation) + { + if (!_isEnabled || _updates.Count == 0) + { + return; + } + + // Get the last update for each test + var lastUpdates = new List(_updates.Count); + foreach (var kvp in _updates) + { + if (kvp.Value.Count > 0) + { + lastUpdates.Add(kvp.Value[kvp.Value.Count - 1]); + } + } + + // Generate JUnit XML + var xmlContent = JUnitXmlWriter.GenerateXml(lastUpdates, Filter); + + if (string.IsNullOrEmpty(xmlContent)) + { + return; + } + + // Write to file with retry logic + await WriteXmlFileAsync(_outputPath, xmlContent, cancellation); + } + + public string? Filter { get; set; } + + internal void SetOutputPath(string path) + { + _outputPath = path; + } + + private static string GetDefaultOutputPath() + { + var assemblyName = Assembly.GetEntryAssembly()?.GetName().Name ?? "TestResults"; + return Path.Combine("TestResults", $"{assemblyName}-junit.xml"); + } + + private static async Task WriteXmlFileAsync(string path, string content, CancellationToken cancellationToken) + { + // Ensure directory exists + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + const int maxAttempts = 5; + var random = new Random(); + + for (int attempt = 1; attempt <= maxAttempts; attempt++) + { + try + { +#if NET + await File.WriteAllTextAsync(path, content, Encoding.UTF8, cancellationToken); +#else + File.WriteAllText(path, content, Encoding.UTF8); +#endif + Console.WriteLine($"JUnit XML report written to: {path}"); + return; + } + catch (IOException ex) when (attempt < maxAttempts && IsFileLocked(ex)) + { + var baseDelay = 50 * Math.Pow(2, attempt - 1); + var jitter = random.Next(0, 50); + var delay = (int)(baseDelay + jitter); + + Console.WriteLine($"JUnit XML file is locked, retrying in {delay}ms (attempt {attempt}/{maxAttempts})"); + await Task.Delay(delay, cancellationToken); + } + } + + Console.WriteLine($"Failed to write JUnit XML report to: {path} after {maxAttempts} attempts"); + } + + private static bool IsFileLocked(IOException exception) + { + // Check if the exception is due to the file being locked/in use + // HResult 0x80070020 is ERROR_SHARING_VIOLATION on Windows + // HResult 0x80070021 is ERROR_LOCK_VIOLATION on Windows + var errorCode = exception.HResult & 0xFFFF; + return errorCode == 0x20 || errorCode == 0x21 || + exception.Message.Contains("being used by another process") || + exception.Message.Contains("access denied", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/TUnit.Engine/Xml/JUnitXmlWriter.cs b/TUnit.Engine/Xml/JUnitXmlWriter.cs new file mode 100644 index 0000000000..414669c0bc --- /dev/null +++ b/TUnit.Engine/Xml/JUnitXmlWriter.cs @@ -0,0 +1,339 @@ +using System.Globalization; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using System.Text; +using System.Xml; +using Microsoft.Testing.Platform.Extensions.Messages; + +namespace TUnit.Engine.Xml; + +internal static class JUnitXmlWriter +{ + public static string GenerateXml( + IEnumerable testUpdates, + string? filter) + { + // Get the last state for each test + var lastStates = GetLastStates(testUpdates); + + if (lastStates.Count == 0) + { + return string.Empty; + } + + // Calculate summary statistics + var summary = CalculateSummary(lastStates.Values); + + // Get assembly and framework information + var assemblyName = Assembly.GetEntryAssembly()?.GetName().Name ?? "TestResults"; + var targetFramework = Assembly.GetExecutingAssembly() + .GetCustomAttributes() + .SingleOrDefault() + ?.FrameworkDisplayName + ?? RuntimeInformation.FrameworkDescription; + + using var stringWriter = new StringWriter(); + var settings = new XmlWriterSettings + { + Indent = true, + Encoding = Encoding.UTF8, + OmitXmlDeclaration = false, + NewLineOnAttributes = false + }; + + using var xmlWriter = XmlWriter.Create(stringWriter, settings); + + // Write XML structure + xmlWriter.WriteStartDocument(); + + // + xmlWriter.WriteStartElement("testsuites"); + xmlWriter.WriteAttributeString("name", assemblyName); + xmlWriter.WriteAttributeString("tests", summary.Total.ToString(CultureInfo.InvariantCulture)); + xmlWriter.WriteAttributeString("failures", summary.Failures.ToString(CultureInfo.InvariantCulture)); + xmlWriter.WriteAttributeString("errors", summary.Errors.ToString(CultureInfo.InvariantCulture)); + xmlWriter.WriteAttributeString("skipped", summary.Skipped.ToString(CultureInfo.InvariantCulture)); + xmlWriter.WriteAttributeString("time", summary.TotalTime.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture)); + xmlWriter.WriteAttributeString("timestamp", summary.Timestamp.ToString("o", CultureInfo.InvariantCulture)); + + // Write test suite + WriteTestSuite(xmlWriter, lastStates.Values, assemblyName, targetFramework, summary, filter); + + xmlWriter.WriteEndElement(); // testsuites + xmlWriter.WriteEndDocument(); + + return stringWriter.ToString(); + } + + private static void WriteTestSuite( + XmlWriter writer, + IEnumerable tests, + string assemblyName, + string targetFramework, + TestSummary summary, + string? filter) + { + writer.WriteStartElement("testsuite"); + writer.WriteAttributeString("name", assemblyName); + writer.WriteAttributeString("tests", summary.Total.ToString(CultureInfo.InvariantCulture)); + writer.WriteAttributeString("failures", summary.Failures.ToString(CultureInfo.InvariantCulture)); + writer.WriteAttributeString("errors", summary.Errors.ToString(CultureInfo.InvariantCulture)); + writer.WriteAttributeString("skipped", summary.Skipped.ToString(CultureInfo.InvariantCulture)); + writer.WriteAttributeString("time", summary.TotalTime.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture)); + writer.WriteAttributeString("timestamp", summary.Timestamp.ToString("o", CultureInfo.InvariantCulture)); + writer.WriteAttributeString("hostname", Environment.MachineName); + + // Write properties + WriteProperties(writer, targetFramework, filter); + + // Write test cases + foreach (var test in tests) + { + WriteTestCase(writer, test); + } + + writer.WriteEndElement(); // testsuite + } + + private static void WriteProperties(XmlWriter writer, string targetFramework, string? filter) + { + writer.WriteStartElement("properties"); + + writer.WriteStartElement("property"); + writer.WriteAttributeString("name", "framework"); + writer.WriteAttributeString("value", targetFramework); + writer.WriteEndElement(); + + writer.WriteStartElement("property"); + writer.WriteAttributeString("name", "platform"); + writer.WriteAttributeString("value", RuntimeInformation.OSDescription); + writer.WriteEndElement(); + + writer.WriteStartElement("property"); + writer.WriteAttributeString("name", "runtime"); + writer.WriteAttributeString("value", RuntimeInformation.FrameworkDescription); + writer.WriteEndElement(); + + if (!string.IsNullOrEmpty(filter)) + { + writer.WriteStartElement("property"); + writer.WriteAttributeString("name", "filter"); + writer.WriteAttributeString("value", filter); + writer.WriteEndElement(); + } + + writer.WriteEndElement(); // properties + } + + private static void WriteTestCase(XmlWriter writer, TestNodeUpdateMessage test) + { + var testNode = test.TestNode; + + // Get test state + var stateProperty = testNode.Properties.AsEnumerable() + .FirstOrDefault(p => p is TestNodeStateProperty); + + // Get timing + var timingProperty = testNode.Properties.AsEnumerable() + .OfType() + .FirstOrDefault(); + + var duration = timingProperty?.GlobalTiming.Duration is { } d ? d : TimeSpan.Zero; + + // Get class and method names + var testMethodIdentifier = testNode.Properties.AsEnumerable() + .OfType() + .FirstOrDefault(); + + var className = testMethodIdentifier?.TypeName ?? "UnknownClass"; + var testName = testNode.DisplayName; + + // Write testcase element + writer.WriteStartElement("testcase"); + writer.WriteAttributeString("name", testName); + writer.WriteAttributeString("classname", className); + writer.WriteAttributeString("time", duration.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture)); + + // Write state-specific child elements + switch (stateProperty) + { + case FailedTestNodeStateProperty failed: + WriteFailure(writer, failed); + break; + + case ErrorTestNodeStateProperty error: + WriteError(writer, error); + break; + + case TimeoutTestNodeStateProperty timeout: + WriteTimeoutError(writer, timeout); + break; + + case CancelledTestNodeStateProperty: + WriteCancellationError(writer); + break; + + case SkippedTestNodeStateProperty skipped: + WriteSkipped(writer, skipped); + break; + + case InProgressTestNodeStateProperty: + WriteInProgressError(writer); + break; + + case PassedTestNodeStateProperty: + // No child element for passed tests + break; + } + + writer.WriteEndElement(); // testcase + } + + private static void WriteFailure(XmlWriter writer, FailedTestNodeStateProperty failed) + { + writer.WriteStartElement("failure"); + + var exception = failed.Exception; + if (exception != null) + { + writer.WriteAttributeString("message", exception.Message); + writer.WriteAttributeString("type", exception.GetType().FullName ?? "AssertionException"); + writer.WriteString(exception.ToString()); + } + else + { + var message = failed.Explanation ?? "Test failed"; + writer.WriteAttributeString("message", message); + writer.WriteAttributeString("type", "TestFailedException"); + writer.WriteString(message); + } + + writer.WriteEndElement(); // failure + } + + private static void WriteError(XmlWriter writer, ErrorTestNodeStateProperty error) + { + writer.WriteStartElement("error"); + + var exception = error.Exception; + if (exception != null) + { + writer.WriteAttributeString("message", exception.Message); + writer.WriteAttributeString("type", exception.GetType().FullName ?? "Exception"); + writer.WriteString(exception.ToString()); + } + else + { + var message = error.Explanation ?? "Test error"; + writer.WriteAttributeString("message", message); + writer.WriteAttributeString("type", "TestErrorException"); + writer.WriteString(message); + } + + writer.WriteEndElement(); // error + } + + private static void WriteTimeoutError(XmlWriter writer, TimeoutTestNodeStateProperty timeout) + { + writer.WriteStartElement("error"); + var message = timeout.Explanation ?? "Test timed out"; + writer.WriteAttributeString("message", message); + writer.WriteAttributeString("type", "TimeoutException"); + writer.WriteString(message); + writer.WriteEndElement(); // error + } + + private static void WriteCancellationError(XmlWriter writer) + { + writer.WriteStartElement("error"); + writer.WriteAttributeString("message", "Test was cancelled"); + writer.WriteAttributeString("type", "CancelledException"); + writer.WriteString("Test was cancelled"); + writer.WriteEndElement(); // error + } + + private static void WriteInProgressError(XmlWriter writer) + { + writer.WriteStartElement("error"); + writer.WriteAttributeString("message", "Test never finished"); + writer.WriteAttributeString("type", "InProgressException"); + writer.WriteString("Test never finished"); + writer.WriteEndElement(); // error + } + + private static void WriteSkipped(XmlWriter writer, SkippedTestNodeStateProperty skipped) + { + writer.WriteStartElement("skipped"); + var message = skipped.Explanation ?? "Test skipped"; + writer.WriteAttributeString("message", message); + writer.WriteString(message); + writer.WriteEndElement(); // skipped + } + + private static Dictionary GetLastStates( + IEnumerable tests) + { + var lastStates = new Dictionary(); + + foreach (var test in tests) + { + lastStates[test.TestNode.Uid.Value] = test; + } + + return lastStates; + } + + private static TestSummary CalculateSummary(IEnumerable tests) + { + var summary = new TestSummary + { + Timestamp = DateTimeOffset.Now + }; + + foreach (var test in tests) + { + summary.Total++; + + var stateProperty = test.TestNode.Properties.AsEnumerable() + .FirstOrDefault(p => p is TestNodeStateProperty); + + var timing = test.TestNode.Properties.AsEnumerable() + .OfType() + .FirstOrDefault(); + + if (timing?.GlobalTiming.Duration is { } durationValue) + { + summary.TotalTime += durationValue; + } + + switch (stateProperty) + { + case FailedTestNodeStateProperty: + summary.Failures++; + break; + case ErrorTestNodeStateProperty: + case TimeoutTestNodeStateProperty: + case CancelledTestNodeStateProperty: + case InProgressTestNodeStateProperty: + summary.Errors++; + break; + case SkippedTestNodeStateProperty: + summary.Skipped++; + break; + } + } + + return summary; + } +} + +internal sealed class TestSummary +{ + public int Total { get; set; } + public int Failures { get; set; } + public int Errors { get; set; } + public int Skipped { get; set; } + public TimeSpan TotalTime { get; set; } + public DateTimeOffset Timestamp { get; set; } +} From c612ac8cf3e868873801aad8c17d6d1e6389350c Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Fri, 5 Dec 2025 18:31:26 +0000 Subject: [PATCH 2/3] refactor: simplify JUnit reporter's disable checks and clean up tests --- TUnit.Engine.Tests/JUnitReporterTests.cs | 34 ------------------------ TUnit.Engine/Reporters/JUnitReporter.cs | 3 +-- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/TUnit.Engine.Tests/JUnitReporterTests.cs b/TUnit.Engine.Tests/JUnitReporterTests.cs index a25ebd3c5a..9320db65d6 100644 --- a/TUnit.Engine.Tests/JUnitReporterTests.cs +++ b/TUnit.Engine.Tests/JUnitReporterTests.cs @@ -21,7 +21,6 @@ public void Cleanup() { // Clean up environment variables after each test Environment.SetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER", null); - Environment.SetEnvironmentVariable("DISABLE_JUNIT_REPORTER", null); Environment.SetEnvironmentVariable("TUNIT_ENABLE_JUNIT_REPORTER", null); Environment.SetEnvironmentVariable("GITLAB_CI", null); Environment.SetEnvironmentVariable("CI_SERVER", null); @@ -44,22 +43,6 @@ public async Task IsEnabledAsync_Should_Return_False_When_TUNIT_DISABLE_JUNIT_RE await Assert.That(isEnabled).IsFalse(); } - [Test] - public async Task IsEnabledAsync_Should_Return_False_When_DISABLE_JUNIT_REPORTER_Is_Set() - { - // Arrange - Environment.SetEnvironmentVariable("DISABLE_JUNIT_REPORTER", "true"); - Environment.SetEnvironmentVariable("GITLAB_CI", "true"); // Even with GitLab CI, should be disabled - var extension = new MockExtension(); - var reporter = new JUnitReporter(extension); - - // Act - var isEnabled = await reporter.IsEnabledAsync(); - - // Assert - await Assert.That(isEnabled).IsFalse(); - } - [Test] public async Task IsEnabledAsync_Should_Return_True_When_GITLAB_CI_Is_Set() { @@ -119,23 +102,6 @@ public async Task IsEnabledAsync_Should_Return_False_When_No_Environment_Variabl await Assert.That(isEnabled).IsFalse(); } - [Test] - public async Task IsEnabledAsync_Should_Return_False_When_Both_Disable_Variables_Are_Set() - { - // Arrange - Environment.SetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER", "true"); - Environment.SetEnvironmentVariable("DISABLE_JUNIT_REPORTER", "true"); - Environment.SetEnvironmentVariable("GITLAB_CI", "true"); // Even with GitLab CI, should be disabled - var extension = new MockExtension(); - var reporter = new JUnitReporter(extension); - - // Act - var isEnabled = await reporter.IsEnabledAsync(); - - // Assert - await Assert.That(isEnabled).IsFalse(); - } - [Test] public async Task IsEnabledAsync_Should_Prefer_Disable_Over_Enable() { diff --git a/TUnit.Engine/Reporters/JUnitReporter.cs b/TUnit.Engine/Reporters/JUnitReporter.cs index 359152fff7..6a6164044d 100644 --- a/TUnit.Engine/Reporters/JUnitReporter.cs +++ b/TUnit.Engine/Reporters/JUnitReporter.cs @@ -18,8 +18,7 @@ public class JUnitReporter(IExtension extension) : IDataConsumer, ITestHostAppli public async Task IsEnabledAsync() { // Check if explicitly disabled - if (EnvironmentVariableCache.Get("TUNIT_DISABLE_JUNIT_REPORTER") is not null || - EnvironmentVariableCache.Get("DISABLE_JUNIT_REPORTER") is not null) + if (EnvironmentVariableCache.Get("TUNIT_DISABLE_JUNIT_REPORTER") is not null) { return false; } From adfd92ce1101c3bc351b01d0f517d00871a01e67 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Fri, 5 Dec 2025 19:31:44 +0000 Subject: [PATCH 3/3] refactor: replace EnvironmentVariableCache with direct Environment calls for improved clarity --- .../JUnitReporterCommandProvider.cs | 5 - .../Helpers/EnvironmentVariableCache.cs | 192 ------------------ TUnit.Engine/Helpers/ExecutionModeHelper.cs | 2 +- TUnit.Engine/Reporters/GitHubReporter.cs | 11 +- TUnit.Engine/Reporters/JUnitReporter.cs | 26 +-- TUnit.Engine/Services/VerbosityService.cs | 7 +- TUnit.Engine/Xml/JUnitXmlWriter.cs | 25 ++- 7 files changed, 40 insertions(+), 228 deletions(-) delete mode 100644 TUnit.Engine/Helpers/EnvironmentVariableCache.cs diff --git a/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs b/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs index ec88c59215..39977ee5d6 100644 --- a/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs +++ b/TUnit.Engine/CommandLineProviders/JUnitReporterCommandProvider.cs @@ -34,11 +34,6 @@ public Task ValidateOptionArgumentsAsync( CommandLineOption commandOption, string[] arguments) { - if (commandOption.Name == JUnitOutputPathOption && arguments.Length != 1) - { - return ValidationResult.InvalidTask("A single output path must be provided for --junit-output-path"); - } - return ValidationResult.ValidTask; } diff --git a/TUnit.Engine/Helpers/EnvironmentVariableCache.cs b/TUnit.Engine/Helpers/EnvironmentVariableCache.cs deleted file mode 100644 index ce0a3ddf70..0000000000 --- a/TUnit.Engine/Helpers/EnvironmentVariableCache.cs +++ /dev/null @@ -1,192 +0,0 @@ -using System.Collections.Concurrent; - -namespace TUnit.Engine.Helpers; - -/// -/// Centralized cache for environment variables to avoid repeated system calls -/// Initializes all environment variables on first access and caches them for the lifetime of the application -/// -internal static class EnvironmentVariableCache -{ - private static readonly ConcurrentDictionary _cache = new(); - private static readonly object _initLock = new(); - private static bool _initialized = false; - - /// - /// All environment variable keys that TUnit cares about - /// This helps us cache only the variables we need rather than all environment variables - /// - private static readonly string[] _tunitEnvironmentVariables = - [ - // TUnit specific variables - "TUNIT_DISCOVERY_DIAGNOSTICS", - "TUNIT_DISCOVERY_TIMEOUT_SECONDS", - "TUNIT_DATA_SOURCE_TIMEOUT_SECONDS", - "TUNIT_EXECUTION_MODE", - "TUNIT_ADAPTIVE_MIN_PARALLELISM", - "TUNIT_ADAPTIVE_MAX_PARALLELISM", - "TUNIT_ADAPTIVE_METRICS", - "TUNIT_DISABLE_GITHUB_REPORTER", - "TUNIT_GITHUB_REPORTER_STYLE", - - // CI environment detection variables - "CI", - "CONTINUOUS_INTEGRATION", - "BUILD_ID", - "BUILD_NUMBER", - "GITHUB_ACTIONS", - "GITLAB_CI", - "AZURE_PIPELINES", - "JENKINS_URL", - "TEAMCITY_VERSION", - "APPVEYOR", - "CIRCLECI", - "TRAVIS", - - // Container detection variables - "DOTNET_RUNNING_IN_CONTAINER", - "CONTAINER", - "KUBERNETES_SERVICE_HOST", - - // GitHub specific variables - "DISABLE_GITHUB_REPORTER", - "GITHUB_STEP_SUMMARY" - ]; - - /// - /// Gets the cached value of an environment variable - /// Initializes the cache on first call - /// - /// The name of the environment variable - /// The environment variable value or null if not set - public static string? Get(string variableName) - { - EnsureInitialized(); - _cache.TryGetValue(variableName, out var value); - return value; - } - - /// - /// Gets multiple cached environment variable values - /// Useful for checking multiple CI or container detection variables - /// - /// The names of the environment variables to retrieve - /// An array of environment variable values (nulls for unset variables) - public static string?[] GetMultiple(params string[] variableNames) - { - EnsureInitialized(); - var result = new string?[variableNames.Length]; - for (var i = 0; i < variableNames.Length; i++) - { - _cache.TryGetValue(variableNames[i], out var value); - result[i] = value; - } - return result; - } - - /// - /// Checks if any of the specified environment variables have non-empty values - /// Useful for CI and container detection - /// - /// The environment variable names to check - /// True if any of the variables have non-empty values - public static bool HasAnyNonEmpty(params string[] variableNames) - { - EnsureInitialized(); - for (var i = 0; i < variableNames.Length; i++) - { - _cache.TryGetValue(variableNames[i], out var value); - if (!string.IsNullOrEmpty(value)) - { - return true; - } - } - return false; - } - - /// - /// CI environment variables for quick access - /// - public static readonly string[] CiVariables = - [ - "CI", - "CONTINUOUS_INTEGRATION", - "BUILD_ID", - "BUILD_NUMBER", - "GITHUB_ACTIONS", - "GITLAB_CI", - "AZURE_PIPELINES", - "JENKINS_URL", - "TEAMCITY_VERSION", - "APPVEYOR", - "CIRCLECI", - "TRAVIS" - ]; - - /// - /// Container environment variables for quick access - /// - public static readonly string[] ContainerVariables = - [ - "DOTNET_RUNNING_IN_CONTAINER", - "CONTAINER", - "KUBERNETES_SERVICE_HOST" - ]; - - /// - /// Convenience method to check if running in a CI environment - /// - public static bool IsRunningInCI() - { - return HasAnyNonEmpty(CiVariables); - } - - /// - /// Convenience method to check if running in a container - /// - public static bool IsRunningInContainer() - { - return HasAnyNonEmpty(ContainerVariables); - } - - /// - /// Initializes the cache with all TUnit environment variables - /// Thread-safe and only runs once - /// - private static void EnsureInitialized() - { - if (_initialized) - { - return; - } - - lock (_initLock) - { - if (_initialized) - { - return; - } - - // Cache all TUnit-related environment variables - foreach (var variableName in _tunitEnvironmentVariables) - { - var value = Environment.GetEnvironmentVariable(variableName); - _cache.TryAdd(variableName, value); - } - - _initialized = true; - } - } - - /// - /// For testing purposes - allows clearing and reinitializing the cache - /// - internal static void ClearCache() - { - lock (_initLock) - { - _cache.Clear(); - _initialized = false; - } - } -} \ No newline at end of file diff --git a/TUnit.Engine/Helpers/ExecutionModeHelper.cs b/TUnit.Engine/Helpers/ExecutionModeHelper.cs index bc654ca3cf..6364fceffd 100644 --- a/TUnit.Engine/Helpers/ExecutionModeHelper.cs +++ b/TUnit.Engine/Helpers/ExecutionModeHelper.cs @@ -67,7 +67,7 @@ public static bool IsSourceGenerationMode(ICommandLineOptions commandLineOptions } // Check environment variable - var envMode = EnvironmentVariableCache.Get("TUNIT_EXECUTION_MODE"); + var envMode = Environment.GetEnvironmentVariable("TUNIT_EXECUTION_MODE"); if (!string.IsNullOrEmpty(envMode)) { var mode = envMode!.ToLowerInvariant(); diff --git a/TUnit.Engine/Reporters/GitHubReporter.cs b/TUnit.Engine/Reporters/GitHubReporter.cs index ab8a2ace43..f7c37ef5c0 100644 --- a/TUnit.Engine/Reporters/GitHubReporter.cs +++ b/TUnit.Engine/Reporters/GitHubReporter.cs @@ -7,7 +7,6 @@ using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestHost; using TUnit.Engine.Framework; -using TUnit.Engine.Helpers; namespace TUnit.Engine.Reporters; @@ -25,18 +24,18 @@ public class GitHubReporter(IExtension extension) : IDataConsumer, ITestHostAppl public async Task IsEnabledAsync() { - if (EnvironmentVariableCache.Get("TUNIT_DISABLE_GITHUB_REPORTER") is not null || - EnvironmentVariableCache.Get("DISABLE_GITHUB_REPORTER") is not null) + if (Environment.GetEnvironmentVariable("TUNIT_DISABLE_GITHUB_REPORTER") is not null || + Environment.GetEnvironmentVariable("DISABLE_GITHUB_REPORTER") is not null) { return false; } - if (EnvironmentVariableCache.Get("GITHUB_ACTIONS") is null) + if (Environment.GetEnvironmentVariable("GITHUB_ACTIONS") is null) { return false; } - if (EnvironmentVariableCache.Get("GITHUB_STEP_SUMMARY") is not { } fileName + if (Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is not { } fileName || !File.Exists(fileName)) { return false; @@ -45,7 +44,7 @@ public async Task IsEnabledAsync() _outputSummaryFilePath = fileName; // Determine reporter style from environment variable or default to collapsible - var styleEnv = EnvironmentVariableCache.Get("TUNIT_GITHUB_REPORTER_STYLE"); + var styleEnv = Environment.GetEnvironmentVariable("TUNIT_GITHUB_REPORTER_STYLE"); if (!string.IsNullOrEmpty(styleEnv)) { _reporterStyle = styleEnv!.ToLowerInvariant() switch diff --git a/TUnit.Engine/Reporters/JUnitReporter.cs b/TUnit.Engine/Reporters/JUnitReporter.cs index 6a6164044d..2a34661558 100644 --- a/TUnit.Engine/Reporters/JUnitReporter.cs +++ b/TUnit.Engine/Reporters/JUnitReporter.cs @@ -5,7 +5,6 @@ using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestHost; using TUnit.Engine.Framework; -using TUnit.Engine.Helpers; using TUnit.Engine.Xml; namespace TUnit.Engine.Reporters; @@ -18,15 +17,15 @@ public class JUnitReporter(IExtension extension) : IDataConsumer, ITestHostAppli public async Task IsEnabledAsync() { // Check if explicitly disabled - if (EnvironmentVariableCache.Get("TUNIT_DISABLE_JUNIT_REPORTER") is not null) + if (Environment.GetEnvironmentVariable("TUNIT_DISABLE_JUNIT_REPORTER") is not null) { return false; } // Check if explicitly enabled OR running in GitLab CI - var explicitlyEnabled = EnvironmentVariableCache.Get("TUNIT_ENABLE_JUNIT_REPORTER") is not null; - var runningInGitLab = EnvironmentVariableCache.Get("GITLAB_CI") is not null || - EnvironmentVariableCache.Get("CI_SERVER") is not null; + var explicitlyEnabled = Environment.GetEnvironmentVariable("TUNIT_ENABLE_JUNIT_REPORTER") is not null; + var runningInGitLab = Environment.GetEnvironmentVariable("GITLAB_CI") is not null || + Environment.GetEnvironmentVariable("CI_SERVER") is not null; if (!explicitlyEnabled && !runningInGitLab) { @@ -34,7 +33,7 @@ public async Task IsEnabledAsync() } // Determine output path - _outputPath = EnvironmentVariableCache.Get("JUNIT_XML_OUTPUT_PATH") + _outputPath = Environment.GetEnvironmentVariable("JUNIT_XML_OUTPUT_PATH") ?? GetDefaultOutputPath(); _isEnabled = true; @@ -76,12 +75,9 @@ public async Task AfterRunAsync(int exitCode, CancellationToken cancellation) // Get the last update for each test var lastUpdates = new List(_updates.Count); - foreach (var kvp in _updates) + foreach (var kvp in _updates.Where(kvp => kvp.Value.Count > 0)) { - if (kvp.Value.Count > 0) - { - lastUpdates.Add(kvp.Value[kvp.Value.Count - 1]); - } + lastUpdates.Add(kvp.Value[kvp.Value.Count - 1]); } // Generate JUnit XML @@ -100,6 +96,11 @@ public async Task AfterRunAsync(int exitCode, CancellationToken cancellation) internal void SetOutputPath(string path) { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("Output path cannot be null or empty", nameof(path)); + } + _outputPath = path; } @@ -119,7 +120,6 @@ private static async Task WriteXmlFileAsync(string path, string content, Cancell } const int maxAttempts = 5; - var random = new Random(); for (int attempt = 1; attempt <= maxAttempts; attempt++) { @@ -136,7 +136,7 @@ private static async Task WriteXmlFileAsync(string path, string content, Cancell catch (IOException ex) when (attempt < maxAttempts && IsFileLocked(ex)) { var baseDelay = 50 * Math.Pow(2, attempt - 1); - var jitter = random.Next(0, 50); + var jitter = Random.Shared.Next(0, 50); var delay = (int)(baseDelay + jitter); Console.WriteLine($"JUnit XML file is locked, retrying in {delay}ms (attempt {attempt}/{maxAttempts})"); diff --git a/TUnit.Engine/Services/VerbosityService.cs b/TUnit.Engine/Services/VerbosityService.cs index 18c36aacc2..f297c026bd 100644 --- a/TUnit.Engine/Services/VerbosityService.cs +++ b/TUnit.Engine/Services/VerbosityService.cs @@ -1,7 +1,6 @@ using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Services; using TUnit.Engine.CommandLineProviders; -using TUnit.Engine.Helpers; using LogLevel = TUnit.Core.Logging.LogLevel; #pragma warning disable TPEXP @@ -42,8 +41,6 @@ public string CreateVerbositySummary() $"(Stack traces: {ShowDetailedStackTrace}, "; } - // Use centralized environment variable cache - private static bool GetOutputLevel(ICommandLineOptions commandLineOptions, IServiceProvider serviceProvider) { // Check for --output flag (Microsoft.Testing.Platform extension) @@ -64,8 +61,8 @@ private static LogLevel GetLogLevel(ICommandLineOptions commandLineOptions) return LogLevelCommandProvider.ParseLogLevel(args); } - // Check cached legacy environment variable for backwards compatibility - if (EnvironmentVariableCache.Get("TUNIT_DISCOVERY_DIAGNOSTICS") == "1") + // Check legacy environment variable for backwards compatibility + if (Environment.GetEnvironmentVariable("TUNIT_DISCOVERY_DIAGNOSTICS") == "1") { return LogLevel.Debug; } diff --git a/TUnit.Engine/Xml/JUnitXmlWriter.cs b/TUnit.Engine/Xml/JUnitXmlWriter.cs index 414669c0bc..185674ec85 100644 --- a/TUnit.Engine/Xml/JUnitXmlWriter.cs +++ b/TUnit.Engine/Xml/JUnitXmlWriter.cs @@ -132,7 +132,8 @@ private static void WriteTestCase(XmlWriter writer, TestNodeUpdateMessage test) // Get test state var stateProperty = testNode.Properties.AsEnumerable() - .FirstOrDefault(p => p is TestNodeStateProperty); + .OfType() + .FirstOrDefault(); // Get timing var timingProperty = testNode.Properties.AsEnumerable() @@ -286,17 +287,17 @@ private static Dictionary GetLastStates( private static TestSummary CalculateSummary(IEnumerable tests) { - var summary = new TestSummary - { - Timestamp = DateTimeOffset.Now - }; + DateTimeOffset? earliestStartTime = null; + + var summary = new TestSummary(); foreach (var test in tests) { summary.Total++; var stateProperty = test.TestNode.Properties.AsEnumerable() - .FirstOrDefault(p => p is TestNodeStateProperty); + .OfType() + .FirstOrDefault(); var timing = test.TestNode.Properties.AsEnumerable() .OfType() @@ -305,6 +306,15 @@ private static TestSummary CalculateSummary(IEnumerable t if (timing?.GlobalTiming.Duration is { } durationValue) { summary.TotalTime += durationValue; + + // Track the earliest start time from actual test execution + if (timing.GlobalTiming.StartTime is { } startTime) + { + if (earliestStartTime is null || startTime < earliestStartTime) + { + earliestStartTime = startTime; + } + } } switch (stateProperty) @@ -324,6 +334,9 @@ private static TestSummary CalculateSummary(IEnumerable t } } + // Use earliest test start time, fallback to current time if no timing data available + summary.Timestamp = earliestStartTime ?? DateTimeOffset.Now; + return summary; } }