Skip to content
Merged
Next Next commit
[wasm] Add support for a random test case orderer, for xunit tests
This is enabled by default for wasm with
`$(XUnitUseRandomizedTestOrderer)=true`.

When the library tests run, they print two messages like:

```
  info: Using random seed for test cases: 700149826
  info: Using random seed for collections: 700149826
  info: Starting:    System.Collections.Immutable.Tests.dll
```

These seeds are picked randomly every time the tests are run. To run the
tests with a specific seed, use environment variable
`XUNIT_RANDOM_ORDER_SEED`.

When running with tests, this can be used as:

    `WasmXHarnessMonoArgs="--setenv=XUNIT_RANDOM_ORDER_SEED=<seed>`
  • Loading branch information
radical authored Feb 20, 2022
commit ce4eef991a0bf81a6bcd050f5aea4fae6927f0ac
4 changes: 4 additions & 0 deletions eng/testing/tests.mobile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@
<AdditionalXHarnessArguments Condition="'$(XUnitClassName)' != ''">$(AdditionalXHarnessArguments) -- -c=$(XUnitClassName)</AdditionalXHarnessArguments>
</PropertyGroup>

<ItemGroup Condition="'$(XUnitUseRandomizedTestOrderer)' == 'true'">
<Compile Include="$(RepoRoot)src\libraries\Common\tests\Tests\RandomizedTestOrderAssemblyInfo.cs" />
</ItemGroup>

<UsingTask Condition="'$(RunAOTCompilation)' == 'true'" TaskName="MonoAOTCompiler" AssemblyFile="$(MonoAOTCompilerTasksAssemblyPath)" />
<Import Condition="'$(RunAOTCompilation)' == 'true'" Project="$(MonoAOTCompilerDir)MonoAOTCompiler.props" />

Expand Down
1 change: 1 addition & 0 deletions eng/testing/tests.wasm.targets
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<_ShellCommandSeparator Condition="'$(OS)' != 'Windows_NT'">&amp;&amp;</_ShellCommandSeparator>
<WasmNativeStrip>false</WasmNativeStrip>
<_WasmMainJSFileName Condition="'$(WasmMainJSPath)' != ''">$([System.IO.Path]::GetFileName('$(WasmMainJSPath)'))</_WasmMainJSFileName>
<XUnitUseRandomizedTestOrderer>true</XUnitUseRandomizedTestOrderer>
</PropertyGroup>

<PropertyGroup>
Expand Down
77 changes: 77 additions & 0 deletions src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using Xunit.Abstractions;
using Xunit.Sdk;

#nullable enable

namespace TestUtilities;

// Based on https://github.com/xunit/xunit/blob/v2/src/xunit.execution/Sdk/DefaultTestCaseOrderer.cs

public class RandomTestCaseOrderer : ITestCaseOrderer
{
public const string RandomSeedEnvironmentVariableName = "XUNIT_RANDOM_ORDER_SEED";

public static readonly Lazy<int> LazySeed = new (GetSeed, LazyThreadSafetyMode.ExecutionAndPublication);
private readonly IMessageSink _diagnosticMessageSink;

private static int GetSeed()
{
string? seedEnvVar = Environment.GetEnvironmentVariable(RandomSeedEnvironmentVariableName);
if (string.IsNullOrEmpty(seedEnvVar) || !int.TryParse(seedEnvVar, out int seed))
{
seed = new Random().Next();
}

return seed;
}

public RandomTestCaseOrderer(IMessageSink diagnosticMessageSink)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using random seed for test cases: {LazySeed.Value}"));
_diagnosticMessageSink = diagnosticMessageSink;
}

public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases) where TTestCase : ITestCase
=> TryRandomize(testCases.ToList(), _diagnosticMessageSink, out List<TTestCase>? randomizedTests)
? randomizedTests
: testCases;

public static bool TryRandomize<T>(List<T> tests, IMessageSink messageSink, [NotNullWhen(true)] out List<T>? randomizedTests)
{
randomizedTests = null;
try
{
randomizedTests = Randomize(tests.ToList());
return true;
}
catch (Exception ex)
{
messageSink.OnMessage(new DiagnosticMessage($"Failed to randomize test cases: {ex}"));
return false;
}

static List<T> Randomize(List<T> tests)
{
var result = new List<T>(tests.Count);

var randomizer = new Random(LazySeed.Value);

while (tests.Count > 0)
{
int next = randomizer.Next(tests.Count);
result.Add(tests[next]);
tests.RemoveAt(next);
}

return result;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using System.Linq;
using Xunit;
using Xunit.Abstractions;
using Xunit.Sdk;

#nullable enable

namespace TestUtilities;

public class RandomTestCollectionOrderer : ITestCollectionOrderer
{
private readonly IMessageSink _diagnosticMessageSink;

public RandomTestCollectionOrderer(IMessageSink diagnosticMessageSink)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage(
$"Using random seed for collections: {RandomTestCaseOrderer.LazySeed.Value}"));
_diagnosticMessageSink = diagnosticMessageSink;
}

public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollection> testCollections)
=> RandomTestCaseOrderer.TryRandomize(testCollections.ToList(), _diagnosticMessageSink, out List<ITestCollection>? randomizedTests)
? randomizedTests
: testCollections;
}
3 changes: 3 additions & 0 deletions src/libraries/Common/tests/TestUtilities/TestUtilities.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@
variant from the Common folder and adding the missing members manually.
-->
<Compile Include="Interop\Interop.Libraries.cs" />

<Compile Include="RandomTestCaseOrderer.cs" />
<Compile Include="RandomTestCollectionOrderer.cs" />
</ItemGroup>
<!-- Windows imports -->
<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Xunit;

[assembly: TestCaseOrderer("TestUtilities.RandomTestCaseOrderer", "TestUtilities")]
[assembly: TestCollectionOrderer("TestUtilities.RandomTestCollectionOrderer", "TestUtilities")]