Add missing refs; add unit tests; ensure stop behavior cannot be chan…

…ged mid-execution; add support for stop behavior and exception behavior to be set in config file

src/libraries/Microsoft.Extensions.Hosting/ref/Microsoft.Extensions.Hosting.cs

@@ -18,6 +18,11 @@ public enum BackgroundServiceExceptionBehavior
         StopHost,
         Ignore
     }
+    public enum BackgroundServiceStopBehavior
+    {
+        Sequential,
+        Asynchronous
+    }
     public partial class ConsoleLifetimeOptions
     {
         public ConsoleLifetimeOptions() { }
@@ -104,6 +109,7 @@ public partial class HostOptions
         public HostOptions() { }
         public System.TimeSpan ShutdownTimeout { get { throw null; } set { } }
         public Microsoft.Extensions.Hosting.BackgroundServiceExceptionBehavior BackgroundServiceExceptionBehavior { get { throw null; } set { } }
+        public Microsoft.Extensions.Hosting.BackgroundServiceStopBehavior BackgroundServiceStopBehavior { get { throw null; } set { } }
     }
 }
 namespace Microsoft.Extensions.Hosting.Internal

src/libraries/Microsoft.Extensions.Hosting/src/HostOptions.cs

@@ -39,11 +39,25 @@ public class HostOptions
         internal void Initialize(IConfiguration configuration)
         {
             var timeoutSeconds = configuration["shutdownTimeoutSeconds"];
-            if (!string.IsNullOrEmpty(timeoutSeconds)
+            if (!string.IsNullOrWhiteSpace(timeoutSeconds)
                 && int.TryParse(timeoutSeconds, NumberStyles.None, CultureInfo.InvariantCulture, out var seconds))
             {
                 ShutdownTimeout = TimeSpan.FromSeconds(seconds);
             }
+
+            var backgroundServiceStopBehavior = configuration["backgroundServiceStopBehavior"];
+            if (!string.IsNullOrWhiteSpace(backgroundServiceStopBehavior)
+                && Enum.TryParse<BackgroundServiceStopBehavior>(backgroundServiceStopBehavior, out var stopBehavior))
+            {
+                BackgroundServiceStopBehavior = stopBehavior;
+            }
+
+            var backgroundServiceExceptionBehavior = configuration["backgroundServiceExceptionBehavior"];
+            if (!string.IsNullOrWhiteSpace(backgroundServiceExceptionBehavior)
+                && Enum.TryParse<BackgroundServiceExceptionBehavior>(backgroundServiceExceptionBehavior, out var exceptionBehavior))
+            {
+                BackgroundServiceExceptionBehavior = exceptionBehavior;
+            }
         }
     }
 }

src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs

@@ -120,15 +120,17 @@ public async Task StopAsync(CancellationToken cancellationToken = default)
                 // Trigger IHostApplicationLifetime.ApplicationStopping
                 _applicationLifetime.StopApplication();
 
-                IList<Exception> exceptions = new List<Exception>();
-                if (_hostedServices != null) // Started?
+                List<Exception> exceptions = new List<Exception>();
+                if (_hostedServices?.Any() == true) // Started?
                 {
-                    Queue<Task> tasks = new Queue<Task>();
-                    foreach (IHostedService hostedService in _hostedServices.Reverse())
+                    // Ensure hosted services are stopped in FILO order
+                    IEnumerable<IHostedService> hostedServices = _hostedServices.Reverse();
+
+                    switch (_options.BackgroundServiceStopBehavior)
                     {
-                        switch (_options.BackgroundServiceStopBehavior)
-                        {
-                            case BackgroundServiceStopBehavior.Sequential:
+                        case BackgroundServiceStopBehavior.Sequential:
+                            foreach (IHostedService hostedService in hostedServices)
+                            {
                                 try
                                 {
                                     await hostedService.StopAsync(token).ConfigureAwait(false);
@@ -137,24 +139,21 @@ public async Task StopAsync(CancellationToken cancellationToken = default)
                                 {
                                     exceptions.Add(ex);
                                 }
-                                break;
-                            case BackgroundServiceStopBehavior.Asynchronous:
-                            default:
-                                tasks.Enqueue(hostedService.StopAsync(token));
-                                break;
-                        }
-                    }
-
-                    foreach (Task task in tasks)
-                    {
-                        try
-                        {
-                            await task.ConfigureAwait(false);
-                        }
-                        catch (Exception ex)
-                        {
-                            exceptions.Add(ex);
-                        }
+                            }
+                            break;
+                        case BackgroundServiceStopBehavior.Asynchronous:
+                        default:
+                            Task tasks = Task.WhenAll(hostedServices.Select(async service => await service.StopAsync(token).ConfigureAwait(false)));
+
+                            try
+                            {
+                                await tasks.ConfigureAwait(false);
+                            }
+                            catch (Exception ex)
+                            {
+                                exceptions.AddRange(tasks.Exception?.InnerExceptions?.ToArray() ?? new[] { ex });
+                            }
+                            break;
                     }
                 }
 

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs

@@ -14,8 +14,10 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Configuration.UserSecrets;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Moq;
 using Xunit;
 
 namespace Microsoft.Extensions.Hosting.Tests
@@ -34,6 +36,85 @@ public async Task StopAsyncWithCancellation()
             await host.StopAsync(cts.Token);
         }
 
+        public static IEnumerable<object[]> StopAsync_ValidBackgroundServiceStopBehavior_TestCases
+        {
+            get
+            {
+                foreach (BackgroundServiceStopBehavior stopBehavior in Enum.GetValues(typeof(BackgroundServiceStopBehavior)))
+                {
+                    foreach (int hostedServiceCount in new[] { 0, 1, 10 })
+                    {
+                        yield return new object[] { stopBehavior, hostedServiceCount };
+                    }
+                }
+            }
+        }
+        [Theory]
+        [MemberData(nameof(StopAsync_ValidBackgroundServiceStopBehavior_TestCases))]
+        public async Task StopAsync_ValidBackgroundServiceStopBehavior(BackgroundServiceStopBehavior stopBehavior, int hostedServiceCount)
+        {
+            var callOrder = hostedServiceCount;
+            var hostedServices = new Mock<BackgroundService>[hostedServiceCount];
+            var executionCount = 0;
+
+            for (int i = 0; i < hostedServiceCount; i++)
+            {
+                var index = i;
+                var service = new Mock<BackgroundService>();
+                service.Setup(y => y.StopAsync(It.IsAny<CancellationToken>()))
+                    .Callback(() =>
+                    {
+                        // Ensures that IHostedService instances are stopped in FILO order
+                        Assert.Equal(index, --callOrder);
+                    })
+                    .Returns(async () =>
+                    {
+                        if (stopBehavior.Equals(BackgroundServiceStopBehavior.Asynchronous) && hostedServiceCount > 1)
+                        {
+                            // Basically this will force all IHostedService instances to have StopAsync called before any one completes
+                            executionCount++;
+                            int waitCount = 0;
+
+                            while (executionCount < hostedServiceCount && waitCount++ < 20)
+                            {
+                                await Task.Delay(10).ConfigureAwait(false);
+                            }
+
+                            // This will ensure that we truly are stopping all services asynchronously
+                            Assert.Equal(hostedServiceCount, executionCount);
+                        }
+                    })
+                    .Verifiable();
+
+                hostedServices[index] = service;
+            }
+
+            var builder = new HostBuilder();
+            using var host = Host.CreateDefaultBuilder().ConfigureHostConfiguration(configBuilder =>
+            {
+                configBuilder.AddInMemoryCollection(new KeyValuePair<string, string>[]
+                {
+                    new KeyValuePair<string, string>("backgroundServiceStopBehavior", stopBehavior.ToString())
+                });
+            }).ConfigureServices(configurer =>
+            {
+                for (int i = 0; i < hostedServiceCount; i++)
+                {
+                    var index = i;
+                    configurer.Add(ServiceDescriptor.Singleton<IHostedService>(hostedServices[index].Object));
+                }
+            }).Build();
+
+            await host.StartAsync();
+
+            await host.StopAsync(CancellationToken.None);
+
+            foreach (var service in hostedServices)
+            {
+                service.Verify();
+            }
+        }
+
         [Fact]
         public void CreateDefaultBuilder_IncludesContentRootByDefault()
         {