diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs
index fcc55b2f5d43ec..13b18f8e3cf153 100644
--- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs
+++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs
@@ -70,23 +70,22 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
 
             if (_options.ServicesStartConcurrently)
             {
-                Task tasks = Task.WhenAll(_hostedServices.Select(async service =>
+                List<Task> tasks = new List<Task>();
+
+                foreach (IHostedService hostedService in _hostedServices)
                 {
-                    await service.StartAsync(combinedCancellationToken).ConfigureAwait(false);
+                    tasks.Add(Task.Run(() => StartAndTryToExecuteAsync(hostedService, combinedCancellationToken), combinedCancellationToken));
+                }
 
-                    if (service is BackgroundService backgroundService)
-                    {
-                        _ = TryExecuteBackgroundServiceAsync(backgroundService);
-                    }
-                }));
+                Task groupedTasks = Task.WhenAll(tasks);
 
                 try
                 {
-                    await tasks.ConfigureAwait(false);
+                    await groupedTasks.ConfigureAwait(false);
                 }
                 catch (Exception ex)
                 {
-                    exceptions.AddRange(tasks.Exception?.InnerExceptions ?? new[] { ex }.AsEnumerable());
+                    exceptions.AddRange(groupedTasks.Exception?.InnerExceptions ?? new[] { ex }.AsEnumerable());
                 }
             }
             else
@@ -96,12 +95,7 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
                     try
                     {
                         // Fire IHostedService.Start
-                        await hostedService.StartAsync(combinedCancellationToken).ConfigureAwait(false);
-
-                        if (hostedService is BackgroundService backgroundService)
-                        {
-                            _ = TryExecuteBackgroundServiceAsync(backgroundService);
-                        }
+                        await StartAndTryToExecuteAsync(hostedService, combinedCancellationToken).ConfigureAwait(false);
                     }
                     catch (Exception ex)
                     {
@@ -134,6 +128,16 @@ public async Task StartAsync(CancellationToken cancellationToken = default)
             _logger.Started();
         }
 
+        private async Task StartAndTryToExecuteAsync(IHostedService service, CancellationToken combinedCancellationToken)
+        {
+            await service.StartAsync(combinedCancellationToken).ConfigureAwait(false);
+
+            if (service is BackgroundService backgroundService)
+            {
+                _ = TryExecuteBackgroundServiceAsync(backgroundService);
+            }
+        }
+
         private async Task TryExecuteBackgroundServiceAsync(BackgroundService backgroundService)
         {
             // backgroundService.ExecuteTask may not be set (e.g. if the derived class doesn't call base.StartAsync)
@@ -185,15 +189,22 @@ public async Task StopAsync(CancellationToken cancellationToken = default)
 
                     if (_options.ServicesStopConcurrently)
                     {
-                        Task tasks = Task.WhenAll(hostedServices.Select(async service => await service.StopAsync(token).ConfigureAwait(false)));
+                        List<Task> tasks = new List<Task>();
+
+                        foreach (IHostedService hostedService in hostedServices)
+                        {
+                            tasks.Add(Task.Run(() => hostedService.StopAsync(token), token));
+                        }
+
+                        Task groupedTasks = Task.WhenAll(tasks);
 
                         try
                         {
-                            await tasks.ConfigureAwait(false);
+                            await groupedTasks.ConfigureAwait(false);
                         }
                         catch (Exception ex)
                         {
-                            exceptions.AddRange(tasks.Exception?.InnerExceptions ?? new[] { ex }.AsEnumerable());
+                            exceptions.AddRange(groupedTasks.Exception?.InnerExceptions ?? new[] { ex }.AsEnumerable());
                         }
                     }
                     else
diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/DelegateHostedService.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/DelegateHostedService.cs
index 6f6b319138f90b..370b4c40aa9116 100644
--- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/DelegateHostedService.cs
+++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/DelegateHostedService.cs
@@ -7,7 +7,7 @@
 
 namespace Microsoft.Extensions.Hosting.Unit.Tests;
 
-internal class DelegateHostedService : IHostedService, IDisposable
+internal class DelegateHostedService : IHostedService, IDisposable, IEquatable<DelegateHostedService>
 {
     private readonly Action _started;
     private readonly Action _stopping;
@@ -20,6 +20,8 @@ public DelegateHostedService(Action started, Action stopping, Action disposing)
         _disposing = disposing;
     }
 
+    public int? Identifier { get; set; }
+
     public Task StartAsync(CancellationToken token)
     {
         StartDate = DateTimeOffset.Now;
@@ -37,4 +39,8 @@ public Task StopAsync(CancellationToken token)
 
     public DateTimeOffset StartDate { get; private set; }
     public DateTimeOffset StopDate { get; private set; }
+
+    public bool Equals(DelegateHostedService other) => this == other;
+
+    public override string ToString() => $"DelegateHostedService: Id={Identifier}, StartDate={StartDate}, StopDate={StopDate}";
 }
diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs
index 4b678b8912b77e..abda9052dffd64 100644
--- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs
+++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/HostTests.cs
@@ -64,7 +64,7 @@ public async Task StartAsync_StopAsync_Concurrency(bool stopConcurrently, bool s
             {
                 var index = i;
                 var service = new DelegateHostedService(() => { events[index, 0] = true; }, () => { events[index, 1] = true; } , () => { });
-
+                service.Identifier = index;
                 hostedServices[index] = service;
             }
 
@@ -92,8 +92,11 @@ public async Task StartAsync_StopAsync_Concurrency(bool stopConcurrently, bool s
                 Assert.False(events[i, 1]);
             }
 
-            // Ensures that IHostedService instances are started in FIFO order
-            AssertExtensions.CollectionEqual(hostedServices, hostedServices.OrderBy(h => h.StartDate), EqualityComparer<DelegateHostedService>.Default);
+            // Ensures that IHostedService instances are started in FIFO order when services are started non concurrently
+            if (hostedServiceCount > 0 && !startConcurrently)
+            {
+                AssertExtensions.Equal(hostedServices, hostedServices.OrderBy(h => h.StartDate).ToArray());
+            }
 
             await host.StopAsync(CancellationToken.None);
 
@@ -103,8 +106,11 @@ public async Task StartAsync_StopAsync_Concurrency(bool stopConcurrently, bool s
                 Assert.True(events[i, 1]);
             }
 
-            // Ensures that IHostedService instances are stopped in LIFO order
-            AssertExtensions.CollectionEqual(hostedServices.Reverse(), hostedServices.OrderBy(h => h.StopDate), EqualityComparer<DelegateHostedService>.Default);
+            // Ensures that IHostedService instances are stopped in LIFO order  when services are stopped non concurrently
+            if (hostedServiceCount > 0 && !stopConcurrently)
+            {
+                AssertExtensions.Equal(hostedServices, hostedServices.OrderByDescending(h => h.StopDate).ToArray());
+            }
         }
 
         [Fact]