diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs
index 19aeeeeec7cd63..ecedc6c1110ab3 100644
--- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs
+++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs
@@ -1721,6 +1721,18 @@ public bool CleanCacheAndDisposeIfUnused()
                 if (NetEventSource.Log.IsEnabled()) Trace("Cleaning pool.");
                 Monitor.Enter(SyncObj, ref tookLock);
 
+                // If there are now no connections associated with this pool, we can dispose of it. We
+                // avoid aggressively cleaning up pools that have recently been used but currently aren't;
+                // if a pool was used since the last time we cleaned up, give it another chance. New pools
+                // start out saying they've recently been used, to give them a bit of breathing room and time
+                // for the initial collection to be added to it.
+                if (_associatedConnectionCount == 0 && !_usedSinceLastCleanup && _http2Connections == null)
+                {
+                    Debug.Assert(list.Count == 0, $"Expected {nameof(list)}.{nameof(list.Count)} == 0");
+                    _disposed = true;
+                    return true; // Pool is disposed of.  It should be removed.
+                }
+
                 // Get the current time.  This is compared against each connection's last returned
                 // time to determine whether a connection is too old and should be closed.
                 long nowTicks = Environment.TickCount64;
@@ -1811,18 +1823,6 @@ public bool CleanCacheAndDisposeIfUnused()
                     // At this point, good connections have been moved below freeIndex, and garbage connections have
                     // been added to the dispose list, so clear the end of the list past freeIndex.
                     list.RemoveRange(freeIndex, list.Count - freeIndex);
-
-                    // If there are now no connections associated with this pool, we can dispose of it. We
-                    // avoid aggressively cleaning up pools that have recently been used but currently aren't;
-                    // if a pool was used since the last time we cleaned up, give it another chance. New pools
-                    // start out saying they've recently been used, to give them a bit of breathing room and time
-                    // for the initial collection to be added to it.
-                    if (_associatedConnectionCount == 0 && !_usedSinceLastCleanup && _http2Connections == null)
-                    {
-                        Debug.Assert(list.Count == 0, $"Expected {nameof(list)}.{nameof(list.Count)} == 0");
-                        _disposed = true;
-                        return true; // Pool is disposed of.  It should be removed.
-                    }
                 }
 
                 // Reset the cleanup flag.  Any pools that are empty and not used since the last cleanup
@@ -1837,7 +1837,12 @@ public bool CleanCacheAndDisposeIfUnused()
                 }
 
                 // Dispose the stale connections outside the pool lock.
-                toDispose?.ForEach(c => c.Dispose());
+                // Dispose them asynchronously to not to block the caller on closing the SslStream or NetworkStream.
+                if (toDispose is not null)
+                {
+                    Task.Factory.StartNew(static s => ((List<HttpConnection>)s!).ForEach(c => c.Dispose()), toDispose,
+                        CancellationToken.None, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
+                }
             }
 
             // Pool is active.  Should not be removed.
diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs
index 84ac7270df15cc..5e99adc87295e6 100644
--- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs
+++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs
@@ -449,7 +449,7 @@ private void SetCleaningTimer(TimeSpan timeout)
         {
             try
             {
-                _cleaningTimer!.Change(timeout, timeout);
+                _cleaningTimer!.Change(timeout, Timeout.InfiniteTimeSpan);
                 _timerIsRunning = timeout != Timeout.InfiniteTimeSpan;
             }
             catch (ObjectDisposedException)
@@ -479,13 +479,10 @@ private void RemoveStalePools()
                 }
             }
 
-            // Stop running the timer if we don't have any pools to clean up.
+            // Restart the timer if we have any pools to clean up.
             lock (SyncObj)
             {
-                if (_pools.IsEmpty)
-                {
-                    SetCleaningTimer(Timeout.InfiniteTimeSpan);
-                }
+                SetCleaningTimer(!_pools.IsEmpty ? _cleanPoolTimeout : Timeout.InfiniteTimeSpan);
             }
 
             // NOTE: There is a possible race condition with regards to a pool getting cleaned up at the same