Client Retry Policy: Adds HTTP timeouts with request-level cross-region retry

Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs

@@ -10,8 +10,10 @@ namespace Microsoft.Azure.Cosmos
     using System.Net;
     using System.Net.Http;
     using System.Net.Http.Headers;
+    using System.Runtime.CompilerServices;
     using System.Threading;
     using System.Threading.Tasks;
+    using global::Azure.Core;
     using Microsoft.Azure.Cosmos.Resource.CosmosExceptions;
     using Microsoft.Azure.Cosmos.Tracing;
     using Microsoft.Azure.Cosmos.Tracing.TraceData;
@@ -296,6 +298,20 @@ private async Task<HttpResponseMessage> SendHttpHelperAsync(
                                     string message =
                                             $"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
                                     e.Data.Add("Message", message);
+
+                                    if (timeoutPolicy.ShouldThrow503OnTimeout)
+                                    {
+                                        throw new CosmosExceptionFactory.CreateServiceUnavailable(
+                                            message: message,
+                                            headers: new Headers()
+                                            {
+                                                ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString()
+                                            },
+                                            innerException: e);
+
+                                        throw new ServiceUnavailableException(message, e, SubStatusCodes.TransportGenerated503);
+                                    }
+
                                     throw;
                                 }
 

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs

@@ -18,21 +18,45 @@ internal abstract class HttpTimeoutPolicy
 
         public abstract bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, HttpResponseMessage responseMessage);
 
+        public virtual bool ShouldThrow503OnTimeout => false;
+
         public static HttpTimeoutPolicy GetTimeoutPolicy(
            DocumentServiceRequest documentServiceRequest)
         {
+            //Query Plan Requests
             if (documentServiceRequest.ResourceType == ResourceType.Document
                 && documentServiceRequest.OperationType == OperationType.QueryPlan)
             {
-                return HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance;
+                return HttpTimeoutPolicyControlPlaneRetriableHotPath.InstanceShouldThrow503OnTimeout;
             }
 
+            //Partition Key Requests
             if (documentServiceRequest.ResourceType == ResourceType.PartitionKeyRange)
             {
                 return HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance;
             }
 
+            //Data Plane Read & Write
+            if (!HttpTimeoutPolicy.IsMetaData(documentServiceRequest))
+            {
+                return HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout;
+            }
+
+            //Meta Data Read
+            if (HttpTimeoutPolicy.IsMetaData(documentServiceRequest) && documentServiceRequest.IsReadOnlyRequest)
+            {
+                return HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout;
+            }
+
+            //Default behavior
             return HttpTimeoutPolicyDefault.Instance;
+
+            static bool IsMetaData(DocumentServiceRequest request)
+            {
+                return (request.OperationType != Documents.OperationType.ExecuteJavaScript && request.ResourceType == ResourceType.StoredProcedure) ||
+                    request.ResourceType != ResourceType.Document;
+
+            }
         }
     }
 }

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRead.cs

@@ -44,5 +44,7 @@ public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, Ht
         {
             return false;
         }
+
+        public override bool ShouldThrow503OnTimeout => true;
     }
 }

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRetriableHotPath.cs

@@ -10,11 +10,14 @@ namespace Microsoft.Azure.Cosmos
 
     internal sealed class HttpTimeoutPolicyControlPlaneRetriableHotPath : HttpTimeoutPolicy
     {
-        public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyControlPlaneRetriableHotPath();
+        public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyControlPlaneRetriableHotPath(false);
+        public static readonly HttpTimeoutPolicy InstanceShouldThrow503OnTimeout = new HttpTimeoutPolicyControlPlaneRetriableHotPath(true);
+        public bool shouldThrow503OnTimeout;
         private static readonly string Name = nameof(HttpTimeoutPolicyControlPlaneRetriableHotPath);
 
-        private HttpTimeoutPolicyControlPlaneRetriableHotPath()
+        private HttpTimeoutPolicyControlPlaneRetriableHotPath(bool shouldThrow503OnTimeout)
         {
+            this.shouldThrow503OnTimeout = shouldThrow503OnTimeout;
         }
 
         private readonly IReadOnlyList<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> TimeoutsAndDelays = new List<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)>()
@@ -61,5 +64,7 @@ public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, Ht
 
             return true;
         }
+
+        public override bool ShouldThrow503OnTimeout => this.shouldThrow503OnTimeout;
     }
 }

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyDefault.cs

@@ -10,11 +10,14 @@ namespace Microsoft.Azure.Cosmos
 
     internal sealed class HttpTimeoutPolicyDefault : HttpTimeoutPolicy
     {
-        public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyDefault();
+        public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyDefault(false);
+        public static readonly HttpTimeoutPolicy InstanceShouldThrow503OnTimeout = new HttpTimeoutPolicyDefault(true);
+        public bool shouldThrow503OnTimeout;
         private static readonly string Name = nameof(HttpTimeoutPolicyDefault);
 
-        private HttpTimeoutPolicyDefault()
+        private HttpTimeoutPolicyDefault(bool shouldThrow503OnTimeout)
         {
+            this.shouldThrow503OnTimeout = shouldThrow503OnTimeout;
         }
 
         private readonly IReadOnlyList<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> TimeoutsAndDelays = new List<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)>()
@@ -46,5 +49,7 @@ public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, Ht
         {
             return false;
         }
+
+        public override bool ShouldThrow503OnTimeout => this.shouldThrow503OnTimeout;
     }
 }

Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosExceptionFactory.cs

@@ -283,6 +283,24 @@ internal static CosmosException CreateBadRequestException(
                 innerException);
         }
 
+        internal static CosmosException CreateServiceUnavailable(
+            string message, 
+            Headers headers,
+            string stackTrace = default, 
+            ITrace trace = default,
+            Error error = default,
+            Exception innerException = default)
+        {
+            return CosmosExceptionFactory.Create(
+                HttpStatusCode.ServiceUnavailable,
+                message,
+                stackTrace,
+                headers,
+                trace,
+                error,
+                innerException);
+        }
+
         internal static CosmosException CreateUnauthorizedException(
             string message,
             Headers headers,

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosHttpClientCoreTests.cs

@@ -16,6 +16,7 @@ namespace Microsoft.Azure.Cosmos.Tests
     using System.Collections.Generic;
     using Microsoft.Azure.Cosmos.Tracing;
     using Microsoft.Azure.Cosmos.Tracing.TraceData;
+    using FluentAssertions;
 
     [TestClass]
     public class CosmosHttpClientCoreTests
@@ -259,6 +260,113 @@ async Task<HttpResponseMessage> sendFunc(HttpRequestMessage request, Cancellatio
             Assert.AreEqual(3, count, "Should retry 3 times");
         }
 
+        [TestMethod]
+        public async Task HttpTimeoutThrow503TestAsync()
+        {
+            int count = 0;
+            async Task<HttpResponseMessage> sendFunc(HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                count++;
+
+                throw new OperationCanceledException("API with exception");
+
+            }
+
+            DocumentClientEventSource eventSource = DocumentClientEventSource.Instance;
+            HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc);
+            using CosmosHttpClient cosmoshttpClient = MockCosmosUtil.CreateCosmosHttpClient(() => new HttpClient(messageHandler));
+
+            //Data plane read
+            try
+            {
+
+                HttpResponseMessage responseMessage1 = await cosmoshttpClient.SendHttpAsync(() =>
+                    new ValueTask<HttpRequestMessage>(
+                        result: new HttpRequestMessage(HttpMethod.Get, new Uri("http://localhost"))),
+                        resourceType: ResourceType.Document,
+                        timeoutPolicy: HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout,
+                        clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()),
+                        cancellationToken: default);
+            }
+            catch (Exception e)
+            {
+                Assert.AreEqual(3, count, "Should retry 3 times");
+                Assert.IsInstanceOfType(e, typeof(ServiceUnavailableException));
+            }
+            count = 0;
+
+            //Data plane write
+            try
+            {
+                HttpResponseMessage responseMessage2 = await cosmoshttpClient.SendHttpAsync(() =>
+                   new ValueTask<HttpRequestMessage>(
+                       result: new HttpRequestMessage(HttpMethod.Post, new Uri("http://localhost"))),
+                       resourceType: ResourceType.Document,
+                       timeoutPolicy: HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout,
+                       clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()),
+                       cancellationToken: default);
+            }
+            catch (Exception e)
+            {
+                Assert.AreEqual(1, count, "Post methods should not retry");
+                Assert.IsInstanceOfType(e, typeof(ServiceUnavailableException));
+            }
+            count = 0;
+
+            //Meta data read
+            try
+            {
+                HttpResponseMessage responseMessage3 = await cosmoshttpClient.SendHttpAsync(() =>
+                   new ValueTask<HttpRequestMessage>(
+                       result: new HttpRequestMessage(HttpMethod.Get, new Uri("http://localhost"))),
+                       resourceType: ResourceType.Database,
+                       timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance,
+                       clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()),
+                       cancellationToken: default);
+            }
+            catch (Exception e)
+            {
+                Assert.AreEqual(3, count, "Should retry 3 times");
+                Assert.IsInstanceOfType(e, typeof(ServiceUnavailableException));
+            }
+            count = 0;
+
+            //Query plan read (note all query plan operations are reads).
+            try
+            {
+                HttpResponseMessage responseMessage2 = await cosmoshttpClient.SendHttpAsync(() =>
+                   new ValueTask<HttpRequestMessage>(
+                       result: new HttpRequestMessage(HttpMethod.Get, new Uri("http://localhost"))),
+                       resourceType: ResourceType.Document,
+                       timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.InstanceShouldThrow503OnTimeout,
+                       clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()),
+                       cancellationToken: default);
+            }
+            catch (Exception e)
+            {
+                Assert.AreEqual(3, count, "Should retry 3 times");
+                Assert.IsInstanceOfType(e, typeof(ServiceUnavailableException));
+            }
+            count = 0;
+
+            //Metadata Write (Should throw a 408 OperationCanceledException rather than a 503)
+            try
+            {
+                HttpResponseMessage responseMessage2 = await cosmoshttpClient.SendHttpAsync(() =>
+                   new ValueTask<HttpRequestMessage>(
+                       result: new HttpRequestMessage(HttpMethod.Get, new Uri("http://localhost"))),
+                       resourceType: ResourceType.Database,
+                       timeoutPolicy: HttpTimeoutPolicyDefault.Instance,
+                       clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()),
+                       cancellationToken: default);
+            }
+            catch (Exception e)
+            {
+                Assert.AreEqual(3, count, "Post methods should not retry");
+                Assert.IsInstanceOfType(e, typeof(OperationCanceledException));
+            }
+        }
+
         [TestMethod]
         public async Task NoRetryOnNoRetryPolicyTestAsync()
         {
@@ -309,7 +417,7 @@ public async Task RetryTransientIssuesForQueryPlanTestAsync()
                 new Documents.Collections.RequestNameValueCollection());
 
             HttpTimeoutPolicy retryPolicy = HttpTimeoutPolicy.GetTimeoutPolicy(documentServiceRequest);
-            Assert.AreEqual(HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance, retryPolicy);
+            Assert.AreEqual(HttpTimeoutPolicyControlPlaneRetriableHotPath.InstanceShouldThrow503OnTimeout, retryPolicy);
 
             int count = 0;
             IEnumerator<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> retry = retryPolicy.GetTimeoutEnumerator();

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs

@@ -61,7 +61,7 @@ public async Task DocumentClient_BuildHttpClientFactory_WithHandler()
                 uri: new Uri("https://localhost"),
                 additionalHeaders: new RequestNameValueCollection(),
                 resourceType: ResourceType.Document,
-                timeoutPolicy: HttpTimeoutPolicyDefault.Instance,
+                timeoutPolicy: HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout,
                 clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()),
                 cancellationToken: default);
 

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs

@@ -808,7 +808,7 @@ public async Task GatewayStatsDurationTest()
 
             await cosmosHttpClient.SendHttpAsync(() => new ValueTask<HttpRequestMessage>(new HttpRequestMessage(HttpMethod.Get, "http://someuri.com")),
                                                   ResourceType.Document,
-                                                  HttpTimeoutPolicyDefault.Instance,
+                                                  HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout,
                                                   clientSideRequestStatistics,
                                                   CancellationToken.None);