diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs b/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs index fa8463e64b..f86bd6b183 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs @@ -296,6 +296,19 @@ private async Task SendHttpHelperAsync( string message = $"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};"; e.Data.Add("Message", message); + + if (timeoutPolicy.ShouldThrow503OnTimeout) + { + throw CosmosExceptionFactory.CreateServiceUnavailableException( + message: message, + headers: new Headers() + { + ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString(), + SubStatusCode = SubStatusCodes.TransportGenerated503 + }, + innerException: e); + } + throw; } diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs index 413205cdba..a3fc51dd91 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs @@ -18,21 +18,45 @@ internal abstract class HttpTimeoutPolicy public abstract bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, HttpResponseMessage responseMessage); + public virtual bool ShouldThrow503OnTimeout => false; + public static HttpTimeoutPolicy GetTimeoutPolicy( DocumentServiceRequest documentServiceRequest) { + //Query Plan Requests if (documentServiceRequest.ResourceType == ResourceType.Document && documentServiceRequest.OperationType == OperationType.QueryPlan) { - return HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance; + return HttpTimeoutPolicyControlPlaneRetriableHotPath.InstanceShouldThrow503OnTimeout; } + //Partition Key Requests if (documentServiceRequest.ResourceType == ResourceType.PartitionKeyRange) { return HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance; } + //Data Plane Read & Write + if (!HttpTimeoutPolicy.IsMetaData(documentServiceRequest)) + { + return HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout; + } + + //Meta Data Read + if (HttpTimeoutPolicy.IsMetaData(documentServiceRequest) && documentServiceRequest.IsReadOnlyRequest) + { + return HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout; + } + + //Default behavior return HttpTimeoutPolicyDefault.Instance; } + + private static bool IsMetaData(DocumentServiceRequest request) + { + return (request.OperationType != Documents.OperationType.ExecuteJavaScript && request.ResourceType == ResourceType.StoredProcedure) || + request.ResourceType != ResourceType.Document; + + } } } diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRead.cs b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRead.cs index 56c55901bb..757134a576 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRead.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRead.cs @@ -44,5 +44,7 @@ public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, Ht { return false; } + + public override bool ShouldThrow503OnTimeout => true; } } diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRetriableHotPath.cs b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRetriableHotPath.cs index c8b8dbfb50..8dbb8a2a44 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRetriableHotPath.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyControlPlaneRetriableHotPath.cs @@ -10,11 +10,14 @@ namespace Microsoft.Azure.Cosmos internal sealed class HttpTimeoutPolicyControlPlaneRetriableHotPath : HttpTimeoutPolicy { - public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyControlPlaneRetriableHotPath(); + public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyControlPlaneRetriableHotPath(false); + public static readonly HttpTimeoutPolicy InstanceShouldThrow503OnTimeout = new HttpTimeoutPolicyControlPlaneRetriableHotPath(true); + public bool shouldThrow503OnTimeout; private static readonly string Name = nameof(HttpTimeoutPolicyControlPlaneRetriableHotPath); - private HttpTimeoutPolicyControlPlaneRetriableHotPath() + private HttpTimeoutPolicyControlPlaneRetriableHotPath(bool shouldThrow503OnTimeout) { + this.shouldThrow503OnTimeout = shouldThrow503OnTimeout; } private readonly IReadOnlyList<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> TimeoutsAndDelays = new List<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)>() @@ -61,5 +64,7 @@ public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, Ht return true; } + + public override bool ShouldThrow503OnTimeout => this.shouldThrow503OnTimeout; } } diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyDefault.cs b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyDefault.cs index 949afc0513..974509c70c 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyDefault.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyDefault.cs @@ -10,11 +10,14 @@ namespace Microsoft.Azure.Cosmos internal sealed class HttpTimeoutPolicyDefault : HttpTimeoutPolicy { - public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyDefault(); + public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyDefault(false); + public static readonly HttpTimeoutPolicy InstanceShouldThrow503OnTimeout = new HttpTimeoutPolicyDefault(true); + public bool shouldThrow503OnTimeout; private static readonly string Name = nameof(HttpTimeoutPolicyDefault); - private HttpTimeoutPolicyDefault() + private HttpTimeoutPolicyDefault(bool shouldThrow503OnTimeout) { + this.shouldThrow503OnTimeout = shouldThrow503OnTimeout; } private readonly IReadOnlyList<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> TimeoutsAndDelays = new List<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)>() @@ -46,5 +49,7 @@ public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, Ht { return false; } + + public override bool ShouldThrow503OnTimeout => this.shouldThrow503OnTimeout; } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosExceptionFactory.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosExceptionFactory.cs index fff41258b2..0648aecbc3 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosExceptionFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosExceptionFactory.cs @@ -283,6 +283,24 @@ internal static CosmosException CreateBadRequestException( innerException); } + internal static CosmosException CreateServiceUnavailableException( + string message, + Headers headers, + string stackTrace = default, + ITrace trace = default, + Error error = default, + Exception innerException = default) + { + return CosmosExceptionFactory.Create( + HttpStatusCode.ServiceUnavailable, + message, + stackTrace, + headers, + trace, + error, + innerException); + } + internal static CosmosException CreateUnauthorizedException( string message, Headers headers, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosHttpClientCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosHttpClientCoreTests.cs index 00d156e8b7..67c653498b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosHttpClientCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosHttpClientCoreTests.cs @@ -259,6 +259,67 @@ async Task sendFunc(HttpRequestMessage request, Cancellatio Assert.AreEqual(3, count, "Should retry 3 times"); } + [TestMethod] + public async Task HttpTimeoutThrow503TestAsync() + { + + async Task TestScenarioAsync(HttpMethod method, ResourceType resourceType, HttpTimeoutPolicy timeoutPolicy, Type expectedException, int expectedNumberOfRetrys) + { + int count = 0; + async Task sendFunc(HttpRequestMessage request, CancellationToken cancellationToken) + { + count++; + + throw new OperationCanceledException("API with exception"); + + } + + DocumentClientEventSource eventSource = DocumentClientEventSource.Instance; + HttpMessageHandler messageHandler = new MockMessageHandler(sendFunc); + using CosmosHttpClient cosmoshttpClient = MockCosmosUtil.CreateCosmosHttpClient(() => new HttpClient(messageHandler)); + + try + { + + HttpResponseMessage responseMessage1 = await cosmoshttpClient.SendHttpAsync(() => + new ValueTask( + result: new HttpRequestMessage(method, new Uri("http://localhost"))), + resourceType: resourceType, + timeoutPolicy: timeoutPolicy, + clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()), + cancellationToken: default); + } + catch (Exception e) + { + Assert.AreEqual(expectedNumberOfRetrys, count, "Should retry 3 times for read methods, for writes should only be tried once"); + Assert.AreEqual(e.GetType(), expectedException); + + if (e.GetType() == typeof(CosmosException)) + { + CosmosException cosmosException = (CosmosException)e; + Assert.AreEqual(cosmosException.StatusCode, System.Net.HttpStatusCode.ServiceUnavailable); + Assert.AreEqual((int)cosmosException.SubStatusCode,(int)SubStatusCodes.TransportGenerated503); + } + } + + } + + //Data plane read + await TestScenarioAsync(HttpMethod.Get, ResourceType.Document, HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout, typeof(CosmosException), 3); + + //Data plane write + await TestScenarioAsync(HttpMethod.Post, ResourceType.Document, HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout, typeof(CosmosException), 1); + + //Meta data read + await TestScenarioAsync(HttpMethod.Get, ResourceType.Database, HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout, typeof(CosmosException), 3); + + //Query plan read (note all query plan operations are reads). + await TestScenarioAsync(HttpMethod.Get, ResourceType.Document, HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout, typeof(CosmosException), 3); + + //Metadata Write (Should throw a 408 OperationCanceledException rather than a 503) + await TestScenarioAsync(HttpMethod.Post, ResourceType.Document, HttpTimeoutPolicyDefault.Instance, typeof(TaskCanceledException), 1); + } + [TestMethod] public async Task NoRetryOnNoRetryPolicyTestAsync() { @@ -309,7 +370,7 @@ public async Task RetryTransientIssuesForQueryPlanTestAsync() new Documents.Collections.RequestNameValueCollection()); HttpTimeoutPolicy retryPolicy = HttpTimeoutPolicy.GetTimeoutPolicy(documentServiceRequest); - Assert.AreEqual(HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance, retryPolicy); + Assert.AreEqual(HttpTimeoutPolicyControlPlaneRetriableHotPath.InstanceShouldThrow503OnTimeout, retryPolicy); int count = 0; IEnumerator<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> retry = retryPolicy.GetTimeoutEnumerator(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index e1cfeb5f40..2fc8359f68 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -61,7 +61,7 @@ public async Task DocumentClient_BuildHttpClientFactory_WithHandler() uri: new Uri("https://localhost"), additionalHeaders: new RequestNameValueCollection(), resourceType: ResourceType.Document, - timeoutPolicy: HttpTimeoutPolicyDefault.Instance, + timeoutPolicy: HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout, clientSideRequestStatistics: new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, new TraceSummary()), cancellationToken: default); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs index fe4287484c..43078d1866 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs @@ -808,7 +808,7 @@ public async Task GatewayStatsDurationTest() await cosmosHttpClient.SendHttpAsync(() => new ValueTask(new HttpRequestMessage(HttpMethod.Get, "http://someuri.com")), ResourceType.Document, - HttpTimeoutPolicyDefault.Instance, + HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout, clientSideRequestStatistics, CancellationToken.None);