ClientRetryPolicy: Fixes behavior to Meta-data write operations in multimaster accounts

Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs

@@ -139,11 +139,6 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
             {
                 // set location-based routing directive based on request retry context
                 request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
-
-                if (!this.canUseMultipleWriteLocations && this.globalEndpointManager.GetTotalAvailableWriteLocations > 1 && !this.isReadRequest)
-                {
-                    request.RequestContext.IsRetry = true;
-                }
             }
 
             // Resolve the endpoint for the request and pin the resolution to the resolved endpoint
@@ -190,14 +185,18 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
                     this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
                     this.documentServiceRequest?.ResourceAddress ?? string.Empty);
 
-                if (!this.canUseMultipleWriteLocations && this.globalEndpointManager.GetTotalAvailableWriteLocations > 1 && !this.isReadRequest)
+                if (this.globalEndpointManager.IsMetadataWriteRequestMultimaster(this.documentServiceRequest))
                 {
-                    return await this.ShouldRetryOnEndpointFailureAsync(
-                    isReadRequest: false,
-                    markBothReadAndWriteAsUnavailable: false,
-                    forceRefresh: true,
-                    retryOnPreferredLocations: false,
-                    isMetadataWrite: true);
+                    Task<ShouldRetryResult> retryResult = this.ShouldRetryOnEndpointFailureAsync(
+                        isReadRequest: false,
+                        markBothReadAndWriteAsUnavailable: false,
+                        forceRefresh: true,
+                        retryOnPreferredLocations: false,
+                        overwriteEndpointDiscovery: true);
+
+                    this.retryContext.RetryLocationIndex = 0;
+
+                    return await retryResult;
                 }
 
                 return await this.ShouldRetryOnEndpointFailureAsync(
@@ -253,9 +252,9 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
             bool markBothReadAndWriteAsUnavailable,
             bool forceRefresh,
             bool retryOnPreferredLocations,
-            bool isMetadataWrite = false)
+            bool overwriteEndpointDiscovery = false)
         {
-            if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !isMetadataWrite))
+            if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !overwriteEndpointDiscovery))
             {
                 DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}", 
                     this.failoverRetryCount,
@@ -265,7 +264,7 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
 
             this.failoverRetryCount++;
 
-            if (this.locationEndpoint != null)
+            if (this.locationEndpoint != null && !overwriteEndpointDiscovery)
             {
                 if (isReadRequest || markBothReadAndWriteAsUnavailable)
                 {

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -95,10 +95,10 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
 
         public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
 
-        /// <summary>
-        /// Gets total availible write locations from the current location information for a Cosmos account
-        /// </summary>
-        public int GetTotalAvailableWriteLocations => this.locationCache.GetTotalAvailableWriteLocations;
+        public bool IsMetadataWriteRequestMultimaster(DocumentServiceRequest request)
+        {
+            return this.locationCache.IsMetadataWriteRequestOnMultimasterAccount(request);
+        }
 
         /// <summary>
         /// This will get the account information.

Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs

@@ -81,7 +81,7 @@ public LocationCache(
         /// <summary>
         /// Gets total availible write locations from the current location information for a Cosmos account
         /// </summary>
-        public int GetTotalAvailableWriteLocations => this.locationInfo.AvailableWriteLocations.Count;
+        public int TotalAvailableWriteLocations => this.locationInfo.AvailableWriteLocations.Count;
         /// <summary>
         /// Gets list of read endpoints ordered by
         /// 1. Preferred location
@@ -209,6 +209,11 @@ public void OnLocationPreferenceChanged(ReadOnlyCollection<string> preferredLoca
                 preferenceList: preferredLocations);
         }
 
+        public bool IsMetadataWriteRequestOnMultimasterAccount(DocumentServiceRequest request)
+        {
+            return !request.IsReadOnlyRequest && this.TotalAvailableWriteLocations > 1 && !this.CanUseMultipleWriteLocations(request);
+        }
+
         /// <summary>
         /// Resolves request to service endpoint. 
         /// 1. If this is a write request
@@ -246,18 +251,12 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request)
                 // first and the second writable region in DatabaseAccount (for manual failover)
                 DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
 
-                if (this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0)
+                if ((this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0) || this.IsMetadataWriteRequestOnMultimasterAccount(request))
                 {
                     locationIndex = Math.Min(locationIndex % 2, currentLocationInfo.AvailableWriteLocations.Count - 1);
                     string writeLocation = currentLocationInfo.AvailableWriteLocations[locationIndex];
                     locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
                 }
-                else if (request.RequestContext.IsRetry && currentLocationInfo.AvailableWriteLocations.Count > 1 && request.ResourceType != ResourceType.Document)
-                {
-                    Console.WriteLine(currentLocationInfo);
-                    string writeLocation = currentLocationInfo.AvailableWriteLocations[0]; //Always want to rout to hub region, the first one in the list of available regions
-                    locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
-                }
             }
             else
             {