Skip to content

Session Consistency: Adds SessionTokenMismatchRetryPolicy optimization through customer supplied region switch hints #4976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dibahlfi wants to merge 8 commits into
base: msdata/direct
Choose a base branch
from
Draft

Session Consistency: Adds SessionTokenMismatchRetryPolicy optimization through customer supplied region switch hints #4976

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4dc4ee2
feature: WIP - Session Consistency Retries
dibahlfi Dec 25, 2024
99241ea
feature: WIP - Session Consistency Retries
dibahlfi Jan 8, 2025
6b2e0b4
feature:WIP - Session Consistency Retries
dibahlfi Jan 17, 2025
f855753
feature: WIP - Session Consistency Retries
dibahlfi Jan 22, 2025
b3580af
feature:WIP - Session Consistency Retries
dibahlfi Jan 25, 2025
b504545
feature: addressing comments
dibahlfi Mar 14, 2025
2d80ec9
fix: changing public contract to only expose RemoteRegionPreferred
dibahlfi Mar 19, 2025
257f74e
fix: changing the public contract to bool
dibahlfi Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions Microsoft.Azure.Cosmos/src/SessionRetryOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,26 @@
/// </summary>
public sealed class SessionRetryOptions : ISessionRetryOptions
Copy link
Member

@kirankumarkolli kirankumarkolli Mar 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Two possible options

  • In CosmsoClientOptions just take a bool as contract
  • Expose a new type of SessionRetryOptions

If we environ this surface area expanding richly forward then later makes sense, otherwise feels like overkill thoughts?

Copy link
Member

@kirankumarkolli kirankumarkolli Mar 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It also then makes the RetryTime and RetryCunt as implementation details not as public contract.

Copy link
Member Author

@dibahlfi dibahlfi Mar 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make sense. I will make the changes to take bool as a contract.

{
/// <summary>
/// Initializes a new instance of the <see cref="SessionRetryOptions"/> class.
/// </summary>
public SessionRetryOptions()
{
this.MinInRegionRetryTime = ConfigurationManager.GetMinRetryTimeInLocalRegionWhenRemoteRegionPreferred();
this.MaxInRegionRetryCount = ConfigurationManager.GetMaxRetriesInLocalRegionWhenRemoteRegionPreferred();
}
/// <summary>
/// Sets the minimum retry time for 404/1002 retries within each region for read and write operations.
/// The minimum value is 100ms - this minimum is enforced to provide a way for the local region to catch-up on replication lag. The default value is 500ms - as a recommendation ensure that this value is higher than the steady-state
/// replication latency between the regions you chose
/// </summary>
public TimeSpan MinInRegionRetryTime { get; set; } = ConfigurationManager.GetMinRetryTimeInLocalRegionWhenRemoteRegionPreferred();
public TimeSpan MinInRegionRetryTime { get; private set; }

/// <summary>
/// Sets the maximum number of retries within each region for read and write operations. The minimum value is 1 - the backoff time for the last in-region retry will ensure that the total retry time within the
/// region is at least the min. in-region retry time.
/// </summary>
public int MaxInRegionRetryCount { get; set; } = ConfigurationManager.GetMaxRetriesInLocalRegionWhenRemoteRegionPreferred();
public int MaxInRegionRetryCount { get; private set; }


/// <summary>
Expand All @@ -31,7 +39,7 @@ public sealed class SessionRetryOptions : ISessionRetryOptions
/// <summary>
/// validates the client options.
/// </summary>
public void Validate()
internal void Validate()
{
if (this.RemoteRegionPreferred)
{
Expand Down
145 changes: 77 additions & 68 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/SessionRetryOptionsTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,12 @@ public async Task ReadOperationWithReadSessionUnavailableTest(FaultInjectionOper
int sessionTokenMismatchRetryAttempts , Boolean remoteRegionPreferred)
{
string[] preferredRegions = this.writeRegionMap.Keys.ToArray();
// if I go to first region for reading an item, I should get a 404/2002 response for 10 minutes
FaultInjectionRule badSessionTokenRule = new FaultInjectionRuleBuilder(
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, "100");
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, Convert.ToString(sessionTokenMismatchRetryAttempts));
try
{
// if I go to first region for reading an item, I should get a 404/2002 response for 10 minutes
FaultInjectionRule badSessionTokenRule = new FaultInjectionRuleBuilder(
id: "badSessionTokenRule",
condition:
new FaultInjectionConditionBuilder()
Expand All @@ -63,58 +67,59 @@ public async Task ReadOperationWithReadSessionUnavailableTest(FaultInjectionOper
List<FaultInjectionRule> rules = new List<FaultInjectionRule>() { badSessionTokenRule };
FaultInjector faultInjector = new FaultInjector(rules);
Assert.IsNotNull(faultInjector);


CosmosClientOptions clientOptions = new CosmosClientOptions()
{

SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = remoteRegionPreferred,
MinInRegionRetryTime = TimeSpan.FromMilliseconds(100),
MaxInRegionRetryCount = sessionTokenMismatchRetryAttempts
},

ConsistencyLevel = ConsistencyLevel.Session,
ApplicationPreferredRegions = preferredRegions,
ConnectionMode = ConnectionMode.Direct,
};

using (CosmosClient faultInjectionClient = new CosmosClient(
connectionString: this.connectionString,
clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions)))
{
Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName);
Container container = await database.CreateContainerIfNotExistsAsync("sessionRetryPolicy", "/id");
string GUID = Guid.NewGuid().ToString();
dynamic testObject = new
{
id = GUID,
name = "customer one",
address = new
Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName);
Container container = await database.CreateContainerIfNotExistsAsync("sessionRetryPolicy", "/id");
string GUID = Guid.NewGuid().ToString();
dynamic testObject = new
{
line1 = "45 new street",
city = "mckinney",
postalCode = "98989",
}
id = GUID,
name = "customer one",
address = new
{
line1 = "45 new street",
city = "mckinney",
postalCode = "98989",
}

};
};

ItemResponse<dynamic> response = await container.CreateItemAsync<dynamic>(testObject);
Assert.IsNotNull(response);

OperationExecutionResult executionResult = await this.PerformDocumentOperation(faultInjectionOperationType, container, testObject);
this.ValidateOperationExecutionResult(executionResult, remoteRegionPreferred);
ItemResponse<dynamic> response = await container.CreateItemAsync<dynamic>(testObject);
Assert.IsNotNull(response);

// For a non-write operation, the request can go to multiple replicas (upto 4 replicas)
// Check if the SessionTokenMismatchRetryPolicy retries on the bad / lagging region
// for sessionTokenMismatchRetryAttempts by tracking the badSessionTokenRule hit count
long hitCount = badSessionTokenRule.GetHitCount();
OperationExecutionResult executionResult = await this.PerformDocumentOperation(faultInjectionOperationType, container, testObject);
this.ValidateOperationExecutionResult(executionResult, remoteRegionPreferred);

if (remoteRegionPreferred)
{
Assert.IsTrue(hitCount >= sessionTokenMismatchRetryAttempts && hitCount <= (1 + sessionTokenMismatchRetryAttempts) * 4);
// For a non-write operation, the request can go to multiple replicas (upto 4 replicas)
// Check if the SessionTokenMismatchRetryPolicy retries on the bad / lagging region
// for sessionTokenMismatchRetryAttempts by tracking the badSessionTokenRule hit count
long hitCount = badSessionTokenRule.GetHitCount();

if (remoteRegionPreferred)
{
Assert.IsTrue(hitCount >= sessionTokenMismatchRetryAttempts && hitCount <= (1 + sessionTokenMismatchRetryAttempts) * 4);
}
}

}
finally
{
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, null);
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, null);
}
}

Expand All @@ -135,7 +140,10 @@ public async Task WriteOperationWithReadSessionUnavailableTest(FaultInjectionOpe
{

string[] preferredRegions = this.writeRegionMap.Keys.ToArray();

Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, "100");
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, Convert.ToString(sessionTokenMismatchRetryAttempts));

try {
FaultInjectionRule badSessionTokenRule = new FaultInjectionRuleBuilder(
id: "badSessionTokenRule",
condition:
Expand All @@ -157,52 +165,53 @@ public async Task WriteOperationWithReadSessionUnavailableTest(FaultInjectionOpe
SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = remoteRegionPreferred,
MinInRegionRetryTime = TimeSpan.FromMilliseconds(100),
MaxInRegionRetryCount = sessionTokenMismatchRetryAttempts
},
ConsistencyLevel = ConsistencyLevel.Session,
ApplicationPreferredRegions = preferredRegions,
ConnectionMode = ConnectionMode.Direct,
};

using (CosmosClient faultInjectionClient = new CosmosClient(
connectionString: this.connectionString,
clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions)))
{
Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName);
Container container = await database.CreateContainerIfNotExistsAsync("sessionRetryPolicy", "/id");
string GUID = Guid.NewGuid().ToString();
dynamic testObject = new
using (CosmosClient faultInjectionClient = new CosmosClient(
connectionString: this.connectionString,
clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions)))
{
id = GUID,
name = "customer one",
address = new
Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName);
Container container = await database.CreateContainerIfNotExistsAsync("sessionRetryPolicy", "/id");
string GUID = Guid.NewGuid().ToString();
dynamic testObject = new
{
line1 = "45 new street",
city = "mckinney",
postalCode = "98989",
id = GUID,
name = "customer one",
address = new
{
line1 = "45 new street",
city = "mckinney",
postalCode = "98989",
}

};

OperationExecutionResult executionResult = await this.PerformDocumentOperation(faultInjectionOperationType, container, testObject);
this.ValidateOperationExecutionResult(executionResult, remoteRegionPreferred);

// For a write operation, the request can just go to the primary replica
// Check if the SessionTokenMismatchRetryPolicy retries on the bad / lagging region
// for sessionTokenMismatchRetryAttempts by tracking the badSessionTokenRule hit count
long hitCount = badSessionTokenRule.GetHitCount();
if (remoteRegionPreferred)
{
// higher hit count is possible while in MinRetryWaitTimeWithinRegion
Assert.IsTrue(hitCount >= sessionTokenMismatchRetryAttempts);
}

};

OperationExecutionResult executionResult = await this.PerformDocumentOperation(faultInjectionOperationType, container, testObject);
this.ValidateOperationExecutionResult(executionResult, remoteRegionPreferred);

// For a write operation, the request can just go to the primary replica
// Check if the SessionTokenMismatchRetryPolicy retries on the bad / lagging region
// for sessionTokenMismatchRetryAttempts by tracking the badSessionTokenRule hit count
long hitCount = badSessionTokenRule.GetHitCount();
if (remoteRegionPreferred)
{
// higher hit count is possible while in MinRetryWaitTimeWithinRegion
Assert.IsTrue(hitCount >= sessionTokenMismatchRetryAttempts);
}

}
finally
{
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, null);
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, null);
}
}



private void ValidateOperationExecutionResult(OperationExecutionResult operationExecutionResult, Boolean remoteRegionPreferred)
{
int sessionTokenMismatchDefaultWaitTime = 5000;
Expand Down
93 changes: 43 additions & 50 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/SessionRetryOptionsUnitTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,77 +14,70 @@
public class SessionRetryOptionsUnitTests
{
[TestMethod]
public void SessionRetryOptionsDefaultValuesTest()
public void SessionRetryOptionsValidValuesTest()
{
CosmosClientOptions clientOptions = new CosmosClientOptions()
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, "200");
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, "1");
try
{
SessionRetryOptions = new SessionRetryOptions()
CosmosClientOptions clientOptions = new CosmosClientOptions()
{
RemoteRegionPreferred = true
},
};


Assert.IsTrue(clientOptions.SessionRetryOptions.MinInRegionRetryTime == ConfigurationManager.GetMinRetryTimeInLocalRegionWhenRemoteRegionPreferred());
Assert.IsTrue(clientOptions.SessionRetryOptions.MaxInRegionRetryCount == ConfigurationManager.GetMaxRetriesInLocalRegionWhenRemoteRegionPreferred());
SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = true
},
};

Assert.IsTrue(clientOptions.SessionRetryOptions.MinInRegionRetryTime == TimeSpan.FromMilliseconds(200));
Assert.IsTrue(clientOptions.SessionRetryOptions.MaxInRegionRetryCount == 1);
}
finally
{
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, null);
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, null);
}

}

[TestMethod]
public void SessionRetryOptionsCustomValuesTest()
public void SessionRetryOptionsDefaultValuesTest()
{
CosmosClientOptions clientOptions = new CosmosClientOptions()
{
SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = true,
MinInRegionRetryTime = TimeSpan.FromSeconds(1),
MaxInRegionRetryCount = 3

},
};

Assert.IsTrue(clientOptions.SessionRetryOptions.MinInRegionRetryTime == TimeSpan.FromSeconds(1));
Assert.IsTrue(clientOptions.SessionRetryOptions.MaxInRegionRetryCount == 3);
Assert.IsTrue(clientOptions.SessionRetryOptions.MinInRegionRetryTime == TimeSpan.FromMilliseconds(500));
Assert.IsTrue(clientOptions.SessionRetryOptions.MaxInRegionRetryCount == 1);

}

[TestMethod]
public void SessionRetryOptionsMinMaxRetriesCountEnforcedTest()
public void SessionRetryOptionsInValidValuesTest()
{

ArgumentException argumentException = Assert.ThrowsException<ArgumentException>(() =>
new CosmosClientOptions()
{
SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = true,
MaxInRegionRetryCount = 0

},
}
);
Assert.IsNotNull(argumentException);

}


[TestMethod]
public void SessionRetryOptionsMinMinRetryTimeEnforcedTest()
{

ArgumentException argumentException = Assert.ThrowsException<ArgumentException>(() =>
new CosmosClientOptions()
{
SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = true,
MinInRegionRetryTime = TimeSpan.FromMilliseconds(99)

},
}
);
Assert.IsNotNull(argumentException);
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, "50");
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, "0");
try
{
CosmosClientOptions clientOptions = new CosmosClientOptions()
{
SessionRetryOptions = new SessionRetryOptions()
{
RemoteRegionPreferred = true
},
};

Assert.IsTrue(clientOptions.SessionRetryOptions.MinInRegionRetryTime == TimeSpan.FromMilliseconds(100));
Assert.IsTrue(clientOptions.SessionRetryOptions.MaxInRegionRetryCount == 1);
}
finally
{
Environment.SetEnvironmentVariable(ConfigurationManager.MinInRegionRetryTimeForWritesInMs, null);
Environment.SetEnvironmentVariable(ConfigurationManager.MaxRetriesInLocalRegionWhenRemoteRegionPreferred, null);
}

}

Expand Down