Skip to content

Change Feed Processor: Fixes LeaseLostException leaks on notification APIs #3401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ealsur merged 6 commits into from
Aug 24, 2022
Merged

Change Feed Processor: Fixes LeaseLostException leaks on notification APIs #3401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
841b420
Removes leaks
ealsur Aug 22, 2022
ddaca26
Tests
ealsur Aug 22, 2022
0664117
Fixing exception logging
ealsur Aug 23, 2022
d6c98c1
Using pattern matching
ealsur Aug 23, 2022
37939da
Comments
ealsur Aug 23, 2022
3a8fdd0
Merge branch 'master' into users/ealsur/leaselost
ealsur Aug 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,22 @@ public override async Task AddOrUpdateLeaseAsync(DocumentServiceLease lease)
catch (Exception ex)
{
await this.RemoveLeaseAsync(lease: lease, wasAcquired: false).ConfigureAwait(false);
await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex);
switch (ex)
{
case LeaseLostException leaseLostException:
// LeaseLostException by itself is not loggable, unless it contains a related inner exception
// For cases when the lease or container has been deleted or the lease has been stolen
if (leaseLostException.InnerException != null)
{
await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, leaseLostException.InnerException);
}
break;

default:
await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex);
break;
}

throw;
}

Expand Down Expand Up @@ -118,7 +133,7 @@ private async Task RemoveLeaseAsync(DocumentServiceLease lease, bool wasAcquired
await this.monitor.NotifyLeaseReleaseAsync(lease.CurrentLeaseToken);
}

DefaultTrace.TraceVerbose("Lease with token {0}: taken by another host during release");
DefaultTrace.TraceVerbose("Lease with token {0}: taken by another host during release", lease.CurrentLeaseToken);
}
catch (Exception ex)
{
Expand Down
13 changes: 12 additions & 1 deletion ...Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseManagerCosmos.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement
using Microsoft.Azure.Cosmos.Core.Trace;
using Microsoft.Azure.Cosmos.Query.Core;
using Microsoft.Azure.Cosmos.Query.Core.Monads;
using Microsoft.Azure.Cosmos.Resource.CosmosExceptions;
using Microsoft.Azure.Cosmos.Routing;
using Microsoft.Azure.Cosmos.Tracing;
using Microsoft.Azure.Documents;
Expand Down Expand Up @@ -89,7 +90,17 @@ public override async Task<DocumentServiceLease> AcquireAsync(DocumentServiceLea
if (serverLease.Owner != oldOwner)
{
DefaultTrace.TraceInformation("{0} lease token was taken over by owner '{1}'", lease.CurrentLeaseToken, serverLease.Owner);
throw new LeaseLostException(lease);
throw new LeaseLostException(
lease,
CosmosExceptionFactory.Create(
statusCode: HttpStatusCode.PreconditionFailed,
message: $"{lease.CurrentLeaseToken} lease token was taken over by owner '{serverLease.Owner}'",
headers: new Headers(),
stackTrace: default,
trace: NoOpTrace.Singleton,
error: default,
innerException: default),
isGone: false);
}
serverLease.Owner = this.options.HostName;
serverLease.Properties = lease.Properties;
Expand Down
6 changes: 3 additions & 3 deletions ...Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseUpdaterCosmos.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public override async Task<DocumentServiceLease> UpdateLeaseAsync(
catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
DefaultTrace.TraceInformation("Lease with token {0} no longer exists", lease.CurrentLeaseToken);
throw new LeaseLostException(lease, true);
throw new LeaseLostException(lease: lease, innerException: ex, isGone: true);
}
}

Expand Down Expand Up @@ -94,7 +94,7 @@ private async Task<DocumentServiceLease> TryReplaceLeaseAsync(
DefaultTrace.TraceWarning("Lease operation exception, status code: {0}", ex.StatusCode);
if (ex.StatusCode == HttpStatusCode.NotFound)
{
throw new LeaseLostException(lease, true);
throw new LeaseLostException(lease: lease, innerException: ex, isGone: true);
}

if (ex.StatusCode == HttpStatusCode.PreconditionFailed)
Expand All @@ -104,7 +104,7 @@ private async Task<DocumentServiceLease> TryReplaceLeaseAsync(

if (ex.StatusCode == HttpStatusCode.Conflict)
{
throw new LeaseLostException(lease, ex, false);
throw new LeaseLostException(lease: lease, innerException: ex, isGone: false);
}

throw;
Expand Down
7 changes: 5 additions & 2 deletions ...s/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseManagerCosmosTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests
{
using System;
using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions;
Expand Down Expand Up @@ -310,7 +311,6 @@ public async Task ReleaseWhen404WithSomeSubstatusDoesThrow()
/// <summary>
/// Verifies that if the updater read a different Owner from the captured in memory, throws a LeaseLost
/// </summary>
[ExpectedException(typeof(LeaseLostException))]
[TestMethod]
public async Task IfOwnerChangedThrow()
{
Expand Down Expand Up @@ -350,7 +350,10 @@ public async Task IfOwnerChangedThrow()
options,
Mock.Of<RequestOptionsFactory>());

await documentServiceLeaseManagerCosmos.AcquireAsync(lease);
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => documentServiceLeaseManagerCosmos.AcquireAsync(lease));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.PreconditionFailed);
}

/// <summary>
Expand Down
24 changes: 15 additions & 9 deletions ...s/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseUpdaterCosmosTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ public async Task ThrowsAfterMaxRetries()
}

[TestMethod]
[ExpectedException(typeof(LeaseLostException))]
public async Task ThrowsOnConflict()
{
string itemId = "1";
Expand Down Expand Up @@ -209,15 +208,17 @@ public async Task ThrowsOnConflict()
});

DocumentServiceLeaseUpdaterCosmos updater = new DocumentServiceLeaseUpdaterCosmos(DocumentServiceLeaseUpdaterCosmosTests.GetMockedContainer(mockedItems));
DocumentServiceLease updatedLease = await updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
{
serverLease.Owner = "newHost";
return serverLease;
});
}));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.Conflict);
}

[TestMethod]
[ExpectedException(typeof(LeaseLostException))]
public async Task ThrowsOnNotFoundReplace()
{
string itemId = "1";
Expand Down Expand Up @@ -257,15 +258,17 @@ public async Task ThrowsOnNotFoundReplace()
});

DocumentServiceLeaseUpdaterCosmos updater = new DocumentServiceLeaseUpdaterCosmos(DocumentServiceLeaseUpdaterCosmosTests.GetMockedContainer(mockedItems));
DocumentServiceLease updatedLease = await updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
{
serverLease.Owner = "newHost";
return serverLease;
});
}));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.NotFound);
}

[TestMethod]
[ExpectedException(typeof(LeaseLostException))]
public async Task ThrowsOnNotFoundRead()
{
string itemId = "1";
Expand Down Expand Up @@ -302,11 +305,14 @@ public async Task ThrowsOnNotFoundRead()
});

DocumentServiceLeaseUpdaterCosmos updater = new DocumentServiceLeaseUpdaterCosmos(DocumentServiceLeaseUpdaterCosmosTests.GetMockedContainer(mockedItems));
DocumentServiceLease updatedLease = await updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
{
serverLease.Owner = "newHost";
return serverLease;
});
}));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.NotFound);
}

private static ContainerInternal GetMockedContainer(Mock<ContainerInternal> mockedContainer)
Expand Down
61 changes: 61 additions & 0 deletions ...ft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions;
using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement;
using Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing;
using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement;
Expand Down Expand Up @@ -317,6 +318,66 @@ public async Task Controller_ShouldNotify_IfProcessingFails_EvenOnLeaseLost()
.Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once);
}

[TestMethod]
public async Task AddLease_ShouldNotNotify_IfLeaseAcquireFailsWithLeaseLost_WithoutInnerException()
{
Mock.Get(this.partitionProcessor)
.Reset();

Mock<PartitionSupervisor> supervisor = new Mock<PartitionSupervisor>();

LeaseLostException exception = new LeaseLostException();

// Fail on Acquire
Mock.Get(this.leaseManager)
.Setup(manager => manager.AcquireAsync(this.lease))
.ThrowsAsync(exception);

Exception thrownException = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => this.sut.AddOrUpdateLeaseAsync(this.lease));

Assert.AreEqual(exception, thrownException);

this.healthMonitor
.Verify(m => m.NotifyErrorAsync(this.lease.CurrentLeaseToken, exception), Times.Never);

Mock.Get(this.leaseManager)
.Verify(manager => manager.ReleaseAsync(this.lease), Times.Once);

this.healthMonitor
.Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once);
}

[TestMethod]
public async Task AddLease_ShouldNotify_IfLeaseAcquireFailsWithLeaseLost_WithInnerException()
{
Mock.Get(this.partitionProcessor)
.Reset();

Mock<PartitionSupervisor> supervisor = new Mock<PartitionSupervisor>();

Exception internalException = new Exception();

LeaseLostException exception = new LeaseLostException("some error", internalException);

// Fail on Acquire
Mock.Get(this.leaseManager)
.Setup(manager => manager.AcquireAsync(this.lease))
.ThrowsAsync(exception);

Exception thrownException = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => this.sut.AddOrUpdateLeaseAsync(this.lease));

Assert.AreEqual(exception, thrownException);

this.healthMonitor
.Verify(m => m.NotifyErrorAsync(this.lease.CurrentLeaseToken, internalException), Times.Once);

Mock.Get(this.leaseManager)
.Verify(manager => manager.ReleaseAsync(this.lease), Times.Once);

this.healthMonitor
.Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once);
}

[TestMethod]
public async Task AddLease_ShouldReleaseLease_IfLeaseAcquireThrows()
{
Expand Down