Skip to content

Change Feed Processor: Fixes LeaseLostException leaks on notification APIs #3401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ealsur merged 6 commits into from
Aug 24, 2022
Merged

Change Feed Processor: Fixes LeaseLostException leaks on notification APIs #3401

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
841b420
Removes leaks
ealsur Aug 22, 2022
ddaca26
Tests
ealsur Aug 22, 2022
0664117
Fixing exception logging
ealsur Aug 23, 2022
d6c98c1
Using pattern matching
ealsur Aug 23, 2022
37939da
Comments
ealsur Aug 23, 2022
3a8fdd0
Merge branch 'master' into users/ealsur/leaselost
ealsur Aug 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Tests
  • Loading branch information
@ealsur
ealsur committed Aug 22, 2022
commit ddaca26f71c440200deb6b471586887133315299
7 changes: 5 additions & 2 deletions ...s/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseManagerCosmosTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests
{
using System;
using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions;
Expand Down Expand Up @@ -310,7 +311,6 @@ public async Task ReleaseWhen404WithSomeSubstatusDoesThrow()
/// <summary>
/// Verifies that if the updater read a different Owner from the captured in memory, throws a LeaseLost
/// </summary>
[ExpectedException(typeof(LeaseLostException))]
[TestMethod]
public async Task IfOwnerChangedThrow()
{
Expand Down Expand Up @@ -350,7 +350,10 @@ public async Task IfOwnerChangedThrow()
options,
Mock.Of<RequestOptionsFactory>());

await documentServiceLeaseManagerCosmos.AcquireAsync(lease);
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => documentServiceLeaseManagerCosmos.AcquireAsync(lease));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.PreconditionFailed);
}

/// <summary>
Expand Down
24 changes: 15 additions & 9 deletions ...s/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseUpdaterCosmosTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ public async Task ThrowsAfterMaxRetries()
}

[TestMethod]
[ExpectedException(typeof(LeaseLostException))]
public async Task ThrowsOnConflict()
{
string itemId = "1";
Expand Down Expand Up @@ -209,15 +208,17 @@ public async Task ThrowsOnConflict()
});

DocumentServiceLeaseUpdaterCosmos updater = new DocumentServiceLeaseUpdaterCosmos(DocumentServiceLeaseUpdaterCosmosTests.GetMockedContainer(mockedItems));
DocumentServiceLease updatedLease = await updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
{
serverLease.Owner = "newHost";
return serverLease;
});
}));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.Conflict);
}

[TestMethod]
[ExpectedException(typeof(LeaseLostException))]
public async Task ThrowsOnNotFoundReplace()
{
string itemId = "1";
Expand Down Expand Up @@ -257,15 +258,17 @@ public async Task ThrowsOnNotFoundReplace()
});

DocumentServiceLeaseUpdaterCosmos updater = new DocumentServiceLeaseUpdaterCosmos(DocumentServiceLeaseUpdaterCosmosTests.GetMockedContainer(mockedItems));
DocumentServiceLease updatedLease = await updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
{
serverLease.Owner = "newHost";
return serverLease;
});
}));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.NotFound);
}

[TestMethod]
[ExpectedException(typeof(LeaseLostException))]
public async Task ThrowsOnNotFoundRead()
{
string itemId = "1";
Expand Down Expand Up @@ -302,11 +305,14 @@ public async Task ThrowsOnNotFoundRead()
});

DocumentServiceLeaseUpdaterCosmos updater = new DocumentServiceLeaseUpdaterCosmos(DocumentServiceLeaseUpdaterCosmosTests.GetMockedContainer(mockedItems));
DocumentServiceLease updatedLease = await updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => updater.UpdateLeaseAsync(leaseToUpdate, itemId, partitionKey, serverLease =>
{
serverLease.Owner = "newHost";
return serverLease;
});
}));

Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException
&& innerCosmosException.StatusCode == HttpStatusCode.NotFound);
}

private static ContainerInternal GetMockedContainer(Mock<ContainerInternal> mockedContainer)
Expand Down
61 changes: 61 additions & 0 deletions ...ft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions;
using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement;
using Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing;
using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement;
Expand Down Expand Up @@ -317,6 +318,66 @@ public async Task Controller_ShouldNotify_IfProcessingFails_EvenOnLeaseLost()
.Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once);
}

[TestMethod]
public async Task AddLease_ShouldNotNotify_IfLeaseAcquireFailsWithLeaseLost_WithoutInnerException()
{
Mock.Get(this.partitionProcessor)
.Reset();

Mock<PartitionSupervisor> supervisor = new Mock<PartitionSupervisor>();

LeaseLostException exception = new LeaseLostException();

// Fail on Acquire
Mock.Get(this.leaseManager)
.Setup(manager => manager.AcquireAsync(this.lease))
.ThrowsAsync(exception);

Exception thrownException = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => this.sut.AddOrUpdateLeaseAsync(this.lease));

Assert.AreEqual(exception, thrownException);

this.healthMonitor
.Verify(m => m.NotifyErrorAsync(this.lease.CurrentLeaseToken, exception), Times.Never);

Mock.Get(this.leaseManager)
.Verify(manager => manager.ReleaseAsync(this.lease), Times.Once);

this.healthMonitor
.Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once);
}

[TestMethod]
public async Task AddLease_ShouldNotify_IfLeaseAcquireFailsWithLeaseLost_WithInnerException()
{
Mock.Get(this.partitionProcessor)
.Reset();

Mock<PartitionSupervisor> supervisor = new Mock<PartitionSupervisor>();

Exception internalException = new Exception();

LeaseLostException exception = new LeaseLostException("some error", internalException);

// Fail on Acquire
Mock.Get(this.leaseManager)
.Setup(manager => manager.AcquireAsync(this.lease))
.ThrowsAsync(exception);

Exception thrownException = await Assert.ThrowsExceptionAsync<LeaseLostException>(() => this.sut.AddOrUpdateLeaseAsync(this.lease));

Assert.AreEqual(exception, thrownException);

this.healthMonitor
.Verify(m => m.NotifyErrorAsync(this.lease.CurrentLeaseToken, internalException), Times.Once);

Mock.Get(this.leaseManager)
.Verify(manager => manager.ReleaseAsync(this.lease), Times.Once);

this.healthMonitor
.Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once);
}

[TestMethod]
public async Task AddLease_ShouldReleaseLease_IfLeaseAcquireThrows()
{
Expand Down