From d1a86fbc8cb566739790576f3a5e07acb4801f1f Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Fri, 7 Nov 2025 10:30:04 -0800 Subject: [PATCH 1/9] LogScan new blobs between the start of polling and the LMT of the last LogScan --- .../src/Listeners/ContainerScanInfo.cs | 2 ++ .../Listeners/ScanBlobScanLogHybridPollingStrategy.cs | 9 ++++++--- .../ScanBlobScanLogHybridPollingStrategyTests.cs | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs index a474cbc76c6b..24c4f5e46acb 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs @@ -11,6 +11,8 @@ internal class ContainerScanInfo { public ICollection> Registrations { get; set; } + public DateTimeOffset PollingStartTime { get; set; } + public DateTime LastSweepCycleLatestModified { get; set; } public DateTime CurrentSweepCycleLatestModified { get; set; } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index ed6099a35e0c..6ca4306d7710 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -217,9 +217,10 @@ public async Task> PollNewBlobsAsync( string continuationToken = containerScanInfo.ContinuationToken; Page page; - // if starting the cycle, reset the sweep time + // if starting the cycle, reset the sweep time and set start time if (continuationToken == null) { + containerScanInfo.PollingStartTime = DateTimeOffset.UtcNow; containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue; } @@ -266,14 +267,16 @@ public async Task> PollNewBlobsAsync( var properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; - if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified) + if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && + lastModifiedTimestamp <= containerScanInfo.PollingStartTime) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } // Blob timestamps are rounded to the nearest second, so make sure we continue to check // the previous timestamp to catch any blobs that came in slightly after our previous poll. - if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified) + if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified && + lastModifiedTimestamp <= containerScanInfo.PollingStartTime) { newBlobs.Add(container.GetBlobClient(currentBlob.Name)); } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs index 13e57b43dc7f..eb8ef1ac626b 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs @@ -370,7 +370,7 @@ public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure() int testScanBlobLimitPerPoll = 6; // we'll introduce multiple errors to make sure we take the earliest timestamp - DateTime earliestErrorTime = DateTime.UtcNow; + DateTimeOffset earliestErrorTime = DateTimeOffset.UtcNow.AddHours(-1); var container = _blobContainerMock.Object; From a098b2c8a225906c08987005f11ce1fa68b4bf0b Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Tue, 18 Nov 2025 12:54:50 -0800 Subject: [PATCH 2/9] Update changelog; Update scanner to check if cont token is null before setting current lmt time --- .../CHANGELOG.md | 1 + .../src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md index dc02e9eda731..07cdcfba6db9 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md @@ -7,6 +7,7 @@ ### Breaking Changes ### Bugs Fixed +- Fixed a bug where polling for new blobs fails to detect newly created blobs when the List Blobs/Get Blobs operation requires multiple requests. If a blob is added or modified after LogScan begins, and is listed during the later portion of the listing, any blobs changed between the listing start time and the last modified timestamp of blobs found later will not be flagged as new in the subsequent LogScan. ### Other Changes diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index 6ca4306d7710..309c2a57ffcc 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -268,7 +268,7 @@ public async Task> PollNewBlobsAsync( DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && - lastModifiedTimestamp <= containerScanInfo.PollingStartTime) + (continuationToken == null || lastModifiedTimestamp <= containerScanInfo.PollingStartTime)) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } From 540ec536c798d94b0aed2541283c0c83cb9966c1 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 3 Dec 2025 14:59:03 -0800 Subject: [PATCH 3/9] Completed test for polling blob with different LMT --- ...anBlobScanLogHybridPollingStrategyTests.cs | 86 +++++++++++++++++++ .../TestAsyncPageableWithContinuationToken.cs | 35 ++++++++ 2 files changed, 121 insertions(+) create mode 100644 sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs index eb8ef1ac626b..70ef180af364 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs @@ -14,6 +14,7 @@ using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; +using Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Tests; using Microsoft.Azure.WebJobs.Extensions.Storage.Common.Listeners; using Microsoft.Azure.WebJobs.Extensions.Storage.Common.Tests; using Microsoft.Azure.WebJobs.Host.Executors; @@ -410,6 +411,91 @@ public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure() Times.Exactly(2)); } + [Test] + public async Task ExecuteAsync_PollNewBlobsAsync_ContinuationTokenUpdatedBlobs() + { + // Arrange + int testScanBlobLimitPerPoll = 3; + IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger.Instance); + LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); + typeof(ScanBlobScanLogHybridPollingStrategy) + .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) + .SetValue(product, testScanBlobLimitPerPoll); + + // Setup container to have multiple GetBlobsAsync calls to simulate continuation tokens + Uri uri = new Uri("https://fakeaccount.blob.core.windows.net/fakecontainer2"); + Mock containerMock = new Mock(uri, null); + containerMock.Setup(x => x.Uri).Returns(uri); + containerMock.Setup(x => x.Name).Returns(ContainerName); + containerMock.Setup(x => x.AccountName).Returns(AccountName); + + // Create first pages of blob to list from + List blobItems = new List(); + List expectedNames = new List(); + for (int i = 0; i < 5; i++) + { + DateTimeOffset lastModified = DateTimeOffset.UtcNow.AddMinutes(-10 * i); + expectedNames.Add(CreateBlobAndUploadToContainer(containerMock, blobItems, lastModified: lastModified)); + } + // Create second page + List blobItemsPage2 = new List(); + for (int i = 0; i < 3; i++) + { + DateTimeOffset lastModified = DateTimeOffset.UtcNow.AddMinutes(-5 * i); + expectedNames.Add(CreateBlobAndUploadToContainer(containerMock, blobItemsPage2, lastModified: lastModified)); + } + + // Add at least one blob that has a LastModifiedTime that goes beyond the start time of polling + DateTimeOffset lastModifiedAfterStartPolling = DateTimeOffset.UtcNow.AddSeconds(5); + string blobNameWithLmtAfterStartedPolling = CreateBlobAndUploadToContainer(containerMock, blobItemsPage2, lastModified: lastModifiedAfterStartPolling); + + // Update all the blobs in the second listing, that way they get detected again in the second polling + List blobItemsUpdated = new List(); + List secondSetExpectedNames = new List(); + for (int i = 0; i < 4; i++) + { + // Create LastModified to be after the LMT of the blob that was beyond the start of the polling time to test if blobs created after that time will also be detected + secondSetExpectedNames.Add(CreateBlobAndUploadToContainer(containerMock, blobItemsUpdated, lastModified: lastModifiedAfterStartPolling.AddSeconds(-2))); + } + // Add blob with LMT after polling started to the second polling expected names. + secondSetExpectedNames.Add(blobNameWithLmtAfterStartedPolling); + + // Set up GetBlobsAsync to return pages with contination token for each page, but not at the end of each polling + containerMock.SetupSequence(x => x.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + // First polling + .Returns(() => + { + return new TestAsyncPageableWithContinuationToken(blobItems, true); + }) + .Returns(() => + { + return new TestAsyncPageableWithContinuationToken(blobItemsPage2, false); + }) + // Second polling + .Returns(() => + { + return new TestAsyncPageableWithContinuationToken(blobItemsUpdated, true); + }) + .Returns(() => + { + return new TestAsyncPageableWithContinuationToken(blobItemsPage2, false); + }); + + // Register the container to initialize _scanInfo + await product.RegisterAsync(_blobClientMock.Object, containerMock.Object, executor, CancellationToken.None); + + // Act / Assert - First Polling + RunExecuteWithMultiPollingInterval(expectedNames, product, executor, blobItems.Count); + + // Wait 5 seconds to ensure that the blob with LMT after polling started is detected as a new blob. + Thread.Sleep(5000); + + // Act / Assert - Second Polling + // We expect all the blobs we updated above to be detected and the blob that was created after the first polling started that wasn't detected + // to be now detected in this polling. + RunExecuteWithMultiPollingInterval(secondSetExpectedNames, product, executor, blobItemsUpdated.Count); + } + private void RunExecuterWithExpectedBlobsInternal(IDictionary blobNameMap, IBlobListenerStrategy product, LambdaBlobTriggerExecutor executor, int expectedCount) { if (blobNameMap.Count == 0) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs new file mode 100644 index 000000000000..8ab8e5d092e8 --- /dev/null +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure; +using Azure.Storage.Blobs.Models; +using Moq; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Tests +{ + public class TestAsyncPageableWithContinuationToken : AsyncPageable + { + private readonly List _page; + private readonly bool _returnsContinuationToken; + + public TestAsyncPageableWithContinuationToken(List page, bool returnsContinuationToken) + { + _page = page; + _returnsContinuationToken = returnsContinuationToken; + } + + public override async IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) + { + string mockContinuationToken = System.Guid.NewGuid().ToString(); + yield return Page.FromValues( + _page.AsReadOnly(), + _returnsContinuationToken ? mockContinuationToken : null, + Mock.Of()); + // Simulate async page boundary + await Task.Yield(); + } + } +} From c68a7b49abb2c6a564ddafe47c36cd9c50b6bac2 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 3 Dec 2025 15:52:10 -0800 Subject: [PATCH 4/9] Updated changelog to hopefully better phrasing --- .../CHANGELOG.md | 2 +- ...icrosoft.Azure.WebJobs.Extensions.Storage.sln | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md index 07cdcfba6db9..0db09f3b9031 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/CHANGELOG.md @@ -7,7 +7,7 @@ ### Breaking Changes ### Bugs Fixed -- Fixed a bug where polling for new blobs fails to detect newly created blobs when the List Blobs/Get Blobs operation requires multiple requests. If a blob is added or modified after LogScan begins, and is listed during the later portion of the listing, any blobs changed between the listing start time and the last modified timestamp of blobs found later will not be flagged as new in the subsequent LogScan. +- Fixed an issue where polling for new blobs could miss recently created or updated blobs when the List Blobs/Get Blobs operation spans multiple requests. If a blob is added or modified after LogScan starts and appears in a later segment of the listing, any blobs changed between the initial listing start time and the last modified timestamp of those later blobs were not flagged as new in the subsequent LogScan. ### Other Changes diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln index f36c918f96be..5e0c8ba0aabd 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30011.22 +# Visual Studio Version 17 +VisualStudioVersion = 17.14.36705.20 d17.14 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution", "Solution", "{3B089351-285A-4829-8F60-30768A7469BD}" ProjectSection(SolutionItems) = preProject @@ -42,6 +42,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.WebJobs.Ext EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Samples.Function.App", "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs\samples\functionapp\Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Samples.Function.App.csproj", "{22D974DB-0B34-48F5-90EC-F65F9902E7D3}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Core", "..\core\Azure.Core\src\Azure.Core.csproj", "{DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -116,10 +120,10 @@ Global {22D974DB-0B34-48F5-90EC-F65F9902E7D3}.Debug|Any CPU.Build.0 = Debug|Any CPU {22D974DB-0B34-48F5-90EC-F65F9902E7D3}.Release|Any CPU.ActiveCfg = Release|Any CPU {22D974DB-0B34-48F5-90EC-F65F9902E7D3}.Release|Any CPU.Build.0 = Release|Any CPU - {03A5BF93-27C7-4923-B006-004DFD0795C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {03A5BF93-27C7-4923-B006-004DFD0795C0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {03A5BF93-27C7-4923-B006-004DFD0795C0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {03A5BF93-27C7-4923-B006-004DFD0795C0}.Release|Any CPU.Build.0 = Release|Any CPU + {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From e1228157d375234e9878cd8d01c8e7ba4e5f9592 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 3 Dec 2025 16:28:30 -0800 Subject: [PATCH 5/9] Copilot comments + changed DateTime usage to DateTimeOffset --- .../src/Listeners/ContainerScanInfo.cs | 4 +-- .../src/Listeners/IBlobScanInfoManager.cs | 4 +-- ...BlobScanLogHybridPollingStrategy.Logger.cs | 4 +-- .../ScanBlobScanLogHybridPollingStrategy.cs | 6 ++-- .../Listeners/StorageBlobScanInfoManager.cs | 8 ++--- ...anBlobScanLogHybridPollingStrategyTests.cs | 30 +++++++++---------- .../StorageBlobScanInfoManagerTests.cs | 6 ++-- ...s.Extensions.Storage.Scenario.Tests.csproj | 1 - 8 files changed, 31 insertions(+), 32 deletions(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs index 24c4f5e46acb..8763e67bc61f 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ContainerScanInfo.cs @@ -13,9 +13,9 @@ internal class ContainerScanInfo public DateTimeOffset PollingStartTime { get; set; } - public DateTime LastSweepCycleLatestModified { get; set; } + public DateTimeOffset LastSweepCycleLatestModified { get; set; } - public DateTime CurrentSweepCycleLatestModified { get; set; } + public DateTimeOffset CurrentSweepCycleLatestModified { get; set; } public string ContinuationToken { get; set; } } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/IBlobScanInfoManager.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/IBlobScanInfoManager.cs index 7876884bdf8e..547b7d8ce1e8 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/IBlobScanInfoManager.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/IBlobScanInfoManager.cs @@ -8,7 +8,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Listeners { internal interface IBlobScanInfoManager { - Task LoadLatestScanAsync(string storageAccountName, string containerName); - Task UpdateLatestScanAsync(string storageAccountName, string containerName, DateTime scanInfo); + Task LoadLatestScanAsync(string storageAccountName, string containerName); + Task UpdateLatestScanAsync(string storageAccountName, string containerName, DateTimeOffset scanInfo); } } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.Logger.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.Logger.cs index d1318bfa62f0..4d5ae525f0d9 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.Logger.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.Logger.cs @@ -26,10 +26,10 @@ private static class Logger LoggerMessage.Define(LogLevel.Debug, new EventId(3, nameof(ContainerDoesNotExist)), "Container '{containerName}' does not exist."); - public static void InitializedScanInfo(ILogger logger, string container, DateTime latestScanInfo) => + public static void InitializedScanInfo(ILogger logger, string container, DateTimeOffset latestScanInfo) => _initializedScanInfo(logger, container, latestScanInfo.ToString(Constants.DateTimeFormatString, CultureInfo.InvariantCulture), null); - public static void PollBlobContainer(ILogger logger, string container, DateTime latestScanInfo, string clientRequestId, int blobCount, long latencyInMilliseconds, bool hasContinuationToken) => + public static void PollBlobContainer(ILogger logger, string container, DateTimeOffset latestScanInfo, string clientRequestId, int blobCount, long latencyInMilliseconds, bool hasContinuationToken) => _pollBlobContainer(logger, latestScanInfo.ToString(Constants.DateTimeFormatString, CultureInfo.InvariantCulture), container, clientRequestId, blobCount, latencyInMilliseconds, hasContinuationToken, null); public static void ContainerDoesNotExist(ILogger logger, string container) => diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index 309c2a57ffcc..7754c1b79f78 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -73,7 +73,7 @@ public async Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContain if (!_scanInfo.TryGetValue(container, out ContainerScanInfo containerScanInfo)) { // First, try to load serialized scanInfo for this container. - DateTime? latestStoredScan = await _blobScanInfoManager.LoadLatestScanAsync(blobServiceClient.AccountName, container.Name).ConfigureAwait(false); + DateTimeOffset? latestStoredScan = await _blobScanInfoManager.LoadLatestScanAsync(blobServiceClient.AccountName, container.Name).ConfigureAwait(false); containerScanInfo = new ContainerScanInfo() { @@ -138,7 +138,7 @@ private async Task PollAndNotify(BlobContainerClient container, ContainerScanInf List failedNotifications, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - DateTime lastScan = containerScanInfo.LastSweepCycleLatestModified; + DateTimeOffset lastScan = containerScanInfo.LastSweepCycleLatestModified; // For tracking string clientRequestId = Guid.NewGuid().ToString(); @@ -154,7 +154,7 @@ private async Task PollAndNotify(BlobContainerClient container, ContainerScanInf // if the 'LatestModified' has changed, update it in the manager if (containerScanInfo.LastSweepCycleLatestModified > lastScan) { - DateTime latestScan = containerScanInfo.LastSweepCycleLatestModified; + DateTimeOffset latestScan = containerScanInfo.LastSweepCycleLatestModified; // It's possible that we had some blobs that we failed to move to the queue. We want to make sure // we continue to find these if the host needs to restart. diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/StorageBlobScanInfoManager.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/StorageBlobScanInfoManager.cs index c9f6cda466eb..336c4012e9e6 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/StorageBlobScanInfoManager.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Listeners/StorageBlobScanInfoManager.cs @@ -43,10 +43,10 @@ public StorageBlobScanInfoManager(string hostId, BlobServiceClient blobClient) _blobContainerClient = blobClient.GetBlobContainerClient(HostContainerNames.Hosts); } - public async Task LoadLatestScanAsync(string storageAccountName, string containerName) + public async Task LoadLatestScanAsync(string storageAccountName, string containerName) { var scanInfoBlob = GetScanInfoBlobReference(storageAccountName, containerName); - DateTime? latestScan = null; + DateTimeOffset? latestScan = null; try { string scanInfoLine = await scanInfoBlob.DownloadTextAsync(CancellationToken.None).ConfigureAwait(false); @@ -74,7 +74,7 @@ public StorageBlobScanInfoManager(string hostId, BlobServiceClient blobClient) } } - public async Task UpdateLatestScanAsync(string storageAccountName, string containerName, DateTime latestScan) + public async Task UpdateLatestScanAsync(string storageAccountName, string containerName, DateTimeOffset latestScan) { string scanInfoLine; ScanInfo scanInfo = new ScanInfo @@ -109,7 +109,7 @@ private BlockBlobClient GetScanInfoBlobReference(string storageAccountName, stri internal class ScanInfo { - public DateTime LatestScan { get; set; } + public DateTimeOffset LatestScan { get; set; } } } } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs index 70ef180af364..13aae4d54c22 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/ScanBlobScanLogHybridPollingStrategyTests.cs @@ -307,7 +307,7 @@ public async Task RegisterAsync_InitializesWithScanInfoManager() // delay slightly so we guarantee a later timestamp await Task.Delay(10); - await scanInfoManager.UpdateLatestScanAsync(AccountName, ContainerName, DateTime.UtcNow); + await scanInfoManager.UpdateLatestScanAsync(AccountName, ContainerName, DateTimeOffset.UtcNow); await product.RegisterAsync(_blobClientMock.Object, container, executor, CancellationToken.None); // delay slightly so we guarantee a later timestamp @@ -404,7 +404,7 @@ public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure() RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll); - DateTime? storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, ContainerName); + DateTimeOffset? storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, ContainerName); Assert.True(storedTime < earliestErrorTime); Assert.AreEqual(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]); _blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), @@ -429,7 +429,7 @@ public async Task ExecuteAsync_PollNewBlobsAsync_ContinuationTokenUpdatedBlobs() containerMock.Setup(x => x.Name).Returns(ContainerName); containerMock.Setup(x => x.AccountName).Returns(AccountName); - // Create first pages of blob to list from + // Create first page of blobs to list from List blobItems = new List(); List expectedNames = new List(); for (int i = 0; i < 5; i++) @@ -460,7 +460,7 @@ public async Task ExecuteAsync_PollNewBlobsAsync_ContinuationTokenUpdatedBlobs() // Add blob with LMT after polling started to the second polling expected names. secondSetExpectedNames.Add(blobNameWithLmtAfterStartedPolling); - // Set up GetBlobsAsync to return pages with contination token for each page, but not at the end of each polling + // Set up GetBlobsAsync to return pages with continuation token for each page, but not at the end of each polling containerMock.SetupSequence(x => x.GetBlobsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) // First polling .Returns(() => @@ -488,7 +488,7 @@ public async Task ExecuteAsync_PollNewBlobsAsync_ContinuationTokenUpdatedBlobs() RunExecuteWithMultiPollingInterval(expectedNames, product, executor, blobItems.Count); // Wait 5 seconds to ensure that the blob with LMT after polling started is detected as a new blob. - Thread.Sleep(5000); + await Task.Delay(TimeSpan.FromSeconds(5)); // Act / Assert - Second Polling // We expect all the blobs we updated above to be detected and the blob that was created after the first polling started that wasn't detected @@ -619,23 +619,23 @@ public Task ExecuteAsync(BlobTriggerExecutorContext value, Cance private class TestBlobScanInfoManager : IBlobScanInfoManager { - private IDictionary> _latestScans; + private IDictionary> _latestScans; public TestBlobScanInfoManager() { - _latestScans = new Dictionary>(); + _latestScans = new Dictionary>(); UpdateCounts = new Dictionary>(); } public IDictionary> UpdateCounts { get; private set; } - public Task LoadLatestScanAsync(string storageAccountName, string containerName) + public Task LoadLatestScanAsync(string storageAccountName, string containerName) { - DateTime? value = null; - IDictionary accounts; + DateTimeOffset? value = null; + IDictionary accounts; if (_latestScans.TryGetValue(storageAccountName, out accounts)) { - DateTime latestScan; + DateTimeOffset latestScan; if (accounts.TryGetValue(containerName, out latestScan)) { value = latestScan; @@ -645,20 +645,20 @@ public TestBlobScanInfoManager() return Task.FromResult(value); } - public Task UpdateLatestScanAsync(string storageAccountName, string containerName, DateTime latestScan) + public Task UpdateLatestScanAsync(string storageAccountName, string containerName, DateTimeOffset latestScan) { SetScanInfo(storageAccountName, containerName, latestScan); IncrementCount(storageAccountName, containerName); return Task.FromResult(0); } - public void SetScanInfo(string storageAccountName, string containerName, DateTime latestScan) + public void SetScanInfo(string storageAccountName, string containerName, DateTimeOffset latestScan) { - IDictionary containers; + IDictionary containers; if (!_latestScans.TryGetValue(storageAccountName, out containers)) { - _latestScans[storageAccountName] = new Dictionary(); + _latestScans[storageAccountName] = new Dictionary(); containers = _latestScans[storageAccountName]; } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs index b6c36147b1cb..02bbeed87c6a 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs @@ -106,8 +106,8 @@ public async Task UpdateLatestScan_Updates() var container = blobServiceClient.GetBlobContainerClient(HostContainerNames.Hosts); await container.CreateIfNotExistsAsync(); - DateTime now = DateTime.UtcNow; - DateTime past = now.AddMinutes(-1); + DateTimeOffset now = DateTimeOffset.UtcNow; + DateTimeOffset past = now.AddMinutes(-1); var blob = GetBlockBlobReference(blobServiceClient, hostId, storageAccountName, containerName); await blob.UploadTextAsync(string.Format("{{ 'LatestScan' : '{0}' }}", past.ToString("o"))); @@ -118,7 +118,7 @@ public async Task UpdateLatestScan_Updates() var scanInfo = GetBlockBlobReference(blobServiceClient, hostId, storageAccountName, containerName).DownloadText(); var scanInfoJson = JObject.Parse(scanInfo); - var storedTime = (DateTime)scanInfoJson["LatestScan"]; + var storedTime = (DateTimeOffset)scanInfoJson["LatestScan"]; Assert.AreEqual(now, storedTime); Assert.AreEqual(now, await manager.LoadLatestScanAsync(storageAccountName, containerName)); diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests.csproj b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests.csproj index 842bf92be72c..b7decbd728ea 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests.csproj +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests.csproj @@ -5,7 +5,6 @@ - From 20e3c5283828f206a0aca2279c00f195a3853e56 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 3 Dec 2025 16:40:01 -0800 Subject: [PATCH 6/9] Add docs for TestAsyncPageableWithContinuationToken --- .../tests/TestAsyncPageableWithContinuationToken.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs index 8ab8e5d092e8..45cf020bfb33 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/TestAsyncPageableWithContinuationToken.cs @@ -10,6 +10,11 @@ namespace Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Tests { + /// + /// This is based on the TestAsyncPageable in Common tests. It adds the ability to specify whether a continuation token is returned. + /// This will allow the strategy to determine whether there are more pages and continue listing. + /// It will NOT use the passed in continuation token to determine what to return; it always returns the page. + /// public class TestAsyncPageableWithContinuationToken : AsyncPageable { private readonly List _page; From c93073440101504bd56d07e9efdea5d516b9f26b Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 3 Dec 2025 16:52:28 -0800 Subject: [PATCH 7/9] Undo changes to sln --- ...icrosoft.Azure.WebJobs.Extensions.Storage.sln | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln index 5e0c8ba0aabd..f36c918f96be 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.14.36705.20 d17.14 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30011.22 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution", "Solution", "{3B089351-285A-4829-8F60-30768A7469BD}" ProjectSection(SolutionItems) = preProject @@ -42,10 +42,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.WebJobs.Ext EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Samples.Function.App", "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs\samples\functionapp\Microsoft.Azure.WebJobs.Extensions.Storage.Blobs.Samples.Function.App.csproj", "{22D974DB-0B34-48F5-90EC-F65F9902E7D3}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Core", "..\core\Azure.Core\src\Azure.Core.csproj", "{DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -120,10 +116,10 @@ Global {22D974DB-0B34-48F5-90EC-F65F9902E7D3}.Debug|Any CPU.Build.0 = Debug|Any CPU {22D974DB-0B34-48F5-90EC-F65F9902E7D3}.Release|Any CPU.ActiveCfg = Release|Any CPU {22D974DB-0B34-48F5-90EC-F65F9902E7D3}.Release|Any CPU.Build.0 = Release|Any CPU - {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DDE0C4C4-BCA3-48A5-534B-1E2121291D0D}.Release|Any CPU.Build.0 = Release|Any CPU + {03A5BF93-27C7-4923-B006-004DFD0795C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03A5BF93-27C7-4923-B006-004DFD0795C0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03A5BF93-27C7-4923-B006-004DFD0795C0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03A5BF93-27C7-4923-B006-004DFD0795C0}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 61d855eba16d2e4d9f4a33c26519e656290c32f7 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 4 Dec 2025 16:15:28 -0800 Subject: [PATCH 8/9] Update to tests from DateTime to DateTimeOffset --- .../tests/Listeners/StorageBlobScanInfoManagerTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs index 02bbeed87c6a..64d433d8ff69 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs @@ -62,7 +62,7 @@ public async Task LoadLatestScan_Returns_Timestamp() var container = blobServiceClient.GetBlobContainerClient(HostContainerNames.Hosts); await container.CreateIfNotExistsAsync(); - DateTime now = DateTime.UtcNow; + DateTimeOffset now = DateTimeOffset.UtcNow; var blob = GetBlockBlobReference(blobServiceClient, hostId, storageAccountName, containerName); await blob.UploadTextAsync(string.Format("{{ \"LatestScan\" : \"{0}\" }}", now.ToString("o"))); @@ -82,7 +82,7 @@ public async Task UpdateLatestScan_Inserts() var container = blobServiceClient.GetBlobContainerClient(HostContainerNames.Hosts); await container.CreateIfNotExistsAsync(); - DateTime now = DateTime.UtcNow; + DateTimeOffset now = DateTimeOffset.UtcNow; var manager = new StorageBlobScanInfoManager(hostId, blobServiceClient); From b294bb72d62d9fbaddb93b20b5b4e935fd892f4a Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 4 Dec 2025 16:52:31 -0800 Subject: [PATCH 9/9] Fix test UpdateLatestScan_Inserts --- .../tests/Listeners/StorageBlobScanInfoManagerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs index 64d433d8ff69..86248e2f970a 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/Listeners/StorageBlobScanInfoManagerTests.cs @@ -90,7 +90,7 @@ public async Task UpdateLatestScan_Inserts() var scanInfo = GetBlockBlobReference(blobServiceClient, hostId, storageAccountName, containerName).DownloadText(); var scanInfoJson = JObject.Parse(scanInfo); - var storedTime = (DateTime)(scanInfoJson["LatestScan"]); + var storedTime = (DateTimeOffset)(scanInfoJson["LatestScan"]); Assert.AreEqual(now, storedTime); Assert.AreEqual(now, await manager.LoadLatestScanAsync(storageAccountName, containerName));