Skip to content

Query: Adds support for the optimized query plan that skips the order by rewrite #5121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
microsoft-github-policy-service merged 6 commits into from
May 6, 2025
Merged

Query: Adds support for the optimized query plan that skips the order by rewrite #5121

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
52d767f
Add support for the optimized query plan that skips the order by rewrite
neildsh Apr 1, 2025
fd9e8cc
Handle undefined values for payload in HybridSearchQueryResult
neildsh Apr 28, 2025
d163287
draft changes for unit tests
neildsh May 2, 2025
20e563f
Add more unit tests for weighted RRF with skip order by rewrite
neildsh May 5, 2025
9d90576
Bump up max concurrency for NonStreamingOrderByQueryTests
neildsh May 5, 2025
9924c6c
Disable the debug trace for NonStreamingOrderByQueryTests
neildsh May 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add more unit tests for weighted RRF with skip order by rewrite
  • Loading branch information
@neildsh
neildsh committed May 5, 2025
commit 20e563fd96e6249b630093fe29befcf2d7a1374f
75 changes: 41 additions & 34 deletions ...Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Pipeline/NonStreamingOrderByQueryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,32 +477,31 @@ public async Task HybridSearchSkipOrderByRewriteWeightedRRFTests()
weights: new double[] { 1.25, 2.0 },
pageSize: 10,
returnEmptyGlobalStatistics: true),

//MakeHybridSearchSkipOrderByRewriteTest(
// leafPageCount: 4,
// backendPageSize: 10,
// requiresGlobalStatistics: false,
// skip: null,
// take: 100,
// weights: new double[] { -1.0, -1.0 },
// pageSize: 1000),
//MakeHybridSearchSkipOrderByRewriteTest(
// leafPageCount: 4,
// backendPageSize: 100,
// requiresGlobalStatistics: true,
// skip: 7,
// take: 10,
// weights: new double[] { -1.33, -0.45 },
// pageSize: 1),
//MakeHybridSearchSkipOrderByRewriteTest(
// leafPageCount: 0,
// backendPageSize: 10,
// requiresGlobalStatistics: true,
// skip: 0,
// take: 10,
// weights: new double[] { -1.25, -2.0 },
// pageSize: 10,
// returnEmptyGlobalStatistics: true),
MakeHybridSearchSkipOrderByRewriteTest(
leafPageCount: 4,
backendPageSize: 10,
requiresGlobalStatistics: false,
skip: null,
take: 100,
weights: new double[] { -1.0, -1.0 },
pageSize: 1000),
MakeHybridSearchSkipOrderByRewriteTest(
leafPageCount: 4,
backendPageSize: 100,
requiresGlobalStatistics: true,
skip: 7,
take: 10,
weights: new double[] { -1.33, -0.45 },
pageSize: 1),
MakeHybridSearchSkipOrderByRewriteTest(
leafPageCount: 0,
backendPageSize: 10,
requiresGlobalStatistics: true,
skip: 0,
take: 10,
weights: new double[] { -1.25, -2.0 },
pageSize: 10,
returnEmptyGlobalStatistics: true),
};

foreach (HybridSearchTest testCase in testCases)
Expand Down Expand Up @@ -544,11 +543,11 @@ private static async Task RunHybridSearchTest(HybridSearchTest testCase)
{
expectedIndices = expectedIndices.Reverse();

feedModes = new PartitionedFeedMode[]
{
PartitionedFeedMode.NonStreaming,
PartitionedFeedMode.NonStreaming,
};
PartitionedFeedMode feedMode = testCase.SkipOrderByRewrite?
PartitionedFeedMode.NonStreamingReversed | PartitionedFeedMode.NegateScores:
PartitionedFeedMode.NonStreaming;

feedModes = new PartitionedFeedMode[] { feedMode, feedMode };
}
}

Expand Down Expand Up @@ -1586,6 +1585,8 @@ enum PartitionedFeedMode
NonStreaming = 1,
Reversed = 2,

NegateScores = 4,

StreamingReversed = Streaming | Reversed,
NonStreamingReversed = NonStreaming | Reversed,
}
Expand Down Expand Up @@ -1654,7 +1655,12 @@ private static IReadOnlyList<IReadOnlyDictionary<FeedRange, IReadOnlyList<IReadO
pageSize,
feedModes[componentIndex],
componentIndex,
(componentIndex, index) => CreateHybridSearchDocument(componentCount: componentCount, index:index, componentIndex: componentIndex, skipOrderByRewrite));
(componentIndex, index) => CreateHybridSearchDocument(
componentCount: componentCount,
index: index,
componentIndex: componentIndex,
skipOrderByRewrite: skipOrderByRewrite,
negateScores: feedModes[componentIndex].HasFlag(PartitionedFeedMode.NegateScores)));

componentPages.Add(pages);
}
Expand Down Expand Up @@ -1784,15 +1790,16 @@ private static CosmosElement CreateHybridSearchGlobalStatistics()
return globalStatistics;
}

private static CosmosElement CreateHybridSearchDocument(int componentCount, int index, int componentIndex, bool skipOrderByRewrite)
private static CosmosElement CreateHybridSearchDocument(int componentCount, int index, int componentIndex, bool skipOrderByRewrite, bool negateScores)
{
CosmosElement indexElement = CosmosNumber64.Create(index);
CosmosElement indexStringElement = CosmosString.Create(index.ToString("D4"));
double[] scores = new double[componentCount];
double delta = 0.1;
double factor = negateScores ? -1.0 : 1.0;
for (int scoreIndex = 0; scoreIndex < componentCount; ++scoreIndex)
{
scores[scoreIndex] = index + ((1 + scoreIndex) * delta);
scores[scoreIndex] = factor * (index + ((1 + scoreIndex) * delta));
}

List<CosmosElement> orderByItems = new List<CosmosElement>
Expand Down