-
Notifications
You must be signed in to change notification settings - Fork 524
Query: Adds hybrid search query pipeline stage #4794
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
microsoft-github-policy-service
merged 15 commits into
master
from
users/ndeshpan/hybridSearch
Oct 18, 2024
Merged
Changes from 8 commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
c2f9a00
Draft of Hybrid Search implementation
neildsh 35b3e95
A bunch of bug fixes for Hybrid search, and the first succesful E2E r…
neildsh f7fcdac
Add full debug tracing for hybrid search
neildsh 94b14ba
Minor clean up and fix regression caused by moving SqlQuerySpec rewrite
neildsh 50c151e
mark hybrid search integration tests as ignore while we wait for the …
neildsh 4f9f5cd
Add optimization for single component pipeline
neildsh 4e6355e
Add an oracle for hybrid search in the form of Lucene
neildsh fa9e5ff
Prevent build break due to lucene
neildsh 828952f
remove skip take counter from hybrid search query pipeline stage
neildsh 923e52e
fix up build break
neildsh 4b3b206
Minor cleanup in CosmosQueryExecutionContextFactory
neildsh 9498e64
incorporate code review feedback
neildsh 4e34937
More lucene junk
neildsh cb5c5f0
Remove references to Lucene from the project
neildsh 143b77e
Merge branch 'master' into users/ndeshpan/hybridSearch
neildsh File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
64 changes: 64 additions & 0 deletions
64
...ft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HybridSearch/FullTextStatistics.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| // ------------------------------------------------------------ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // ------------------------------------------------------------ | ||
|
|
||
| namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.CrossPartition.HybridSearch | ||
| { | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using Microsoft.Azure.Cosmos.CosmosElements; | ||
|
|
||
| internal sealed class FullTextStatistics | ||
| { | ||
| public long TotalWordCount { get; } | ||
|
|
||
| public ReadOnlyMemory<long> HitCounts => this.hitCounts; | ||
|
|
||
| private readonly long[] hitCounts; | ||
|
|
||
| public FullTextStatistics(long totalWordCount, long[] hitCounts) | ||
| { | ||
| this.TotalWordCount = totalWordCount; | ||
| this.hitCounts = hitCounts; | ||
| } | ||
|
|
||
| public FullTextStatistics(CosmosObject cosmosObject) | ||
| { | ||
| if (cosmosObject == null) | ||
| { | ||
| throw new System.ArgumentNullException($"{nameof(cosmosObject)} must not be null."); | ||
| } | ||
|
|
||
| if (!cosmosObject.TryGetValue(FieldNames.TotalWordCount, out CosmosNumber totalWordCount)) | ||
| { | ||
| throw new System.ArgumentException($"{FieldNames.TotalWordCount} must exist and be a number"); | ||
| } | ||
|
|
||
| if (!cosmosObject.TryGetValue(FieldNames.HitCounts, out CosmosArray hitCountsArray)) | ||
| { | ||
| throw new System.ArgumentException($"{FieldNames.HitCounts} must exist and be an array"); | ||
| } | ||
|
|
||
| long[] hitCounts = new long[hitCountsArray.Count]; | ||
| for (int index = 0; index < hitCountsArray.Count; ++index) | ||
| { | ||
| if (!(hitCountsArray[index] is CosmosNumber cosmosNumber)) | ||
| { | ||
| throw new System.ArgumentException($"{FieldNames.HitCounts} must be an array of numbers"); | ||
| } | ||
|
|
||
| hitCounts[index] = Number64.ToLong(cosmosNumber.Value); | ||
| } | ||
|
|
||
| this.TotalWordCount = Number64.ToLong(totalWordCount.Value); | ||
| this.hitCounts = hitCounts; | ||
| } | ||
|
|
||
| private static class FieldNames | ||
| { | ||
| public const string TotalWordCount = "totalWordCount"; | ||
|
|
||
| public const string HitCounts = "hitCounts"; | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.