Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
540a16d
Initial changes
aayush3011 Apr 24, 2024
6f49c75
Initial changes
aayush3011 Apr 25, 2024
97509eb
Merge branch 'main' into users/akataria/nonStreamingOrderBy
aayush3011 Apr 25, 2024
528a0eb
[Cosmos][VectorIndex]Adding changes for vectorIndex and vectorEmbeddi…
aayush3011 May 2, 2024
86b36d3
Merge branch 'main' into users/akataria/nonStreamingOrderBy
aayush3011 May 3, 2024
a979c11
Initial changes
aayush3011 May 3, 2024
e2756a5
Initial changes
aayush3011 May 3, 2024
8be2277
Initial changes
aayush3011 May 3, 2024
e491b9d
Resolving comments
aayush3011 May 7, 2024
151bb50
Fixing build issues
aayush3011 May 7, 2024
73afd5b
Merge branch 'Azure:main' into users/akataria/nonStreamingOrderBy
aayush3011 May 8, 2024
148cba5
[Cosmos][VectorSearch] Non Streaming Order By Query (#40085)
aayush3011 May 8, 2024
3b0d751
Merge branch 'Azure:main' into users/akataria/nonStreamingOrderBy
aayush3011 May 9, 2024
87572f7
Merge branch 'feature/vector_search' into users/akataria/nonStreaming…
aayush3011 May 9, 2024
df7e838
[Cosmos][VectorSearch] Non Streaming Order By Query (#40096)
aayush3011 May 9, 2024
179f904
Initial changes
aayush3011 May 9, 2024
5602e33
Merge branch 'users/akataria/nonStreamingOrderBy' of github.com:aayus…
aayush3011 May 9, 2024
36ab9b7
Merge branch 'feature/vector_search' into users/akataria/nonStreaming…
aayush3011 May 9, 2024
70639b5
Initial changes
aayush3011 May 10, 2024
c45c3a5
Fixes
aayush3011 May 10, 2024
6c255ee
Merge branch 'Azure:main' into users/akataria/nonStreamingOrderBy
aayush3011 May 10, 2024
9d427e6
Users/akataria/vectorindexing (#40117)
aayush3011 May 10, 2024
1cadb1b
Merge branch 'feature/VectorSearch' into users/akataria/nonStreamingO…
aayush3011 May 10, 2024
0f1be0c
Users/akataria/non streaming order by (#40118)
aayush3011 May 10, 2024
d4dcad2
Fixing some merge issues
aayush3011 May 10, 2024
cdaa5bc
Fixing some merge issues
aayush3011 May 10, 2024
dfa8b64
Fixing some merge issues
aayush3011 May 10, 2024
7549cbe
Resolving comments
aayush3011 May 14, 2024
c6e2376
Merge branch 'feature/VectorSearch' into users/akataria/nonStreamingO…
aayush3011 May 14, 2024
30d8370
Users/akataria/vectorindexing (#40158)
aayush3011 May 14, 2024
8279358
Users/akataria/non streaming order by (#40159)
aayush3011 May 14, 2024
dd9d13b
Fixing build issue
aayush3011 May 14, 2024
d966af4
Merge branch 'feature/VectorSearch' into users/akataria/nonStreamingO…
aayush3011 May 14, 2024
9eb9208
Fixing build issue
aayush3011 May 14, 2024
1aaecef
Merge branch 'users/akataria/nonStreamingOrderBy' of github.com:aayus…
aayush3011 May 14, 2024
8bf4f8d
Merge branch 'main' into users/akataria/nonStreamingOrderBy
aayush3011 May 16, 2024
943cc4c
Adding tests
aayush3011 May 16, 2024
1db9b31
Adding capability for CI pipeline
aayush3011 May 16, 2024
2770b0f
Updating the PQ logic
aayush3011 May 17, 2024
7002362
Resolving comments, adding new test cases
aayush3011 May 17, 2024
508e94a
Adding argument to run emulator tests
aayush3011 May 17, 2024
822bd67
fixing emulator test pipeline
aayush3011 May 17, 2024
46fe7cb
fixing emulator test pipeline
aayush3011 May 17, 2024
5657b75
Adding logging for variable AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY
aayush3011 May 17, 2024
015a77c
Adding logging for variable AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY
aayush3011 May 17, 2024
f87be45
fixing emulator test pipeline
aayush3011 May 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Resolving comments, adding new test cases
  • Loading branch information
aayush3011 committed May 17, 2024
commit 7002362cedd42ffef2a9a52cdc5bf450421907cf

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ public class NonStreamingOrderByDocumentQueryExecutionContext
private final Collection<ClientSideRequestStatistics> clientSideRequestStatistics;
private Flux<OrderByRowResult<Document>> orderByObservable;

private int maxPageSizePerPartition;

public NonStreamingOrderByDocumentQueryExecutionContext(
DiagnosticsClientContext diagnosticsClientContext,
IDocumentQueryClient client,
Expand Down Expand Up @@ -170,7 +168,7 @@ protected NonStreamingOrderByDocumentProducer createDocumentProducer(
createRetryPolicyFunc,
Document.class,
correlatedActivityId,
maxPageSizePerPartition,
initialPageSize,
continuationToken,
top,
this.getOperationContextTextProvider());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ public Flux<OrderByRowResult<Document>> apply(Flux<DocumentProducer<Document>.Do
// item and then remove the element. If we don't do this, then when adding this element the size of the pq
// will be increased automatically by 50% and then there would be inconsistent results for later pages.
PriorityBlockingQueue<OrderByRowResult<Document>> priorityQueue = new PriorityBlockingQueue<>(initialPageSize + 1, consumeComparer);

return source.flatMap(documentProducerFeedResponse -> {
clientSideRequestStatistics.addAll(
diagnosticsAccessor.getClientSideRequestStatisticsForQueryPipelineAggregations(documentProducerFeedResponse
Expand All @@ -94,19 +93,19 @@ public Flux<OrderByRowResult<Document>> apply(Flux<DocumentProducer<Document>.Do
null);
priorityQueue.add(orderByRowResult);
if (priorityQueue.size() > initialPageSize) {
PriorityBlockingQueue<OrderByRowResult<Document>> resultPriorityQueue = new PriorityBlockingQueue<>(initialPageSize + 1, consumeComparer);
for (int i=0;i<initialPageSize;i++) {
resultPriorityQueue.add(priorityQueue.poll());
PriorityBlockingQueue<OrderByRowResult<Document>> tempPriorityQueue = new PriorityBlockingQueue<>(initialPageSize + 1, consumeComparer);
for (int i = 0; i < initialPageSize; i++) {
tempPriorityQueue.add(priorityQueue.poll());
}
priorityQueue.clear();
priorityQueue.addAll(resultPriorityQueue);
priorityQueue.addAll(tempPriorityQueue);
}
});
tracker.addCharge(documentProducerFeedResponse.pageResult.getRequestCharge());
// Returning an empty Flux since we are only processing and managing state here
return Flux.empty();
}, 1)
.thenMany(Flux.defer(()-> Flux.fromIterable(priorityQueue)));
.thenMany(Flux.defer(() -> Flux.fromIterable(priorityQueue)));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ class QueryPlanRetriever {
ImplementationBridgeHelpers.CosmosQueryRequestOptionsHelper.getCosmosQueryRequestOptionsAccessor();

private static final String TRUE = "True";

// For a limited time, if the query runs against a region or emulator that has not yet been updated with the
// new NonStreamingOrderBy query feature the client might run into some issue of not being able to recognize this,
// and throw a 400 exception. If the environment variable `AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY` is set to
// True to opt out of this new query feature, we will return the OLD query features to operate correctly.
private static final String SUPPORTED_QUERY_FEATURES = QueryFeature.Aggregate.name() + ", " +
QueryFeature.CompositeAggregate.name() + ", " +
QueryFeature.MultipleOrderBy.name() + ", " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -265,17 +265,6 @@ Integer getMaxItemCountForVectorSearch() {
return this.actualRequestOptions.getMaxItemCountForVectorSearch();
}

/**
* Sets the maximum item size to fetch during non-streaming order by queries.
*
* @param maxItemCountForVectorSearch the max number of items for vector search.
* @return the CosmosQueryRequestOptions.
*/
CosmosQueryRequestOptions setMaxItemCountForVectorSearch(Integer maxItemCountForVectorSearch) {
this.actualRequestOptions.setMaxItemCountForVectorSearch(maxItemCountForVectorSearch);
return this;
}

/**
* Gets the request continuation token.
*
Expand Down
6 changes: 5 additions & 1 deletion sdk/cosmos/test-resources.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,11 @@
"maxStalenessPrefix": 100
},
"locations": "[variables('locationsConfiguration')]",
"capabilities": [],
"capabilities": [
{
"name": "EnableNoSQLVectorSearch"
}
],
"ipRules": []
}
}
Expand Down