Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
69 commits
Select commit Hold shift + click to select a range
991a274
sync changes and sample for vector search control plane
simorenoh Mar 21, 2024
3a3a652
Update index_management.py
simorenoh Mar 21, 2024
20f533c
Update index_management.py
simorenoh Mar 21, 2024
09f33b7
async and samples
simorenoh Mar 28, 2024
8e527fd
sync and async tests
simorenoh Mar 28, 2024
7c44137
Update CHANGELOG.md
simorenoh Mar 28, 2024
7eb5439
developed typehints
simorenoh Mar 28, 2024
c428476
skip tests
simorenoh Mar 29, 2024
58000fd
create_if_not_exists, README
simorenoh Apr 2, 2024
4c4b1ab
Update README.md
simorenoh Apr 2, 2024
0e6b24f
add provisional, add dimension limit
simorenoh Apr 3, 2024
b42f3cb
Merge branch 'main' into vector-search-query
simorenoh Apr 16, 2024
fef391d
adds sync changes, adds changelog
simorenoh May 3, 2024
8583dbf
async changes
simorenoh May 3, 2024
158f60f
some comments addressed
simorenoh May 3, 2024
c880436
Update CHANGELOG.md
simorenoh May 3, 2024
a414f05
bug fix on ordering
simorenoh May 8, 2024
d217210
ordering bug fix
simorenoh May 8, 2024
8869ea4
fix datetime
simorenoh May 8, 2024
0c6d8eb
samples added
simorenoh May 8, 2024
30b0645
small fixes
simorenoh May 9, 2024
5056d89
fix some additional PQ logic
simorenoh May 9, 2024
358deae
last bit of pq fixes
simorenoh May 9, 2024
617c709
Update non_streaming_order_by_aggregator.py
simorenoh May 9, 2024
73e3709
memory optimization
simorenoh May 10, 2024
6bb8090
Update sdk/cosmos/azure-cosmos/azure/cosmos/_execution_context/aio/do…
simorenoh May 10, 2024
326b155
Merge branch 'main' into vector-search-query
simorenoh May 10, 2024
540a645
addressing comments
simorenoh May 10, 2024
98a4fc9
test name fix, improve readme/ samples
simorenoh May 10, 2024
d487519
add sync tests, improve readme
simorenoh May 10, 2024
abd2bc0
async tests
simorenoh May 10, 2024
a0547b1
pylint
simorenoh May 10, 2024
07acb93
remove print
simorenoh May 10, 2024
7cd5b92
pylint
simorenoh May 10, 2024
5834b29
adds env variable
simorenoh May 10, 2024
f615f3e
adds JS tests
simorenoh May 13, 2024
0081bbe
error logic improvements
simorenoh May 13, 2024
674f483
readme updates
simorenoh May 13, 2024
0e26bf6
more fixes to logic
simorenoh May 13, 2024
a65eb0a
oops
simorenoh May 13, 2024
6563bc3
memory optimization
simorenoh May 13, 2024
9935dc1
Update sdk/cosmos/azure-cosmos/README.md
simorenoh May 13, 2024
ad36a9c
update variable for naming conventions
simorenoh May 13, 2024
86b78b7
remove/ comment out diskANN
simorenoh May 13, 2024
3cff42f
offset + limit fix, tests fixes
simorenoh May 14, 2024
dd187dd
add capabilities env var flag
simorenoh May 14, 2024
d2fbb1b
use feature flag for existing query tests
simorenoh May 14, 2024
fe7742a
disable emulator for query tests
simorenoh May 14, 2024
7cd4d9d
missed some tests
simorenoh May 14, 2024
b3876c6
Update test_aggregate.py
simorenoh May 14, 2024
d8bc50d
Update test-resources.bicep
simorenoh May 15, 2024
1e699e4
forgot tests were being skipped
simorenoh May 15, 2024
e79839b
Update sdk/cosmos/azure-cosmos/test/test_vector_policy.py
Pilchie May 15, 2024
16860dc
Update sdk/cosmos/azure-cosmos/test/test_vector_policy_async.py
Pilchie May 15, 2024
1431e9e
test fixes
simorenoh May 15, 2024
28bef5b
Merge branch 'vector-search-query' of https://github.com/simorenoh/az…
simorenoh May 15, 2024
8701b80
Update README.md
simorenoh May 15, 2024
58af1bb
create separate db for vectors
simorenoh May 15, 2024
9bfdf57
tests
simorenoh May 15, 2024
45e5b6d
tests
simorenoh May 15, 2024
c4a7c60
more tests
simorenoh May 15, 2024
b6dbe45
small bit
simorenoh May 15, 2024
fca1294
final fixes hopefully
simorenoh May 15, 2024
445ba94
raise time limit on test so it doesnt fail
simorenoh May 15, 2024
f64775d
Update test_query_vector_similarity_async.py
simorenoh May 15, 2024
ae9524d
add date for release prep
simorenoh May 15, 2024
e616c4a
Merge branch 'main' into vector-search-query
simorenoh May 15, 2024
8ad2591
Update CHANGELOG.md
simorenoh May 15, 2024
fd10e89
Merge branch 'main' into vector-search-query
simorenoh May 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
some comments addressed
  • Loading branch information
simorenoh committed May 3, 2024
commit 158f60f73edfd6454ea4e0e88b02798d07b16a32
Original file line number Diff line number Diff line change
Expand Up @@ -312,18 +312,7 @@ async def compare(self, doc_producer1, doc_producer2):
negative integer if doc_producers1 < doc_producers2
:rtype: int
"""
order1 = doc_producer1._item_result["orderByItems"][0]
order2 = doc_producer2._item_result["orderByItems"][0]
type1_ord = _OrderByHelper.getTypeOrd(order1)
type2_ord = _OrderByHelper.getTypeOrd(order2)

type_ord_diff = type1_ord - type2_ord

if type_ord_diff:
return type_ord_diff

# the same type,
if type1_ord == 0:
return 0

return _compare_helper(order1['item'], order2['item'])
# TODO: this is not fully safe - doesn't deal with scenario of having orderByItems of [{}]
rank1 = doc_producer1._item_result["orderByItems"][0]
rank2 = doc_producer2._item_result["orderByItems"][0]
return await _OrderByHelper.compare(rank1, rank2)
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ async def _create_pipelined_execution_context(self, query_execution_info):
raise CosmosHttpResponseError(StatusCodes.BAD_REQUEST,
"Cross partition query only supports 'VALUE <AggregateFunc>' for aggregates")
# throw exception here for vector search query without limit filter
if query_execution_info.get_has_non_streaming_order_by():
if query_execution_info.has_non_streaming_order_by():
if query_execution_info.get_top() is None and query_execution_info.get_limit() is None:
# TODO: missing one last if statement here to check for the system variable bypass - need name
raise CosmosHttpResponseError(StatusCodes.BAD_REQUEST,
Expand Down Expand Up @@ -149,7 +149,7 @@ def __init__(self, client, options, execution_context, query_execution_info):
self._endpoint = endpoint_component._QueryExecutionEndpointComponent(execution_context)

order_by = query_execution_info.get_order_by()
if query_execution_info.get_has_non_streaming_order_by():
if query_execution_info.has_non_streaming_order_by():
self._endpoint = endpoint_component._QueryExecutionNonStreamingEndpointComponent(self._endpoint)
elif order_by:
self._endpoint = endpoint_component._QueryExecutionOrderByEndpointComponent(self._endpoint)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,59 +285,12 @@ class _NonStreamingDocumentProducer(object):
to properly sort items as they get inserted.
"""

def __init__(self, item_result, sort_order):
def __init__(self, item_result):
"""
Constructor
"""
self._item_result = item_result
self._doc_producer_comp = _NonStreamingOrderByComparator(sort_order)

def __lt__(self, other):
return self._doc_producer_comp.compare(self, other) < 0


class _NonStreamingOrderByComparator(object):
"""Provide a Comparator for item results which respects orderby sort order.
"""

def __init__(self, sort_order): # pylint: disable=super-init-not-called
"""Instantiates this class

:param list sort_order:
List of sort orders (i.e., Ascending, Descending)

:ivar list sort_order:
List of sort orders (i.e., Ascending, Descending)

"""
self._sort_order = sort_order

def compare(self, doc_producer1, doc_producer2):
"""Compares the given two instances of DocumentProducers.

Based on the orderby query items and whether the sort order is Ascending
or Descending compares the peek result of the two DocumentProducers.

:param _DocumentProducer doc_producer1: first instance to be compared
:param _DocumentProducer doc_producer2: second instance to be compared
:return:
Integer value of compare result.
positive integer if doc_producers1 > doc_producers2
negative integer if doc_producers1 < doc_producers2
:rtype: int
"""
order1 = doc_producer1._item_result["orderByItems"][0]
order2 = doc_producer2._item_result["orderByItems"][0]
type1_ord = _OrderByHelper.getTypeOrd(order1)
type2_ord = _OrderByHelper.getTypeOrd(order2)

type_ord_diff = type1_ord - type2_ord

if type_ord_diff:
return type_ord_diff

# the same type,
if type1_ord == 0:
return 0

return _compare_helper(order1['item'], order2['item'])
return _OrderByHelper.compare(self._item_result["orderByItems"][0],
other._item_result["orderByItems"]) < 0
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ def _create_pipelined_execution_context(self, query_execution_info):
"Cross partition query only supports 'VALUE <AggregateFunc>' for aggregates")

# throw exception here for vector search query without limit filter
if query_execution_info.get_has_non_streaming_order_by():
if query_execution_info.has_non_streaming_order_by():
if query_execution_info.get_top() is None and query_execution_info.get_limit() is None:
# TODO: missing one last if statement here to check for the system variable bypass - need name
raise CosmosHttpResponseError(StatusCodes.BAD_REQUEST,
Expand Down Expand Up @@ -161,7 +161,7 @@ def __init__(self, client, options, execution_context, query_execution_info):
self._endpoint = endpoint_component._QueryExecutionEndpointComponent(execution_context)

order_by = query_execution_info.get_order_by()
if query_execution_info.get_has_non_streaming_order_by():
if query_execution_info.has_non_streaming_order_by():
self._endpoint = endpoint_component._QueryExecutionNonStreamingEndpointComponent(self._endpoint)
elif order_by:
self._endpoint = endpoint_component._QueryExecutionOrderByEndpointComponent(self._endpoint)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def __init__(self, client, resource_link, query, options, partitioned_query_ex_i
# will be a list of (partition_min, partition_max) tuples
targetPartitionRanges = self._get_target_partition_key_range()

self._document_producer_comparator = document_producer._NonStreamingOrderByComparator(self._sort_orders)
self._document_producer_comparator = document_producer._OrderByDocumentProducerComparator(self._sort_orders)

targetPartitionQueryExecutionContextList = []
for partitionTargetRange in targetPartitionRanges:
Expand Down Expand Up @@ -90,7 +90,7 @@ def __init__(self, client, resource_link, query, options, partitioned_query_ex_i
while True:
try:
result = doc_producer.peek()
item_result = document_producer._NonStreamingDocumentProducer(result, self._sort_orders)
item_result = document_producer._NonStreamingDocumentProducer(result)
self._orderByPQ.push(item_result)
next(doc_producer)
except StopIteration:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def get_rewritten_query(self):
rewrittenQuery = rewrittenQuery.replace("{documentdb-formattableorderbyquery-filter}", "true")
return rewrittenQuery

def get_has_non_streaming_order_by(self):
def has_non_streaming_order_by(self):
"""Returns if the query is a non-streaming order by query.
:returns: Query is a non-streaming order by query.
:rtype: bool
Expand Down