From 24c898edcae22b9a6049ad64aa470d63ec525a1d Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Fri, 12 Feb 2016 23:23:36 +1100 Subject: [PATCH 01/10] Ensure index builds in background --- mongorm/connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongorm/connection.py b/mongorm/connection.py index d61757f..5620056 100644 --- a/mongorm/connection.py +++ b/mongorm/connection.py @@ -72,6 +72,9 @@ def ensureIndexes( ): # database[collection].drop_index(key) # droppedIndexes.append(key) + # Build the index in the background + kwargs['background'] = True + database[collection].ensure_index(key_or_list, **kwargs) registeredIndexes += stagedIndexes From c3a4e50ece41eef56aff9dc23f330ffeedfd7a4c Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Fri, 12 Feb 2016 23:24:10 +1100 Subject: [PATCH 02/10] Make ListFields choose the correct index key based on their contents --- mongorm/fields/BaseField.py | 5 ++++- mongorm/fields/ListField.py | 3 +++ mongorm/fields/ReferenceField.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mongorm/fields/BaseField.py b/mongorm/fields/BaseField.py index 24c4c19..60245cb 100644 --- a/mongorm/fields/BaseField.py +++ b/mongorm/fields/BaseField.py @@ -28,4 +28,7 @@ def setOwnerDocument( self, ownerDocument ): self.ownerDocument = ownerDocument def optimalIndex( self ): - return self.dbField \ No newline at end of file + if self.dbField: + return self.dbField + else: + return "" \ No newline at end of file diff --git a/mongorm/fields/ListField.py b/mongorm/fields/ListField.py index 32d3a00..c6abbf2 100644 --- a/mongorm/fields/ListField.py +++ b/mongorm/fields/ListField.py @@ -32,3 +32,6 @@ def toPython( self, bsonValue ): def setOwnerDocument( self, ownerDocument ): super(ListField, self).setOwnerDocument( ownerDocument ) self.itemClass.setOwnerDocument( ownerDocument ) + + def optimalIndex( self ): + return super(ListField, self).optimalIndex() + self.itemClass.optimalIndex() \ No newline at end of file diff --git a/mongorm/fields/ReferenceField.py b/mongorm/fields/ReferenceField.py index f17bc08..78983af 100644 --- a/mongorm/fields/ReferenceField.py +++ b/mongorm/fields/ReferenceField.py @@ -104,4 +104,4 @@ def toPython( self, bsonValue ): return documentClass( )._fromMongo( initialData ) def optimalIndex( self ): - return self.dbField + '._ref' + return super(ReferenceField, self).optimalIndex() + '._ref' From b0b1b23d64d95f2cc1dfec8175ba50b388494a28 Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Fri, 12 Feb 2016 23:38:07 +1100 Subject: [PATCH 03/10] 0.5.8.12 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67ecdc9..f36f518 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='mongorm', - version='0.5.8.11', + version='0.5.8.12', packages=find_packages(), author='Theo Julienne, John Garland', author_email='theo@icy.com.au, john@openlearning.com', From 87a8975f2a8875d7134d1eca97031c3481826266 Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Tue, 16 Feb 2016 15:03:57 +1100 Subject: [PATCH 04/10] Revert "Make ListFields choose the correct index key based on their contents" This reverts commit c3a4e50ece41eef56aff9dc23f330ffeedfd7a4c. --- mongorm/fields/BaseField.py | 5 +---- mongorm/fields/ListField.py | 3 --- mongorm/fields/ReferenceField.py | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/mongorm/fields/BaseField.py b/mongorm/fields/BaseField.py index 60245cb..24c4c19 100644 --- a/mongorm/fields/BaseField.py +++ b/mongorm/fields/BaseField.py @@ -28,7 +28,4 @@ def setOwnerDocument( self, ownerDocument ): self.ownerDocument = ownerDocument def optimalIndex( self ): - if self.dbField: - return self.dbField - else: - return "" \ No newline at end of file + return self.dbField \ No newline at end of file diff --git a/mongorm/fields/ListField.py b/mongorm/fields/ListField.py index c6abbf2..32d3a00 100644 --- a/mongorm/fields/ListField.py +++ b/mongorm/fields/ListField.py @@ -32,6 +32,3 @@ def toPython( self, bsonValue ): def setOwnerDocument( self, ownerDocument ): super(ListField, self).setOwnerDocument( ownerDocument ) self.itemClass.setOwnerDocument( ownerDocument ) - - def optimalIndex( self ): - return super(ListField, self).optimalIndex() + self.itemClass.optimalIndex() \ No newline at end of file diff --git a/mongorm/fields/ReferenceField.py b/mongorm/fields/ReferenceField.py index 78983af..f17bc08 100644 --- a/mongorm/fields/ReferenceField.py +++ b/mongorm/fields/ReferenceField.py @@ -104,4 +104,4 @@ def toPython( self, bsonValue ): return documentClass( )._fromMongo( initialData ) def optimalIndex( self ): - return super(ReferenceField, self).optimalIndex() + '._ref' + return self.dbField + '._ref' From 9507770481276d1cd83f2847e57ada612d4580df Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Wed, 17 Feb 2016 15:13:30 +1100 Subject: [PATCH 05/10] In Queries, do not unpack elements of field values passed in --- mongorm/fields/DictField.py | 5 --- mongorm/fields/EmbeddedDocumentField.py | 4 +- mongorm/fields/ReferenceField.py | 6 +-- mongorm/queryset/Q.py | 49 ++++++------------------- 4 files changed, 13 insertions(+), 51 deletions(-) diff --git a/mongorm/fields/DictField.py b/mongorm/fields/DictField.py index 12f48d6..03715d8 100644 --- a/mongorm/fields/DictField.py +++ b/mongorm/fields/DictField.py @@ -18,8 +18,3 @@ def toPython( self, bsonValue ): if bsonValue is None: bsonValue = {} return bsonValue - - def toQuery( self, pythonValue, dereferences=[] ): - return { - '.'.join( dereferences ): pythonValue - } \ No newline at end of file diff --git a/mongorm/fields/EmbeddedDocumentField.py b/mongorm/fields/EmbeddedDocumentField.py index f6792c8..c2f8d18 100644 --- a/mongorm/fields/EmbeddedDocumentField.py +++ b/mongorm/fields/EmbeddedDocumentField.py @@ -35,6 +35,4 @@ def toPython( self, bsonValue ): def toQuery( self, pythonValue, dereferences=[] ): # FIXME: this doesn't consider converting the value to/from python # should check the types of the fields the whole way down the chain - return { - '.'.join( dereferences ): pythonValue - } \ No newline at end of file + return pythonValue \ No newline at end of file diff --git a/mongorm/fields/ReferenceField.py b/mongorm/fields/ReferenceField.py index f17bc08..393e8dc 100644 --- a/mongorm/fields/ReferenceField.py +++ b/mongorm/fields/ReferenceField.py @@ -61,11 +61,7 @@ def fromPython( self, pythonValue, dereferences=[], modifier=None ): return data def toQuery( self, pythonValue, dereferences=[] ): - if pythonValue is None: - return None - return { - '_ref': self.fromPython( pythonValue )['_ref'] - } + return self.fromPython( pythonValue ) def toPython( self, bsonValue ): self._getClassInfo( ) diff --git a/mongorm/queryset/Q.py b/mongorm/queryset/Q.py index 62cea2e..4863e4d 100644 --- a/mongorm/queryset/Q.py +++ b/mongorm/queryset/Q.py @@ -64,36 +64,20 @@ def toMongo( self, document, forUpdate=False, modifier=None ): field = document._fields[fieldName] if not forUpdate: if comparison in ARRAY_VALUE_COMPARISONS: - searchValues = [field.toQuery( item, dereferences=dereferences ) for item in value] - if searchValues and isinstance(searchValues[0], dict): - searchValue = {} - for dictValue in searchValues: - for key in dictValue: - # using setdefault instead of defaultdict in case python < 2.5 - searchValue.setdefault(key, []).append( dictValue[key] ) - else: - searchValue = searchValues + searchValue = [field.toQuery( item, dereferences=dereferences ) for item in value] else: searchValue = field.toQuery( value, dereferences=dereferences ) - targetSearchKey = field.dbField else: if comparison in ARRAY_VALUE_COMPARISONS: - searchValues = [field.fromPython( item, dereferences=dereferences, modifier=modifier ) for item in value] - if searchValues and isinstance(searchValues[0], dict): - searchValue = {} - for dictValue in searchValues: - for key in dictValue: - # using setdefault instead of defaultdict in case python < 2.5 - searchValue.setdefault(key, []).append( dictValue[key] ) - else: - searchValue = searchValues + searchValue = [field.fromPython( item, dereferences=dereferences, modifier=modifier ) for item in value] else: searchValue = field.fromPython( value, dereferences=dereferences, modifier=modifier ) - targetSearchKey = '.'.join( [field.dbField] + dereferences) - + + targetSearchKey = '.'.join( [field.dbField] + dereferences) + valueMapper = lambda value: value - - + + if comparison is not None: if comparison in REGEX_COMPARISONS: regex,options = REGEX_COMPARISONS[comparison] @@ -113,24 +97,13 @@ def toMongo( self, document, forUpdate=False, modifier=None ): valueMapper = lambda value: { '$'+comparison: value } #newSearch[targetSearchKey] = { '$'+comparison: searchValue } - if isinstance(searchValue, dict): - if not forUpdate: - for name,value in searchValue.iteritems( ): - if name: - key = targetSearchKey + '.' + name - else: - key = targetSearchKey - newSearch[key] = valueMapper(value) - else: - newSearch[targetSearchKey] = valueMapper(searchValue) - else: - newSearch[targetSearchKey] = valueMapper(searchValue) - + newSearch[targetSearchKey] = valueMapper(searchValue) + return newSearch - + def __or__( self, other ): return self.do_merge( other, '$or' ) - + def __and__( self, other ): if len( set( self.query.keys() ).intersection( other.query.keys() ) ) > 0: # if the 2 queries have overlapping keys, we need to use a $and to join them. From e4a67c664703eb102818407cb8bc4648d30b41e1 Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Wed, 17 Feb 2016 15:55:05 +1100 Subject: [PATCH 06/10] bump to 0.5.8.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f36f518..572767e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='mongorm', - version='0.5.8.12', + version='0.5.8.13', packages=find_packages(), author='Theo Julienne, John Garland', author_email='theo@icy.com.au, john@openlearning.com', From 48a8b3c56169b4f1efb80bbfc67f15357a7518ef Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Wed, 17 Feb 2016 17:14:22 +1100 Subject: [PATCH 07/10] Now that referenceFields are queried by their entire values, build indexes on the whole values --- mongorm/fields/ReferenceField.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongorm/fields/ReferenceField.py b/mongorm/fields/ReferenceField.py index 393e8dc..452364d 100644 --- a/mongorm/fields/ReferenceField.py +++ b/mongorm/fields/ReferenceField.py @@ -100,4 +100,4 @@ def toPython( self, bsonValue ): return documentClass( )._fromMongo( initialData ) def optimalIndex( self ): - return self.dbField + '._ref' + return self.dbField From 04e96a4f4fab6ce6a2c3e180222a0e8c90a56f1f Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Wed, 17 Feb 2016 17:14:49 +1100 Subject: [PATCH 08/10] bump 14 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 572767e..355af67 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='mongorm', - version='0.5.8.13', + version='0.5.8.14', packages=find_packages(), author='Theo Julienne, John Garland', author_email='theo@icy.com.au, john@openlearning.com', From 2b926f4557aa858a696453dca35db3d58860abae Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Thu, 18 Feb 2016 18:00:05 +1100 Subject: [PATCH 09/10] Add support for operator --- mongorm/queryset/Q.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongorm/queryset/Q.py b/mongorm/queryset/Q.py index 4863e4d..b9afb9c 100644 --- a/mongorm/queryset/Q.py +++ b/mongorm/queryset/Q.py @@ -24,7 +24,7 @@ def toMongo( self, document, forUpdate=False, modifier=None ): fieldName = name - MONGO_COMPARISONS = ['gt', 'lt', 'lte', 'gte', 'exists', 'ne', 'all', 'in', 'nin', 'elemMatch'] + MONGO_COMPARISONS = ['gt', 'lt', 'lte', 'gte', 'exists', 'eq', 'ne', 'all', 'in', 'nin', 'elemMatch'] REGEX_COMPARISONS = { 'contains': ( '%s', '' ), 'icontains': ( '%s', 'i' ), From 9452c6a56ae5a2da8a7c2729e097efd3c00a9992 Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Thu, 18 Feb 2016 18:00:34 +1100 Subject: [PATCH 10/10] bump 15 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 355af67..3cb2197 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='mongorm', - version='0.5.8.14', + version='0.5.8.15', packages=find_packages(), author='Theo Julienne, John Garland', author_email='theo@icy.com.au, john@openlearning.com',