Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 25 additions & 20 deletions mongorm/BaseDocument.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,41 @@
from builtins import object
from mongorm import connection
from mongorm.DocumentMetaclass import DocumentMetaclass

from mongorm.blackMagic import serialiseTypesForDocumentType
from future.utils import with_metaclass

class BaseDocument(object):
__metaclass__ = DocumentMetaclass
class BaseDocument(with_metaclass(DocumentMetaclass, object)):
__internal__ = True

class DoesNotExist(Exception):
pass

def __init__( self, **kwargs ):
self._is_lazy = False
self._data = {}
self._values = {}

self._data['_types'] = serialiseTypesForDocumentType( self.__class__ )
for name,value in kwargs.iteritems( ):

for name,value in kwargs.items( ):
setattr(self, name, value)

def _fromMongo( self, data, overwrite=True ):
self._is_lazy = True
for (name,field) in self._fields.iteritems( ):

for (name,field) in self._fields.items( ):
dbField = field.dbField
if dbField in data and ( overwrite or not name in self._values ):
pythonValue = field.toPython( data[dbField] )
setattr(self, name, pythonValue)

return self

# For python 2/3 compatibility
def __bool__( self ):
return True

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

objects are not truthy by default?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image

Neither in python 2 or 3 are empty objects considered truthy. But because the file overwrites __getattr__ (so that it returns None instead of an AttributeError), the way the future object class checks truthiness makes it error because it believes the attribute (the __bool__ function) exists on itself, but when it calls it, it dies. But the old behaviour of the library is that an empty Document is truthy, so this just keeps that compatibility

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

damn

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image

Neither in python 2 or 3 are empty objects considered truthy. But because the file overwrites __getattr__ (so that it returns None instead of an AttributeError), the way the future object class checks truthiness makes it error because it believes the attribute (the __bool__ function) exists on itself, but when it calls it, it dies. But the old behaviour of the library is that an empty Document is truthy, so this just keeps that compatibility

That sounds like a bug; __getattr__ should raise an AttributeError for things that aren't fields. Actually all objects in Python are considered True unless they specify a __bool__ method which returns False or __len__ method which returns 0. Also if you decide to go ahead with this workaround you should probably also specify a __nonzero__ method for Python 2.x (__bool__ was added in Python 3.x)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I don't like that __getattr__ doesn't raise an AttributeError either ahaha. But it's been that way in the library since it's inception it seems, so I don't want to break functionality as core as that in OpenLearning which expects it. After a better inspection across the whole OpenLearning codebase to check it's uses, I'd be more comfortable to recommend that we make that change!

Actually all objects in Python are considered True

Yeap, whoops. Working on JS my brain confused object and dict. Yeap, it's not that object is falsy, it's that the future version of object that we're importing (aka from builtins import object) first checks if the __bool__ method is available and uses that to determine it's truthiness (because it defines the __nonzero__ method) that python 2 uses. But because of the above AttributeError not being raised, it tries to call that method, and dies because it doesn't actually exist to be called. So this was just there to short circuit it into an acceptable response that we want!


# The following three methods are used for pickling/unpickling.
# If it weren't for the fact that __getattr__ returns None
# for non-existing attributes (rather than raising an AttributeError),
Expand All @@ -49,7 +54,7 @@ def __setattr__( self, name, value ):
assert name[0] == '_' or (name in self._fields), \
"Field '%s' does not exist in document '%s'" \
% (name, self.__class__.__name__)

if name in self._fields:
field = self._fields[name]
mongoValue = field.fromPython( value )
Expand All @@ -61,7 +66,7 @@ def __setattr__( self, name, value ):
else:
assert name.startswith( '_' ), 'Only internal variables should ever be set as an attribute'
super(BaseDocument, self).__setattr__( name, value )

def __getattr__( self, name ):
if name not in self._values and self._is_lazy and \
'_id' in self._data and self._data['_id'] is not None:
Expand All @@ -78,9 +83,9 @@ def __getattr__( self, name ):
if result is None:
raise self.DoesNotExist
self._fromMongo( result, overwrite=False )

self._is_lazy = False

field = self._fields.get( name, None )

if not name in self._values:
Expand All @@ -89,18 +94,18 @@ def __getattr__( self, name ):
default = field.getDefault( )

self._values[name] = default

value = self._values.get( name )

return value

def _resyncFromPython( self ):
# before we go any further, re-sync from python values where needed
for (name,field) in self._fields.iteritems( ):
for (name,field) in self._fields.items( ):
requiresDefaultCall = (name not in self._values and callable(field.default))
if field._resyncAtSave or requiresDefaultCall:
dbField = field.dbField
pythonValue = getattr(self, name)
self._data[dbField] = field.fromPython( pythonValue )
#print 'resyncing', dbField, 'to', self._data[dbField]

7 changes: 4 additions & 3 deletions mongorm/Document.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from builtins import str
import pymongo
import warnings

Expand Down Expand Up @@ -43,11 +44,11 @@ def save( self, forceInsert=False, **kwargs ):
newId = collection.insert( self._data, **kwargs )
else:
newId = collection.save( self._data, **kwargs )
except pymongo.errors.OperationFailure, err:
except pymongo.errors.OperationFailure as err:
message = 'Could not save document (%s)'
if u'duplicate key' in unicode(err):
if u'duplicate key' in str(err):
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Of all the things Python 2.x. did horribly, strings are definitely at the top of the list! :P I would highly suggest including the six library and using its six.text_type here to save yourself further headaches.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I ended up using future over six for this migration (just because it seems to better transform the code into something that's py3 compatible, and doesn't require you to do almost two rewrites (one that's six compatible, and one that's solely py3 compatible in the future)). And I used the future library with the from builtins import str at the top :D

message = u'Tried to save duplicate unique keys (%s)'
raise OperationError( message % unicode(err) )
raise OperationError( message % str(err) )
if newId is not None:
setattr(self, self._primaryKeyField, newId)

Expand Down
6 changes: 3 additions & 3 deletions mongorm/DocumentMetaclass.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def __new__( cls, name, bases, attrs ):
attrs['_collection'] = collection

# find all fields and add them to our field list
for attrName, attrValue in attrs.items( ):
for attrName, attrValue in list(attrs.items( )):
if hasattr(attrValue, '__class__') and \
issubclass(attrValue.__class__, BaseField):
field = attrValue
Expand All @@ -65,7 +65,7 @@ def indexConverter( fieldName ):
return fields[fieldName].optimalIndex( )
return fieldName

for field,value in fields.iteritems( ):
for field,value in fields.items( ):
if value.primaryKey:
assert primaryKey is None, "Can only have one primary key per document"
primaryKey = field
Expand Down Expand Up @@ -109,7 +109,7 @@ def indexConverter( fieldName ):
newClass = superNew( cls, name, bases, attrs )

# record the document in the fields
for field in newClass._fields.values( ):
for field in list(newClass._fields.values( )):
#field.ownerDocument = newClass
field.setOwnerDocument( newClass )

Expand Down
1 change: 1 addition & 0 deletions mongorm/DocumentRegistry.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from builtins import object
class DocumentRegistry(object):
documentTypes = {}

Expand Down
3 changes: 2 additions & 1 deletion mongorm/blackMagic.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from future.types.newobject import newobject

def serialiseTypesForDocumentType( documentType ):
return [ cls.__name__ for cls in documentType.mro() if cls != object \
return [ cls.__name__ for cls in documentType.mro() if cls not in [object, newobject] \
and cls.__name__ not in ['Document', 'BaseDocument', 'EmbeddedDocument'] ]
1 change: 1 addition & 0 deletions mongorm/connection.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from past.builtins import basestring
from pymongo import MongoClient, MongoReplicaSetClient
from pymongo.collection import Collection
connection = None
Expand Down
1 change: 1 addition & 0 deletions mongorm/fields/BaseField.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from builtins import object
class BaseField(object):
_resyncAtSave = False

Expand Down
5 changes: 3 additions & 2 deletions mongorm/fields/DateField.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from past.builtins import basestring
from mongorm.fields.BaseField import BaseField

import time
Expand All @@ -14,11 +15,11 @@ def fromPython( self, pythonValue, dereferences=[], modifier=None ):
try:
pythonValue = date.fromtimestamp( time.mktime( time.strptime( pythonValue, '%Y-%m-%d' ) ) )
except:
raise ValueError, "String format of date must be YYYY-MM-DD"
raise ValueError("String format of date must be YYYY-MM-DD")

# make sure we ended up with a date() object
if not isinstance(pythonValue, date):
raise ValueError, "Value must be a date object"
raise ValueError("Value must be a date object")

# convert it to a string since mongo doesn't have a date-only type and datetime
# searches would be wrong. this format should still allow sorting, etc.
Expand Down
1 change: 1 addition & 0 deletions mongorm/fields/DateTimeField.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from past.builtins import basestring
from mongorm.fields.BaseField import BaseField

from datetime import datetime
Expand Down
6 changes: 4 additions & 2 deletions mongorm/fields/DecimalField.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from builtins import str
from past.builtins import basestring
from mongorm.fields.BaseField import BaseField

from decimal import Decimal
Expand All @@ -7,8 +9,8 @@ def fromPython( self, pythonValue, dereferences=[], modifier=None ):
if isinstance(pythonValue, (basestring, int, float)):
pythonValue = Decimal(pythonValue)
if not isinstance(pythonValue, Decimal):
raise ValueError, "Value (%s: %s) must be a Decimal object, or must be able to be passed to the Decimal constructor" % (type(pythonValue), pythonValue,)
raise ValueError("Value (%s: %s) must be a Decimal object, or must be able to be passed to the Decimal constructor" % (type(pythonValue), pythonValue))
return str(pythonValue)

def toPython( self, bsonValue ):
return Decimal(bsonValue)
4 changes: 2 additions & 2 deletions mongorm/fields/ObjectIdField.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
class ObjectIdField(BaseField):
def fromPython( self, pythonValue, dereferences=[], modifier=None ):
if pythonValue is not None:
return objectid.ObjectId( unicode(pythonValue) )
return objectid.ObjectId( str(pythonValue) )
else:
return None

def toPython( self, bsonValue ):
return bsonValue
35 changes: 18 additions & 17 deletions mongorm/fields/ReferenceField.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from past.builtins import basestring
from bson import objectid, dbref
import bson.errors

Expand All @@ -11,12 +12,12 @@ def __init__( self, documentClass, *args, **kwargs ):
super(ReferenceField, self).__init__( *args, **kwargs )
self._use_ref_id = kwargs.get('use_ref_id', False)
self.inputDocumentClass = documentClass

def _getClassInfo( self ):
if hasattr(self, 'documentName'): return

documentClass = self.inputDocumentClass

if isinstance(documentClass, basestring):
if documentClass == 'self':
self.documentName = self.ownerDocument.__name__
Expand All @@ -27,13 +28,13 @@ def _getClassInfo( self ):
else:
self.documentClass = documentClass
self.documentName = documentClass.__name__

def fromPython( self, pythonValue, dereferences=[], modifier=None ):
self._getClassInfo( )

if pythonValue is None:
return None

if isinstance(pythonValue, dbref.DBRef):
return {
'_ref': pythonValue
Expand All @@ -48,18 +49,18 @@ def fromPython( self, pythonValue, dereferences=[], modifier=None ):
return {
'_ref': dbref.DBRef( self.documentClass._collection, objectId ),
}

assert isinstance(pythonValue, self.documentClass), \
"Referenced value must be a document of type %s" % (self.documentName,)
assert pythonValue.id is not None, "Referenced Document must be saved before being assigned"

data = {
'_types': serialiseTypesForDocumentType(pythonValue.__class__),
'_ref': dbref.DBRef( pythonValue.__class__._collection, pythonValue.id ),
}

return data

def toQuery( self, pythonValue, dereferences=[] ):
if pythonValue is None:
return None
Expand All @@ -72,13 +73,13 @@ def toQuery( self, pythonValue, dereferences=[] ):
return {
'_ref': self.fromPython( pythonValue )['_ref']
}

def toPython( self, bsonValue ):
self._getClassInfo( )

if bsonValue is None:
return None

documentClass = None

if isinstance(bsonValue, dbref.DBRef):
Expand All @@ -95,19 +96,19 @@ def toPython( self, bsonValue ):
if '_cls' in bsonValue:
# mongoengine GenericReferenceField compatibility
documentName = bsonValue['_cls']
elif '_types' in bsonValue:
elif '_types' in bsonValue:
documentName = bsonValue['_types'][0]
else:
return dbRef

documentClass = DocumentRegistry.getDocument( documentName )

initialData = {
'_id': dbRef.id,
}
initialData.update( bsonValue.get( '_cache', {} ) )

return documentClass( )._fromMongo( initialData )

def optimalIndex( self ):
return self.dbField + '._ref'
12 changes: 8 additions & 4 deletions mongorm/fields/SafeDictField.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import binascii

from builtins import str, bytes
from past.builtins import basestring
from mongorm.fields.DictField import DictField

from collections import deque
Expand All @@ -9,7 +13,7 @@ def deepCoded( dictionary, coder ):
toCode = deque( [dictionary] )
while toCode:
nextDictionary = toCode.popleft( )
for key, value in nextDictionary.items( ): # can't be iteritems as we're changing the dict
for key, value in list(nextDictionary.items( )): # can't be iteritems as we're changing the dict
if isinstance(key, basestring):
# Keys have to be strings in mongo so this should always occur
del nextDictionary[key]
Expand All @@ -19,12 +23,12 @@ def deepCoded( dictionary, coder ):
return dictionary

def encode( string ):
if isinstance(string, unicode):
if isinstance(string, str):
string = string.encode( 'utf-8' )
return string.encode( 'hex' )
return bytes(binascii.hexlify(string)).decode('utf-8')

def decode( string ):
return string.decode( 'hex' ).decode( 'utf-8' )
return bytes(binascii.unhexlify(string)).decode('utf-8')

class SafeDictField(DictField):
def fromPython( self, *args, **kwargs ):
Expand Down
9 changes: 6 additions & 3 deletions mongorm/fields/StringField.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
from builtins import str

from mongorm.fields.BaseField import BaseField


class StringField(BaseField):
def fromPython( self, pythonValue, dereferences=[], modifier=None ):
if pythonValue is not None:
pythonValue = unicode(pythonValue)
pythonValue = str(pythonValue)
return pythonValue

def toPython( self, bsonValue ):
if bsonValue is not None:
bsonValue = unicode(bsonValue)
bsonValue = str(bsonValue)
return bsonValue
Loading