Completed basic implementation of external storage.

datajoint/base_relation.py

@@ -30,6 +30,7 @@ class BaseRelation(RelationalOperand):
     _context = None
     database = None
     _log_ = None
+    _external_table = None
 
     # -------------- required by RelationalOperand ----------------- #
     @property
@@ -121,6 +122,12 @@ def _log(self):
             self._log_ = Log(self.connection, database=self.database)
         return self._log_
 
+    @property
+    def external_table(self):
+        if self._external_table is None:
+            self._external_table = self.connection.schemas[self.database].external_table
+        return self._external_table
+
     def insert1(self, row, **kwargs):
         """
         Insert one data record or one Mapping (like a dict).
@@ -181,11 +188,11 @@ def safe_query(*args, **kwargs):
             if not ignore_extra_fields:
                 try:
                     raise DataJointError(
-                            "Attribute %s not found.  To ignore extra attributes in insert, set ignore_extra_fields=True." %
-                            next(name for name in rows.heading if name not in heading))
+                        "Attribute %s not found.  To ignore extra attributes in insert, set ignore_extra_fields=True." %
+                        next(name for name in rows.heading if name not in heading))
                 except StopIteration:
                     pass
-            fields=list(name for name in heading if name in rows.heading)
+            fields = list(name for name in heading if name in rows.heading)
             query = 'INSERT{ignore} INTO {table} ({fields}) {select}'.format( 
                ignore=" IGNORE" if ignore_errors or skip_duplicates else "",
                fields='`' + '`,`'.join(fields) + '`',
@@ -216,7 +223,10 @@ def make_placeholder(name, value):
                 """
                 if ignore_extra_fields and name not in heading:
                     return None
-                if heading[name].is_blob:
+                if heading[name].is_external:
+                    value = self.external_table.put(heading[name].type, value)
+                    placeholder = '%s'
+                elif heading[name].is_blob:
                     value = pack(value)
                     placeholder = '%s'
                 elif heading[name].numeric:
@@ -471,8 +481,9 @@ def describe(self):
                             attributes_declared.update(fk_props['referencing_attributes'])
             if do_include:
                 attributes_declared.add(attr.name)
+                name = attr.name.lstrip('_')  # for external
                 definition += '%-20s : %-28s # %s\n' % (
-                    attr.name if attr.default is None else '%s=%s' % (attr.name, attr.default),
+                    name if attr.default is None else '%s=%s' % (name, attr.default),
                     '%s%s' % (attr.type, ' auto_increment' if attr.autoincrement else ''), attr.comment)
         print(definition)
         return definition

datajoint/declare.py

@@ -209,10 +209,10 @@ def compile_attribute(line, in_key, foreign_key_sql):
             raise DataJointError('The external store `{type}` is not configured.'.format(**match))
 
         # append external configuration name to the end of the comment
-        sql = '`_{name}` {hash_type} {default} COMMENT "{comment}:{type}"'.format(
+        sql = '`{name}` {hash_type} {default} COMMENT ":{type}:{comment}"'.format(
             hash_type=HASH_DATA_TYPE, **match)
         foreign_key_sql.append(
-            "FOREIGN KEY (`_{name}`) REFERENCES {{external_table}} (`hash`) "
+            "FOREIGN KEY (`{name}`) REFERENCES {{external_table}} (`hash`) "
             "ON UPDATE RESTRICT ON DELETE RESTRICT".format(**match))
 
     return match['name'], sql, is_external

datajoint/fetch.py

@@ -194,8 +194,12 @@ def __call__(self, *attrs, **kwargs):
             else:
                 ret = list(cur.fetchall())
                 ret = np.array(ret, dtype=heading.as_dtype)
-                for blob_name in heading.blobs:
-                    ret[blob_name] = list(map(unpack_, ret[blob_name]))
+                for name in heading:
+                    if heading[name].is_external:
+                        external_table = self._relation.connection.schemas[heading[name].database].external_table
+                        ret[name] = list(map(external_table.get, ret[name]))
+                    elif heading[name].is_blob:
+                        ret[name] = list(map(unpack_, ret[name]))
 
         else:  # if list of attributes provided
             attributes = [a for a in attrs if a is not PRIMARY_KEY]
@@ -292,13 +296,19 @@ def __call__(self, *attrs, **kwargs):
         ext_behavior = update_dict(self.ext_behavior, kwargs)
         unpack_ = partial(unpack, squeeze=ext_behavior['squeeze'])
 
-        if len(attrs) == 0:  # fetch all attributes
+        if len(attrs) == 0:  # fetch all attributes, return as ordered dict
             cur = self._relation.cursor(as_dict=True)
             ret = cur.fetchone()
             if not ret or cur.fetchone():
                 raise DataJointError('fetch1 should only be used for relations with exactly one tuple')
-            ret = OrderedDict((name, unpack_(ret[name]) if heading[name].is_blob else ret[name])
-                               for name in heading.names)
+
+            def get_external(attr, _hash):
+                external_table = self._relation.connection[attr.database].external_table
+                external_table.get(_hash)
+
+            ret = OrderedDict((name, get_external(heading[name], ret[name])) if heading[name].is_external
+                              else (name, unpack_(ret[name]) if heading[name].is_blob else ret[name])
+                              for name in heading.names)
         else:
             attributes = [a for a in attrs if a is not PRIMARY_KEY]
             result = self._relation.proj(*attributes).fetch(**ext_behavior)

datajoint/heading.py

@@ -8,7 +8,8 @@
 
 default_attribute_properties = dict(    # these default values are set in computed attributes
     name=None, type='expression', in_key=False, nullable=False, default=None, comment='calculated attribute',
-    autoincrement=False, numeric=None, string=None, is_blob=False, is_external=False, sql_expression=None, dtype=object)
+    autoincrement=False, numeric=None, string=None, is_blob=False, is_external=False, sql_expression=None,
+    database=None, dtype=object)
 
 
 class Attribute(namedtuple('_Attribute', default_attribute_properties.keys())):
@@ -87,6 +88,9 @@ def __getitem__(self, name):
         return self.attributes[name]
 
     def __repr__(self):
+        """
+        :return:  heading representation in DataJoint declaration format but without foreign key expansion
+        """
         if self.attributes is None:
             return 'heading not loaded'
         in_key = True
@@ -120,8 +124,7 @@ def as_sql(self):
         """
         represent heading as SQL field list
         """
-        return ','.join('`%s`' % ('_' + name if self.attributes[name].is_external else name)
-                        if self.attributes[name].sql_expression is None
+        return ','.join('`%s`' % name if self.attributes[name].sql_expression is None
                         else '%s as `%s`' % (self.attributes[name].sql_expression, name)
                         for name in self.names)
 
@@ -186,21 +189,20 @@ def init_from_database(self, conn, database, table_name):
         # additional attribute properties
         for attr in attributes:
             # process external attributes
-            is_external = attr['name'].startswith('_')
-            if is_external:
-                comment = attr['comment'].split(':')
-                attr['type'] = comment[-1]
-                attr['comment'] = ':'.join(comment[:-1])
-                attr['name'] = attr['name'][1:]
+            split_comment = attr['comment'].split(':')
+            attr['is_external'] = len(split_comment) >= 3 and split_comment[1].startswith('external')
+            if attr['is_external']:
+                attr['comment'] = ':'.join(split_comment[2:])
+                attr['type'] = split_comment[1]
 
             attr['nullable'] = (attr['nullable'] == 'YES')
             attr['in_key'] = (attr['in_key'] == 'PRI')
             attr['autoincrement'] = bool(re.search(r'auto_increment', attr['Extra'], flags=re.IGNORECASE))
             attr['type'] = re.sub(r'int\(\d+\)', 'int', attr['type'], count=1)   # strip size off integers
             attr['numeric'] = bool(re.match(r'(tiny|small|medium|big)?int|decimal|double|float', attr['type']))
             attr['string'] = bool(re.match(r'(var)?char|enum|date|year|time|timestamp', attr['type']))
-            attr['is_blob'] = is_external or bool(re.match(r'(tiny|medium|long)?blob', attr['type']))
-            attr['is_external'] = is_external
+            attr['is_blob'] = attr['is_external'] or bool(re.match(r'(tiny|medium|long)?blob', attr['type']))
+            attr['database'] = database
 
             if attr['string'] and attr['default'] is not None and attr['default'] not in sql_literals:
                 attr['default'] = '"%s"' % attr['default']

tests/schema_external.py

@@ -58,7 +58,7 @@ class Dimension(dj.Lookup):
 
 
 @schema
-class Image(dj.Manual):
+class Image(dj.Computed):
     definition = """
     # table for storing
     -> Seed

tests/test_external_class.py

@@ -1,13 +1,29 @@
-
-from nose.tools import assert_true, raises, assert_equal, assert_dict_equal, assert_list_equal
+from nose.tools import assert_true, assert_list_equal
 
 from . import schema_external as modu
 
 
+def test_heading():
+    heading = modu.Simple().heading
+    assert_true('item' in heading)
+    assert_true(heading['item'].is_external)
+
+
 def test_insert_and_fetch():
-    assert_list_equal(modu.Simple().heading.externals, ['item'])
+    original_list = [1, 3, 8]
+    modu.Simple().insert1(dict(simple=1, item=original_list))
+    # test fetch
+    q = (modu.Simple() & {'simple': 1}).fetch('item')[0]
+    assert_list_equal(list(q), original_list)
+    # test fetch1
+    q = (modu.Simple() & {'simple': 1}).fetch1('item')
+    assert_list_equal(list(q), original_list)
 
 
 def test_populate():
-    print('here')
-#    modu.Image().populate()
+    img = modu.Image()
+    img.populate()
+    remaining, total = img.progress()
+    assert_true(total > 0 and remaining == 0)
+    for img, dimensions in zip(*(img * modu.Dimension()).fetch('img', 'dimensions')):
+        assert_list_equal(list(img.shape), list(dimensions))