implemented declaration of external fields

datajoint/base_relation.py

@@ -50,11 +50,14 @@ def context(self):
 
     def declare(self):
         """
-        Loads the table heading. If the table is not declared, use self.definition to declare
+        Use self.definition to declare the table in the database
         """
         try:
-            self.connection.query(
-                declare(self.full_table_name, self.definition, self._context))
+            sql, uses_external = declare(self.full_table_name, self.definition, self._context)
+            if uses_external:
+                # trigger the creation of the external hash lookup for the current schema
+                sql = sql.format(external_table_name=self.connection.schemas[self.database].external_table)
+            self.connection.query(sql)
         except pymysql.OperationalError as error:
             if error.args[0] == server_error_codes['command denied']:
                 logger.warning(error.args[1])

datajoint/declare.py

@@ -6,7 +6,10 @@
 import pyparsing as pp
 import logging
 
-from . import DataJointError
+from . import DataJointError, config
+
+STORE_NAME_LENGTH = 8
+HASH_DATA_TYPE = 'varchar(43)'
 
 logger = logging.getLogger(__name__)
 
@@ -114,6 +117,7 @@ def declare(full_table_name, definition, context):
     attribute_sql = []
     foreign_key_sql = []
     index_sql = []
+    uses_external = False
 
     for line in definition:
         if line.startswith('#'):  # additional comments are ignored
@@ -127,7 +131,8 @@ def declare(full_table_name, definition, context):
         elif re.match(r'^(unique\s+)?index[^:]*$', line, re.I):   # index
             index_sql.append(line)  # the SQL syntax is identical to DataJoint's
         else:
-            name, sql = compile_attribute(line, in_key)
+            name, sql, is_external = compile_attribute(line, in_key, foreign_key_sql)
+            uses_external = uses_external or is_external
             if in_key and name not in primary_key:
                 primary_key.append(name)
             if name not in attributes:
@@ -141,16 +146,17 @@ def declare(full_table_name, definition, context):
                        ['PRIMARY KEY (`' + '`,`'.join(primary_key) + '`)'] +
                        foreign_key_sql +
                        index_sql) +
-            '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment)
+            '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment), uses_external
 
 
-def compile_attribute(line, in_key=False):
+def compile_attribute(line, in_key, foreign_key_sql):
     """
     Convert attribute definition from DataJoint format to SQL
 
     :param line: attribution line
     :param in_key: set to True if attribute is in primary key set
-    :returns: (name, sql) -- attribute name and sql code for its declaration
+    :param foreign_key_sql:
+    :returns: (name, sql, is_external) -- attribute name and sql code for its declaration
     """
 
     try:
@@ -177,5 +183,33 @@ def compile_attribute(line, in_key=False):
         else:
             match['default'] = 'NOT NULL'
     match['comment'] = match['comment'].replace('"', '\\"')   # escape double quotes in comment
-    sql = ('`{name}` {type} {default}' + (' COMMENT "{comment}"' if match['comment'] else '')).format(**match)
-    return match['name'], sql
+
+    is_external = match['type'] == 'external' or match['type'].startswith('external_')
+    if not is_external:
+        sql = ('`{name}` {type} {default}' + (' COMMENT "{comment}"'
+                                              if match['comment'] else '')).format(**match)
+    else:
+        # process externally stored attribute
+        if in_key:
+            raise DataJointError('External attributes cannot be primary.')
+        if match['type'] not in config:
+            raise DataJointError('The external store `{type}` is not configured.'.format(**match))
+        if len(match['type']) > STORE_NAME_LENGTH + STORE_NAME_LENGTH + 1:
+            raise DataJointError(
+                'The external store name `{type}` is too long. Must be <={max_len} characters.'.format(
+                    max_len=STORE_NAME_LENGTH, **match))
+        sql = """
+        `_{name}` char({store_name_len}) COMMENT "{type}",
+        `{name}` {hash_type} {default}{comment}'
+        """.format(
+            comment=' COMMENT "{comment}"' if match['comment'] else '',
+            hash_type=HASH_DATA_TYPE,
+            store_name_len=STORE_NAME_LENGTH,
+            **match)
+        foreign_key_sql.append(
+            """
+            FOREIGN KEY (`_{name}`,`name`) REFERENCES {{external_table}} (`store`, `hash`) 
+            ON UPDATE RESTRICT ON DELETE RESTRICT
+            """.format(**match))
+
+    return match['name'], sql, is_external

datajoint/external.py

@@ -1,14 +1,15 @@
 import os
-import pymysql
 from . import config, DataJointError
 from .hash import long_hash
 from .blob import pack, unpack
 from .base_relation import BaseRelation
+from .declare import HASH_DATA_TYPE, STORE_NAME_LENGTH
 
 
 class ExternalTable(BaseRelation):
     """
-    The table tracking externally stored objects
+    The table tracking externally stored objects.
+    Declare as ExternalTable(connection, database)
     """
     def __init__(self, arg, database=None):
         if isinstance(arg, ExternalTable):
@@ -29,13 +30,13 @@ def __init__(self, arg, database=None):
     def definition(self):
         return """
         # external storage tracking 
-        store :char(8)  # the name of external store
-        hash  :char(43) # the hash of stored object
+        store :char({store_name_length})  # the name of external store
+        hash  :{hash_data_type} # the hash of stored object
         ---
         count = 1 :int               # reference count
         size      :bigint unsigned   # size of object in bytes
         timestamp=CURRENT_TIMESTAMP  :timestamp   # automatic timestamp
-        """
+        """.format(store_name_length=STORE_NAME_LENGTH, hash_data_type=HASH_DATA_TYPE)
 
     @property
     def table_name(self):
@@ -45,43 +46,42 @@ def put(self, store, obj):
         """
         put an object in external store
         """
-	# serialize object 
+        # serialize object
         blob = pack(obj)
         hash = long_hash(blob)
 
-	# write object
+        # write object
         try:
             spec = config['external.%s' % store]
         except KeyError:
             raise DataJointError('storage.%s is not configured' % store)
 
-	if spec['protocol'] == 'file':
+        if spec['protocol'] == 'file':
             folder = os.path.join(spec['location'], self.database)
             full_path = os.path.join(folder, hash)
             if not os.path.isfile(full_path):
                 try:
-		    with open(full_path, 'wb') as f:
+                    with open(full_path, 'wb') as f:
                         f.write(blob)
                 except FileNotFoundError:
                     os.makedirs(folder)
-		    with open(full_path, 'wb') as f:
-                        f.write(blob)
+                    with open(full_path, 'wb') as f:
+                                f.write(blob)
         else:
             raise DataJointError('Unknown external storage %s' % store)
 
-	# insert tracking info
-        query = """INSERT INTO `{db}`.`{table}` (store, hash, size) VALUES ({store}, {hash}, {size}) 
-                   ON DUPLICATE KEY count=count+1, timestamp=CURRENT_TIMESTAMP""".format(
-                          db=self.database,
-                          table=self.table_name,
-                          store=store,
-                          hash=hash,
-                          size=len(blob))
-	self.connection.
-
+        # insert tracking info
+        self.connection.query(
+            """
+            INSERT INTO `{db}`.`{table}` (store, hash, size) VALUES ({store}, {hash}, {size}) 
+            ON DUPLICATE KEY count=count+1, timestamp=CURRENT_TIMESTAMP""".format(
+                db=self.database,
+                table=self.table_name,
+                store=store,
+                hash=hash,
+                size=len(blob)))
         return hash
 
-
     def get(self, store, hash):
         """
         get an object from external store
@@ -91,26 +91,28 @@ def get(self, store, hash):
         except KeyError:
             raise DataJointError('storage.%s is not configured' % store)
 
-	if spec['protocol'] == 'file': 
+        if spec['protocol'] == 'file':
             full_path = os.path.join(spec['location'], self.database, hash)
-	    try:
-	        with open(full_path, 'rb') as f:
-	            blob = f.read()
-	    except FileNotFoundError:
-                raise DataJointError('Lost external blob')
+            try:
+                with open(full_path, 'rb') as f:
+                    blob = f.read()
+            except FileNotFoundError:
+                    raise DataJointError('Lost external blob')
         else:
             raise DataJointError('Unknown external storage %s' % store)
 
-	return unpack(blob)        
-
+        return unpack(blob)
 
-    def remove(self, store, hash)
+    def remove(self, store, hash):
         """
         delete an object from external store
         """
         # decrement count 
-        query = "UPDATE `{db}`.`{table}` count=count-1 WHERE store={store} and hash={hash}".format(
-                     db=self.database,
-                     table=self.table_name,
-                     store=store,
-                     hash=hash)
+        self.connection.query(
+            """
+            UPDATE `{db}`.`{table}` count=count-1 WHERE store={store} and hash={hash}
+            """.format(
+                db=self.database,
+                table=self.table_name,
+                store=store,
+                hash=hash))

datajoint/schema.py

@@ -6,6 +6,7 @@
 from . import conn, DataJointError, config
 from .erd import ERD
 from .jobs import JobTable
+from .external import ExternalTable
 from .heading import Heading
 from .utils import user_choice, to_camel_case
 from .user_relations import Part, Computed, Imported, Manual, Lookup
@@ -51,7 +52,8 @@ def __init__(self, database, context, connection=None, create_tables=True):
         self.connection = connection
         self.context = context
         self.create_tables = create_tables
-        self._jobs=None
+        self._jobs = None
+        self._external = None
         if not self.exists:
             if not self.create_tables:
                 raise DataJointError("Database named `{database}` was not defined. "
@@ -212,16 +214,25 @@ def __call__(self, cls):
     def jobs(self):
         """
         schema.jobs provides a view of the job reservation table for the schema
-        :return: jobs relation
+        :return: jobs table
         """
         if self._jobs is None:
             self._jobs = JobTable(self.connection, self.database)
         return self._jobs
 
+    @property
+    def external_table(self):
+        """
+        schema.external provides a view of the external hash table for the schema
+        :return: external table
+        """
+        if self._external is None:
+            self._external = ExternalTable(self.connection, self.database)
+        return self._external
+
     def erd(self):
         # get the caller's locals()
         import inspect
         frame = inspect.currentframe()
         context = frame.f_back.f_locals
         return ERD(self, context=context)
-