added external storage tests

datajoint/base_relation.py

@@ -56,11 +56,15 @@ def declare(self):
             sql, uses_external = declare(self.full_table_name, self.definition, self._context)
             if uses_external:
                 # trigger the creation of the external hash lookup for the current schema
-                sql = sql.format(external_table_name=self.connection.schemas[self.database].external_table)
+                external_table = self.connection.schemas[self.database].external_table
+                sql = sql.format(external_table=external_table.full_table_name)
             self.connection.query(sql)
         except pymysql.OperationalError as error:
+            # skip if no create privilege
             if error.args[0] == server_error_codes['command denied']:
                 logger.warning(error.args[1])
+            else:
+                raise
         else:
             self._log('Declared ' + self.full_table_name)
 

datajoint/declare.py

@@ -185,22 +185,31 @@ def compile_attribute(line, in_key, foreign_key_sql):
             match['default'] = 'NOT NULL'
     match['comment'] = match['comment'].replace('"', '\\"')   # escape double quotes in comment
 
-    is_external = match['type'] == 'external'
+    is_external = match['type'].startswith('external')
     if not is_external:
         sql = ('`{name}` {type} {default}' + (' COMMENT "{comment}"' if match['comment'] else '')).format(**match)
     else:
         # process externally stored attribute
         if in_key:
             raise DataJointError('External attributes cannot be primary in:\n%s' % line)
+        store_name = match['type'].split('-')
+        if store_name[0] != 'external':
+            raise DataJointError('External store types must be in format external-<name>')
+        store_name = '-'.join(store_name[1:])
+        if not store_name.isidentifier():
+            raise DataJointError(
+                'The external store name `{type}` is invalid. Make like a python identifier.'.format(**match))
+        if len(store_name)>STORE_NAME_LENGTH:
+            raise DataJointError(
+                'The external store name `{type}` is too long. Must be <={max_len} characters.'.format(
+                    max_len=STORE_NAME_LENGTH, **match))
         if not match['default'] in ('DEFAULT NULL', 'NOT NULL'):
             raise DataJointError('The only acceptable default value for an external field is null in:\n%s' % line)
         if match['type'] not in config:
             raise DataJointError('The external store `{type}` is not configured.'.format(**match))
-        if len(match['type']) > STORE_NAME_LENGTH + STORE_NAME_LENGTH + 1:
-            raise DataJointError(
-                'The external store name `{type}` is too long. Must be <={max_len} characters.'.format(
-                    max_len=STORE_NAME_LENGTH, **match))
-        sql = "`_{name}` {hash_type} {default} COMMENT {comment:type}'".format(
+
+        # append external configuration name to the end of the comment
+        sql = '`_{name}` {hash_type} {default} COMMENT "{comment}:{type}"'.format(
             hash_type=HASH_DATA_TYPE, **match)
         foreign_key_sql.append(
             "FOREIGN KEY (`_{name}`) REFERENCES {{external_table}} (`hash`) "

datajoint/external.py

@@ -30,7 +30,6 @@ def definition(self):
         # external storage tracking 
         hash  : {hash_data_type}  # the hash of stored object + store name
         ---
-        count = 1 :int               # reference count
         size      :bigint unsigned   # size of object in bytes
         timestamp=CURRENT_TIMESTAMP  :timestamp   # automatic timestamp
         """.format(hash_data_type=HASH_DATA_TYPE)
@@ -75,7 +74,7 @@ def put(self, store, obj):
         # insert tracking info
         self.connection.query(
             "INSERT INTO {tab} (hash, size) VALUES ('{hash}', {size}) "
-            "ON DUPLICATE KEY UPDATE count=count+1, timestamp=CURRENT_TIMESTAMP".format(
+            "ON DUPLICATE KEY UPDATE timestamp=CURRENT_TIMESTAMP".format(
                 tab=self.full_table_name,
                 hash=hash,
                 size=len(blob)))
@@ -90,7 +89,7 @@ def get(self, hash):
         try:
             spec = config[store]
         except KeyError:
-            raise DataJointError('storage.%s is not configured' % store)
+            raise DataJointError('Store `%s` is not configured' % store)
 
         try:
             protocol = spec['protocol']
@@ -107,12 +106,4 @@ def get(self, hash):
         else:
             raise DataJointError('Unknown external storage %s' % store)
 
-        return unpack(blob)
-
-    def remove(self, hash):
-        """
-        delete an object from external store
-        """
-        # decrement count 
-        self.connection.query(
-            'UPDATE {tab} SET count=count-1 WHERE hash="{hash}"'.format(tab=self.full_table_name, hash=hash))
+        return unpack(blob)

tests/schema_external.py

@@ -0,0 +1,65 @@
+"""
+a schema for testing external attributes
+"""
+
+import datajoint as dj
+
+from . import PREFIX, CONN_INFO
+import numpy as np
+
+schema = dj.schema(PREFIX + '_extern', locals(), connection=dj.conn(**CONN_INFO))
+
+
+dj.config['external'] = {
+    'protocol': 'file',
+    'location': 'dj-store/external'}
+
+dj.config['external-raw'] = {
+    'protocol': 'file',
+    'location': 'dj-store/raw'}
+
+dj.config['external-compute'] = {
+    'protocol': 's3',
+    'location': '/datajoint-projects/test',
+    'user': 'djtest',
+    'token': '2e05709792545ce'}
+
+dj.config['cache'] = {
+    'protocol': 'file',
+    'location': '/media/dimitri/ExtraDrive1/dj-store/cache'}
+
+
+
+@schema
+class Seed(dj.Lookup):
+    definition = """
+    seed :  int 
+    """
+    contents = zip(range(4))
+
+
+@schema
+class Dimension(dj.Lookup):
+    definition = """
+    dim  : int
+    ---
+    dimensions  : blob  
+    """
+    contents = (
+        [0, [100, 50]],
+        [1, [3, 4, 8, 6]])
+
+
+@schema
+class Image(dj.Manual):
+    definition = """
+    # table for storing
+    -> Seed
+    -> Dimension
+    ----
+    img  : external-raw    #  objects are stored as specified by dj.config['external-raw']
+    """
+
+    def make(self, key):
+        np.random.seed(key['seed'])
+        self.insert1(dict(key, img=np.random.rand(*(Dimension() & key).fetch1('dimensions'))))

tests/test_external.py

@@ -1,36 +1,12 @@
 import numpy as np
 from numpy.testing import assert_array_equal
 from nose.tools import assert_true, assert_equal
-import datajoint as dj
 from datajoint.external import ExternalTable
 
-from . import PREFIX, CONN_INFO
+from . schema_external import schema
 
 
-dj.config['external'] = {
-    'protocol': 'file',
-    'location': 'dj-store/external'}
-
-dj.config['external-raw'] = {
-    'protocol': 'file',
-    'location': 'dj-store/raw'}
-
-dj.config['external-compute'] = {
-    'protocol': 's3',
-    'location': '/datajoint-projects/test',
-    'user': 'dimitri',
-    'token': '2e05709792545ce'
-}
-
-dj.config['cache'] = {
-    'protocol': 'file',
-    'location': '/media/dimitri/ExtraDrive1/dj-store/cache'}
-
-
-schema = dj.schema(PREFIX + '_external_test1', locals(), connection=dj.conn(**CONN_INFO))
-
-
-def test_external_put_get_remove():
+def test_external_put():
     """
     external storage put and get and remove
     """
@@ -43,12 +19,8 @@ def test_external_put_get_remove():
     for i in range(extra):
         hash2 = ext.put('external-raw', np.random.randn(4, 3, 2))
 
-    assert_true(hash1 in ext.fetch('hash'))
-    assert_equal(count, (ext & {'hash': hash1}).fetch1('count'))
+    assert_true(all(hash in ext.fetch('hash') for hash in (hash1, hash2)))
     assert_equal(len(ext), 1 + extra)
 
     output_ = ext.get(hash1)
     assert_array_equal(input_, output_)
-
-    ext.remove(hash1)
-    assert_equal(count-1, (ext & {'hash': hash1}).fetch1('count'))

tests/test_external_class.py

@@ -0,0 +1,8 @@
+
+from nose.tools import assert_true, raises, assert_equal, assert_dict_equal
+
+from . import schema_external as mod
+
+def test_populate():
+    print('here')
+#    mod.Image().populate()

tests/test_fetch.py

@@ -1,6 +1,6 @@
+from nose.tools import assert_true, raises, assert_equal, assert_dict_equal
 from operator import itemgetter
 import itertools
-from nose.tools import assert_true, raises, assert_equal, assert_dict_equal
 import numpy as np
 import decimal
 import warnings