added tests for external storage

datajoint/autopopulate.py

@@ -68,7 +68,7 @@ def _job_key(self, key):
         return key
 
     def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False,
-		 order="original", limit=None, max_calls=None, display_progress=False):
+                 order="original", limit=None, max_calls=None, display_progress=False):
         """
         rel.populate() calls rel.make(key) for every primary key in self.key_source
         for which there is not already a tuple in rel.

datajoint/declare.py

@@ -9,7 +9,8 @@
 from . import DataJointError, config
 
 STORE_NAME_LENGTH = 8
-HASH_DATA_TYPE = 'varchar(43)'
+STORE_HASH_LENGTH = 43
+HASH_DATA_TYPE = 'char(51)'
 
 logger = logging.getLogger(__name__)
 
@@ -191,25 +192,19 @@ def compile_attribute(line, in_key, foreign_key_sql):
     else:
         # process externally stored attribute
         if in_key:
-            raise DataJointError('External attributes cannot be primary.')
+            raise DataJointError('External attributes cannot be primary in:\n%s' % line)
+        if not match['default'] in ('DEFAULT NULL', 'NOT NULL'):
+            raise DataJointError('The only acceptable default value for an external field is null in:\n%s' % line)
         if match['type'] not in config:
             raise DataJointError('The external store `{type}` is not configured.'.format(**match))
         if len(match['type']) > STORE_NAME_LENGTH + STORE_NAME_LENGTH + 1:
             raise DataJointError(
                 'The external store name `{type}` is too long. Must be <={max_len} characters.'.format(
                     max_len=STORE_NAME_LENGTH, **match))
-        sql = """
-        `_{name}` char({store_name_len}) COMMENT "{type}",
-        `{name}` {hash_type} {default}{comment}'
-        """.format(
-            comment=' COMMENT "{comment}"' if match['comment'] else '',
-            hash_type=HASH_DATA_TYPE,
-            store_name_len=STORE_NAME_LENGTH,
-            **match)
+        sql = "`_{name}` {hash_type} {default} COMMENT {comment:type}'".format(
+            hash_type=HASH_DATA_TYPE, **match)
         foreign_key_sql.append(
-            """
-            FOREIGN KEY (`_{name}`,`name`) REFERENCES {{external_table}} (`store`, `hash`) 
-            ON UPDATE RESTRICT ON DELETE RESTRICT
-            """.format(**match))
+            "FOREIGN KEY (`_{name}`) REFERENCES {{external_table}} (`hash`) "
+            "ON UPDATE RESTRICT ON DELETE RESTRICT".format(**match))
 
     return match['name'], sql, is_external

datajoint/external.py

@@ -3,7 +3,7 @@
 from .hash import long_hash
 from .blob import pack, unpack
 from .base_relation import BaseRelation
-from .declare import HASH_DATA_TYPE, STORE_NAME_LENGTH
+from .declare import STORE_HASH_LENGTH, STORE_NAME_LENGTH
 
 
 class ExternalTable(BaseRelation):
@@ -17,8 +17,6 @@ def __init__(self, arg, database=None):
             # copy constructor
             self.database = arg.database
             self._connection = arg._connection
-            self._definition = arg._definition
-            self._user = arg._user
             return
         super().__init__()
         self.database = database
@@ -30,13 +28,12 @@ def __init__(self, arg, database=None):
     def definition(self):
         return """
         # external storage tracking 
-        store :char({store_name_length})  # the name of external store
-        hash  :{hash_data_type} # the hash of stored object
+        hash  :char({hash_len})  # the hash of stored object + store name
         ---
         count = 1 :int               # reference count
         size      :bigint unsigned   # size of object in bytes
         timestamp=CURRENT_TIMESTAMP  :timestamp   # automatic timestamp
-        """.format(store_name_length=STORE_NAME_LENGTH, hash_data_type=HASH_DATA_TYPE)
+        """.format(hash_len=STORE_HASH_LENGTH + STORE_NAME_LENGTH)
 
     @property
     def table_name(self):
@@ -46,17 +43,21 @@ def put(self, store, obj):
         """
         put an object in external store
         """
+        try:
+            spec = config[store]
+        except KeyError:
+            raise DataJointError('Storage {store} is not configured'.format(store=store))
+
         # serialize object
         blob = pack(obj)
-        hash = long_hash(blob)
+        hash = long_hash(blob) + store[len('external-'):]
 
-        # write object
         try:
-            spec = config['external.%s' % store]
+            protocol = spec['protocol']
         except KeyError:
-            raise DataJointError('storage.%s is not configured' % store)
+            raise DataJointError('Storage {store} config is missing the protocol field'.format(store=store))
 
-        if spec['protocol'] == 'file':
+        if protocol == 'file':
             folder = os.path.join(spec['location'], self.database)
             full_path = os.path.join(folder, hash)
             if not os.path.isfile(full_path):
@@ -66,32 +67,38 @@ def put(self, store, obj):
                 except FileNotFoundError:
                     os.makedirs(folder)
                     with open(full_path, 'wb') as f:
-                                f.write(blob)
+                        f.write(blob)
         else:
-            raise DataJointError('Unknown external storage %s' % store)
+            raise DataJointError('Unknown external storage protocol {protocol} for {store}'.format(
+                store=store, protocol=protocol))
 
         # insert tracking info
         self.connection.query(
-            """
-            INSERT INTO `{db}`.`{table}` (store, hash, size) VALUES ({store}, {hash}, {size}) 
-            ON DUPLICATE KEY count=count+1, timestamp=CURRENT_TIMESTAMP""".format(
+            "INSERT INTO `{db}`.`{table}` (hash, size) VALUES ('{hash}', {size}) "
+            "ON DUPLICATE KEY UPDATE count=count+1, timestamp=CURRENT_TIMESTAMP".format(
                 db=self.database,
                 table=self.table_name,
-                store=store,
-                hash=hash,
+                hash=hash,  # append the name of the store to the hash
                 size=len(blob)))
         return hash
 
-    def get(self, store, hash):
+    def get(self, hash):
         """
         get an object from external store
         """
+        store = hash[STORE_HASH_LENGTH:]
+        store = 'external' + ('-' if store else '') + store
         try:
-            spec = config['external.%s' % store]
+            spec = config[store]
         except KeyError:
             raise DataJointError('storage.%s is not configured' % store)
 
-        if spec['protocol'] == 'file':
+        try:
+            protocol = spec['protocol']
+        except KeyError:
+            raise DataJointError('Storage {store} config is missing the protocol field'.format(store=store))
+
+        if protocol == 'file':
             full_path = os.path.join(spec['location'], self.database, hash)
             try:
                 with open(full_path, 'rb') as f:
@@ -103,16 +110,13 @@ def get(self, store, hash):
 
         return unpack(blob)
 
-    def remove(self, store, hash):
+    def remove(self, hash):
         """
         delete an object from external store
         """
         # decrement count 
         self.connection.query(
-            """
-            UPDATE `{db}`.`{table}` count=count-1 WHERE store={store} and hash={hash}
-            """.format(
+            'UPDATE `{db}`.`{table}` SET count=count-1 WHERE hash="{hash}"'.format(
                 db=self.database,
                 table=self.table_name,
-                store=store,
-                hash=hash))
+                hash=hash))

datajoint/settings.py

@@ -176,8 +176,6 @@ def __getitem__(self, key):
 
         def __setitem__(self, key, value):
             logger.log(logging.INFO, u"Setting {0:s} to {1:s}".format(str(key), str(value)))
-            if isinstance(value, collections.Mapping):
-                raise ValueError("Nested settings are not supported!")
             if validators[key](value):
                 self._conf[key] = value
             else:

tests/test_blob.py

@@ -1,6 +1,6 @@
 import numpy as np
 from datajoint.blob import pack, unpack
-from numpy.testing import assert_array_equal, raises
+from numpy.testing import assert_array_equal
 from nose.tools import assert_equal, assert_true
 
 

tests/test_external.py

@@ -0,0 +1,54 @@
+import numpy as np
+from numpy.testing import assert_array_equal
+from nose.tools import assert_true, assert_equal
+import datajoint as dj
+from datajoint.external import ExternalTable
+
+from . import PREFIX, CONN_INFO
+
+
+dj.config['external'] = {
+    'protocol': 'file',
+    'location': 'dj-store/external'}
+
+dj.config['external-raw'] = {
+    'protocol': 'file',
+    'location': 'dj-store/raw'}
+
+dj.config['external-compute'] = {
+    'protocol': 's3',
+    'location': '/datajoint-projects/test',
+    'user': 'dimitri',
+    'token': '2e05709792545ce'
+}
+
+dj.config['cache'] = {
+    'protocol': 'file',
+    'location': '/media/dimitri/ExtraDrive1/dj-store/cache'}
+
+
+schema = dj.schema(PREFIX + '_external_test1', locals(), connection=dj.conn(**CONN_INFO))
+
+
+def test_external_put_get_remove():
+    """
+    external storage put and get and remove
+    """
+    ext = ExternalTable(schema.connection, schema.database)
+    input_ = np.random.randn(3, 7, 8)
+    count = 7
+    extra = 3
+    for i in range(count):
+        hash1 = ext.put('external-raw', input_)
+    for i in range(extra):
+        hash2 = ext.put('external-raw', np.random.randn(4, 3, 2))
+
+    assert_true(hash1 in ext.fetch('hash'))
+    assert_equal(count, (ext & {'hash': hash1}).fetch1('count'))
+    assert_equal(len(ext), 1 + extra)
+
+    output_ = ext.get(hash1)
+    assert_array_equal(input_, output_)
+
+    ext.remove(hash1)
+    assert_equal(count-1, (ext & {'hash': hash1}).fetch1('count'))

tests/test_settings.py

@@ -37,13 +37,6 @@ def test_singleton2():
     assert_true(conf['dummy.val'] == 2, 'Config does not behave like a singleton.')
 
 
-@raises(ValueError)
-def test_nested_check():
-    """Testing nested rejection"""
-    dummy = {'dummy.testval': {'notallowed': 2}}
-    dj.config.update(dummy)
-
-
 @raises(dj.DataJointError)
 def test_validator():
     """Testing validator"""
@@ -73,12 +66,6 @@ def test_repr():
     assert_equal(repr(dj.config), pprint.pformat(dj.config._conf, indent=4))
 
 
-@raises(ValueError)
-def test_nested_check2():
-    """Testing nested dictionary rejection"""
-    dj.config['dummy'] = {'dummy2':2}
-
-
 def test_save():
     """Testing save of config"""
     tmpfile = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))