add test for configurable blobs

datajoint/heading.py

@@ -197,7 +197,7 @@ def init_from_database(self, conn, database, table_name):
 
             # recognize configurable fields
             configurable_field = re.match(
-                r'^:(?P<type>(blob|external|attach)(-\w+)?):(?P<comment>.*)$', attr['comment'])
+                r'^:(?P<type>(blob|external|attach)(-\w*)?):(?P<comment>.*)$', attr['comment'])
             if configurable_field is None:
                 attr['is_external'] = False
                 attr['is_attachment'] = False

tests/schema_external.py

@@ -11,19 +11,22 @@
 schema = dj.schema(PREFIX + '_extern', connection=dj.conn(**CONN_INFO))
 
 
-dj.config['external'] = {
+dj.config['stores'] = {
+    '-': {
     'protocol': 'file',
-    'location': 'dj-store/external'}
+    'location': 'dj-store/external'
+    },
 
-dj.config['external-raw'] = {
+    '-raw': {
     'protocol': 'file',
-    'location': 'dj-store/raw'}
+    'location': 'dj-store/raw'},
 
-dj.config['external-compute'] = {
+    '-compute': {
     'protocol': 's3',
     'location': '/datajoint-projects/test',
     'user': 'djtest',
     'token': '2e05709792545ce'}
+}
 
 dj.config['cache'] = tempfile.mkdtemp('dj-cache')
 
@@ -33,7 +36,7 @@ class Simple(dj.Manual):
     definition = """
     simple  : int
     ---
-    item  : external-raw
+    item  : blob-
     """
 
 
@@ -64,8 +67,8 @@ class Image(dj.Computed):
     -> Seed
     -> Dimension
     ----
-    img  : external-raw    #  objects are stored as specified by dj.config['external-raw']
-    neg : external    # objects are stored as specified by dj.config['external']
+    img  : blob-raw    #  objects are stored as specified by dj.config['stores'][-raw']
+    neg : blob-    # objects are stored as specified by dj.config['stores']['-']
     """
 
     def make(self, key):

tests/schema_legacy_external.py

@@ -0,0 +1,74 @@
+"""
+a schema for testing external attributes using legacy syntax pre-version 0.12.0
+"""
+
+import tempfile
+import datajoint as dj
+
+from . import PREFIX, CONN_INFO
+import numpy as np
+
+schema = dj.schema(PREFIX + '_legacy_extern', connection=dj.conn(**CONN_INFO))
+
+
+dj.config['external'] = {
+    'protocol': 'file',
+    'location': 'dj-legacy/external'}
+
+dj.config['external-raw'] = {
+    'protocol': 'file',
+    'location': 'dj-legacy/raw'}
+
+dj.config['external-compute'] = {
+    'protocol': 's3',
+    'location': '/datajoint-legacy/test',
+    'user': 'djtest',
+    'token': '2e05709792545ce'}
+
+dj.config['cache'] = tempfile.mkdtemp('dj-legacy-cache')
+
+
+@schema
+class Simple(dj.Manual):
+    definition = """
+    simple  : int
+    ---
+    item  : external-raw
+    """
+
+
+@schema
+class Seed(dj.Lookup):
+    definition = """
+    seed :  int
+    """
+    contents = zip(range(4))
+
+
+@schema
+class Dimension(dj.Lookup):
+    definition = """
+    dim  : int
+    ---
+    dimensions  : blob
+    """
+    contents = (
+        [0, [100, 50]],
+        [1, [3, 4, 8, 6]])
+
+
+@schema
+class Image(dj.Computed):
+    definition = """
+    # table for storing
+    -> Seed
+    -> Dimension
+    ----
+    img  : external-raw    #  objects are stored as specified by dj.config['external-raw']
+    neg : external    # objects are stored as specified by dj.config['external']
+    """
+
+    def make(self, key):
+        np.random.seed(key['seed'])
+        img = np.random.rand(*(Dimension() & key).fetch1('dimensions'))
+        self.insert1(dict(key, img=img, neg=-img.astype(np.float32)))

tests/test_external_class.py

@@ -5,7 +5,7 @@
 
 
 def test_heading():
-    heading = modu.Simple().heading
+    heading = modu.Simple.heading
     assert_true('item' in heading)
     assert_true(heading['item'].is_external)
 

tests/test_legacy_external.py

@@ -0,0 +1,28 @@
+import numpy as np
+from numpy.testing import assert_array_equal
+from nose.tools import assert_true, assert_equal
+from datajoint.external import ExternalTable
+from datajoint.blob import pack, unpack
+
+from . schema_legacy_external import schema
+
+
+def test_external_put():
+    """
+    external storage put and get and remove
+    """
+    ext = ExternalTable(schema.connection, schema.database)
+    input_ = np.random.randn(3, 7, 8)
+    count = 7
+    extra = 3
+    for i in range(count):
+        hash1 = ext.put('external-raw', pack(input_))
+    for i in range(extra):
+        hash2 = ext.put('external-raw', pack(np.random.randn(4, 3, 2)))
+
+    fetched_hashes = ext.fetch('hash')
+    assert_true(all(hash in fetched_hashes for hash in (hash1, hash2)))
+    assert_equal(len(ext), 1 + extra)
+
+    output_ = unpack(ext.get(hash1))
+    assert_array_equal(input_, output_)

tests/test_legacy_external_class.py

@@ -0,0 +1,63 @@
+from nose.tools import assert_true, assert_list_equal, raises
+from numpy.testing import assert_almost_equal
+import datajoint as dj
+from . import schema_legacy_external as modu
+
+
+def test_heading():
+    heading = modu.Simple().heading
+    assert_true('item' in heading)
+    assert_true(heading['item'].is_external)
+
+
+def test_insert_and_fetch():
+    original_list = [1, 3, 8]
+    modu.Simple().insert1(dict(simple=1, item=original_list))
+    # test fetch
+    q = (modu.Simple() & {'simple': 1}).fetch('item')[0]
+    assert_list_equal(list(q), original_list)
+    # test fetch1 as a tuple
+    q = (modu.Simple() & {'simple': 1}).fetch1('item')
+    assert_list_equal(list(q), original_list)
+    # test fetch1 as a dict
+    q = (modu.Simple() & {'simple': 1}).fetch1()
+    assert_list_equal(list(q['item']), original_list)
+    # test without cache
+    previous_cache = dj.config['cache']
+    dj.config['cache'] = None
+    q = (modu.Simple() & {'simple': 1}).fetch1()
+    assert_list_equal(list(q['item']), original_list)
+    # test with cache
+    dj.config['cache'] = previous_cache
+    q = (modu.Simple() & {'simple': 1}).fetch1()
+    assert_list_equal(list(q['item']), original_list)
+
+
+def test_populate():
+    image = modu.Image()
+    image.populate()
+    remaining, total = image.progress()
+    image.external_table.clean_store('raw')
+    assert_true(total == len(modu.Dimension() * modu.Seed()) and remaining == 0)
+    for img, neg, dimensions in zip(*(image * modu.Dimension()).fetch('img', 'neg', 'dimensions')):
+        assert_list_equal(list(img.shape), list(dimensions))
+        assert_almost_equal(img, -neg)
+    image.delete()
+    image.external_table.delete_garbage()
+    image.external_table.clean_store('raw')
+
+
+@raises(dj.DataJointError)
+def test_drop():
+    image = modu.Image()
+    image.populate()
+    image.external_table.drop()
+
+
+@raises(dj.DataJointError)
+def test_delete():
+    image = modu.Image()
+    image.external_table.delete()
+
+
+