WIP: Apply suggestions 3. All except Docker

- Bump version
- Add recording assertion
- Remove path steps from session assertion
- Revert pipeline session import
- Update test_data fixture for mult roots

docker/docker-compose-test.yaml

@@ -20,6 +20,7 @@ services:
       dockerfile: ./docker/Dockerfile.test
       args:
         - GITHUB_USERNAME=datajoint
+    env_file: .env
     image: workflow-miniscope-test:0.1.0
     container_name: workflow-miniscope-test
     environment:

tests/__init__.py

@@ -68,32 +68,19 @@ def dj_config():
 
 @pytest.fixture(autouse=True)
 def test_data(dj_config):
-    test_data_dir = pathlib.Path(dj.config['custom']['miniscope_root_data_dir'])
-
-    test_data_exists = np.all([(test_data_dir / p).exists() for p in sessions_dirs])
+    mini_root_dirs = dj.config['custom']['miniscope_root_data_dir']
+
+    test_data_exists = np.all([find_full_path(mini_root_dirs, p).exists() 
+                               for p in sessions_dirs])
 
     if not test_data_exists:
-        try:
-            dj.config['custom'].update({
-                'djarchive.client.endpoint': os.environ['DJARCHIVE_CLIENT_ENDPOINT'],
-                'djarchive.client.bucket': os.environ['DJARCHIVE_CLIENT_BUCKET'],
-                'djarchive.client.access_key': os.environ['DJARCHIVE_CLIENT_ACCESSKEY'],
-                'djarchive.client.secret_key': os.environ['DJARCHIVE_CLIENT_SECRETKEY']
-            })
-        except KeyError as e:
-            raise FileNotFoundError(
-                f'Test data not available at {test_data_dir}.'
-                f'\nAttempting to download from DJArchive,'
-                f' but no credentials found in environment variables.'
-                f'\nError: {str(e)}')
-
         import djarchive_client
 
-        client = djarchive_client.client()
-
-        client.download('workflow-miniscope-test-set',
-                        'v1',
-                        str(test_data_dir), create_target=False)
+        if not isinstance(mini_root_dirs, list):
+            mini_root_dirs = list(mini_root_dirs)
+        djarchive_client.client().download('workflow-miniscope-test-set',
+                                           'v1',
+                                           str(mini_root_dirs[0]), create_target=False)
     return
 
 
@@ -187,10 +174,10 @@ def testdata_paths():
 
 
 @pytest.fixture
-def caiman2D_paramset(pipeline):
+def caiman_paramset(pipeline):
     miniscope = pipeline['miniscope']
 
-    params_caiman_2d = dict(decay_time=0.4,
+    params_caiman = dict(decay_time=0.4,
                             pw_rigid=False,
                             max_shifts=(5, 5),
                             gSig_filt=(3, 3),
@@ -230,9 +217,9 @@ def caiman2D_paramset(pipeline):
         processing_method='caiman',
         paramset_id=0,
         paramset_desc='Calcium imaging analysis with CaImAn using default parameters',
-        params=params_caiman_2d)
+        params=params_caiman)
 
-    yield params_caiman_2d
+    yield params_caiman
 
     if _tear_down:
         if verbose:
@@ -259,9 +246,7 @@ def recording_info(pipeline, ingest_sessions):
 
 
 @pytest.fixture
-def processing_tasks(pipeline, caiman2D_paramset, recording_info):
-    global is_multi_scan_processing
-
+def processing_tasks(pipeline, caiman_paramset, recording_info):
     miniscope = pipeline['miniscope']
     session = pipeline['session']
     get_miniscope_root_data_dir = pipeline['get_miniscope_root_data_dir']

tests/test_ingest.py

@@ -2,12 +2,12 @@
 
 from . import (dj_config, pipeline, subjects_csv, ingest_subjects,
                sessions_csv, ingest_sessions,
-               testdata_paths, caiman2D_paramset, recording_info, processing_tasks,
+               testdata_paths, caiman_paramset, recording_info, processing_tasks,
                processing, curations)
 
 
 __all__ = ['dj_config', 'pipeline', 'subjects_csv', 'ingest_subjects', 'sessions_csv',
-           'ingest_sessions', 'testdata_paths', 'caiman2D_paramset',
+           'ingest_sessions', 'testdata_paths', 'caiman_paramset',
            'recording_info', 'processing_tasks', 'processing', 'curations']
 
 
@@ -22,18 +22,16 @@ def test_ingest_sessions(pipeline, sessions_csv, ingest_sessions):
     get_miniscope_root_data_dir = pipeline['get_miniscope_root_data_dir']
 
     assert len(session.Session()) == 1
+    assert len(miniscope.Recording()) == 1
 
     sessions, _ = sessions_csv
     sess = sessions[1].split(",")[1]
-    sess_dir_full = find_full_path(get_miniscope_root_data_dir(), sess)
-    root_dir = find_root_directory(get_miniscope_root_data_dir(), sess_dir_full)
-    sess_dir = sess_dir_full.relative_to(root_dir)
     assert (session.SessionDirectory
             & {'subject': sessions[1].split(",")[0]}
-            ).fetch1('session_dir') == str(sess_dir)
+            ).fetch1('session_dir') == sess
 
 
-def test_paramset_insert(caiman2D_paramset, pipeline):
+def test_paramset_insert(caiman_paramset, pipeline):
     miniscope = pipeline['miniscope']
     from element_interface.utils import dict_to_uuid
 
@@ -42,4 +40,4 @@ def test_paramset_insert(caiman2D_paramset, pipeline):
                                             'param_set_hash')
     assert method == 'caiman'
     assert desc == 'Calcium imaging analysis with CaImAn using default parameters'
-    assert dict_to_uuid(caiman2D_paramset) == paramset_hash
+    assert dict_to_uuid(caiman_paramset) == paramset_hash

tests/test_populate.py

@@ -1,9 +1,9 @@
 from . import (dj_config, pipeline, subjects_csv, ingest_subjects,
-               sessions_csv, ingest_sessions, testdata_paths, caiman2D_paramset,
+               sessions_csv, ingest_sessions, testdata_paths, caiman_paramset,
                recording_info, processing_tasks, processing, curations)
 
 __all__ = ['dj_config', 'pipeline', 'subjects_csv', 'ingest_subjects', 'sessions_csv',
-           'ingest_sessions', 'testdata_paths', 'caiman2D_paramset', 'recording_info',
+           'ingest_sessions', 'testdata_paths', 'caiman_paramset', 'recording_info',
            'processing_tasks', 'processing', 'curations']
 
 

workflow_miniscope/paths.py

@@ -2,8 +2,13 @@
 
 
 def get_miniscope_root_data_dir():
-    return dj.config.get('custom', {}).get('miniscope_root_data_dir', None)
-
+    mini_root_dirs = dj.config.get('custom', {}).get('miniscope_root_data_dir')
+    if not mini_root_dirs:
+        return None
+    elif not isinstance(mini_root_dirs, list):
+        return list(mini_root_dirs)
+    else:
+        return mini_root_dirs
 
 def get_session_directory(session_key: dict) -> str:
     from .pipeline import session

workflow_miniscope/pipeline.py

@@ -2,7 +2,7 @@
 
 from element_lab import lab
 from element_animal import subject
-from element_session import session
+from element_session import session_with_datetime as session
 from element_event import trial, event
 from element_miniscope import miniscope
 

workflow_miniscope/version.py

@@ -1,2 +1,2 @@
 """Package metadata"""
-__version__ = '0.1.0'
+__version__ = '0.1.1'