From 2d0068d3a7fa523540de347307ebb080d5fd0ba2 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 6 May 2022 11:34:12 -0500 Subject: [PATCH 1/8] Rename process.py to populate_all.py --- workflow_miniscope/populate_all.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 workflow_miniscope/populate_all.py diff --git a/workflow_miniscope/populate_all.py b/workflow_miniscope/populate_all.py new file mode 100644 index 0000000..d424ab8 --- /dev/null +++ b/workflow_miniscope/populate_all.py @@ -0,0 +1,31 @@ +from workflow_miniscope.pipeline import miniscope +import warnings + +warnings.filterwarnings('ignore') + + +def run(display_progress=True): + + populate_settings = {'display_progress': display_progress, + 'reserve_jobs': False, + 'suppress_errors': False} + + print('\n---- Populate imported and computed tables ----') + + miniscope.RecordingInfo.populate(**populate_settings) + + miniscope.Processing.populate(**populate_settings) + + miniscope.MotionCorrection.populate(**populate_settings) + + miniscope.Segmentation.populate(**populate_settings) + + miniscope.Fluorescence.populate(**populate_settings) + + miniscope.Activity.populate(**populate_settings) + + print('\n---- Successfully completed workflow_miniscope/populate_all.py ----') + + +if __name__ == '__main__': + run() \ No newline at end of file From 8f69a6dadec8507016fd5e60f32994677b862425 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 6 May 2022 17:06:57 +0000 Subject: [PATCH 2/8] Change `acquisition_hardware` to `equipment` --- workflow_miniscope/ingest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow_miniscope/ingest.py b/workflow_miniscope/ingest.py index 392462e..316dc33 100644 --- a/workflow_miniscope/ingest.py +++ b/workflow_miniscope/ingest.py @@ -51,7 +51,7 @@ def ingest_sessions(session_csv_path='./user_data/sessions.csv'): metadata_filepath.stat().st_ctime) with open(metadata_filepath) as json_file: recording_metadata = json.load(json_file) - acquisition_hardware = recursive_search('deviceType', + equipment = recursive_search('deviceType', recording_metadata) break except OSError: @@ -62,7 +62,8 @@ def ingest_sessions(session_csv_path='./user_data/sessions.csv'): session_key = dict(subject=single_session['subject'], session_datetime=recording_time) if session_key not in session.Session(): - hardware_list.append(dict(acquisition_hardware=acquisition_hardware)) + hardware_list.append(dict(equipment=equipment, + modality='miniscope')) session_list.append(session_key) @@ -71,7 +72,7 @@ def ingest_sessions(session_csv_path='./user_data/sessions.csv'): recording_list.append(dict(**session_key, recording_id=0, # Assumes one recording per session - acquisition_hardware=acquisition_hardware, + equipment=equipment, acquisition_software=acquisition_software, recording_directory=session_dir.as_posix())) From fa640f6ccf4c1553591a86e146224cf16314ba73 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 6 May 2022 17:14:15 +0000 Subject: [PATCH 3/8] Add element-event --- docker/Dockerfile.dev | 3 +++ docker/Dockerfile.test | 3 +++ docker/docker-compose-dev.yaml | 1 + docker/docker-compose-test.yaml | 1 + 4 files changed, 8 insertions(+) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 17cef9f..f4bf1c1 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -18,6 +18,7 @@ WORKDIR /main RUN mkdir /main/element-lab \ /main/element-animal \ /main/element-session \ + /main/element-event \ /main/element-interface \ /main/element-miniscope \ /main/workflow-miniscope @@ -26,6 +27,7 @@ RUN mkdir /main/element-lab \ COPY --chown=anaconda:anaconda ./element-lab /main/element-lab COPY --chown=anaconda:anaconda ./element-animal /main/element-animal COPY --chown=anaconda:anaconda ./element-session /main/element-session +COPY --chown=anaconda:anaconda ./element-event /main/element-event COPY --chown=anaconda:anaconda ./element-interface /main/element-interface COPY --chown=anaconda:anaconda ./element-miniscope /main/element-miniscope COPY --chown=anaconda:anaconda ./workflow-miniscope /main/workflow-miniscope @@ -34,6 +36,7 @@ COPY --chown=anaconda:anaconda ./workflow-miniscope /main/workflow-miniscope RUN pip install -e /main/element-lab RUN pip install -e /main/element-animal RUN pip install -e /main/element-session +RUN pip install -e /main/element-event RUN pip install -e /main/element-interface RUN pip install -e /main/element-miniscope RUN pip install -e /main/workflow-miniscope diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 6d04352..806afcc 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -31,6 +31,7 @@ WORKDIR /main RUN mkdir /main/element-lab \ /main/element-animal \ /main/element-session \ + /main/element-event \ /main/element-interface \ /main/element-miniscope \ /main/workflow-miniscope @@ -38,6 +39,7 @@ RUN mkdir /main/element-lab \ COPY --chown=anaconda:anaconda ./element-lab /main/element-lab COPY --chown=anaconda:anaconda ./element-animal /main/element-animal COPY --chown=anaconda:anaconda ./element-session /main/element-session +COPY --chown=anaconda:anaconda ./element-event /main/element-event COPY --chown=anaconda:anaconda ./element-interface /main/element-interface COPY --chown=anaconda:anaconda ./element-miniscope /main/element-miniscope COPY --chown=anaconda:anaconda ./workflow-miniscope /main/workflow-miniscope @@ -45,6 +47,7 @@ COPY --chown=anaconda:anaconda ./workflow-miniscope /main/workflow-miniscope RUN pip install -e /main/element-lab RUN pip install -e /main/element-animal RUN pip install -e /main/element-session +RUN pip install -e /main/element-event RUN pip install -e /main/element-interface RUN pip install -e /main/element-miniscope RUN rm -f /main/workflow-miniscope/dj_local_conf.json diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index 1fb5cde..489f20d 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -28,6 +28,7 @@ services: - ../../element-lab:/main/element-lab - ../../element-animal:/main/element-animal - ../../element-session:/main/element-session + - ../../element-event:/main/element-event - ../../element-interface:/main/element-interface - ../../element-miniscope:/main/element-miniscope - ..:/main/workflow-miniscope diff --git a/docker/docker-compose-test.yaml b/docker/docker-compose-test.yaml index 847e1b9..bf1bdcb 100644 --- a/docker/docker-compose-test.yaml +++ b/docker/docker-compose-test.yaml @@ -39,6 +39,7 @@ services: - ../../element-lab:/main/element-lab - ../../element-animal:/main/element-animal - ../../element-session:/main/element-session + - ../../element-event:/main/element-event - ../../element-interface:/main/element-interface - ../../element-miniscope:/main/element-miniscope - ..:/main/workflow-miniscope From b6ed0410fac2df6fdc9f7e159e4836c277a3cab8 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Mon, 24 Apr 2023 20:52:44 -0500 Subject: [PATCH 4/8] Add GitHub Actions workflows --- .../u24_workflow_before_release.yaml | 18 +++++++++++++++++ .../workflows/u24_workflow_release_call.yaml | 20 +++++++++++++++++++ .../u24_workflow_tag_to_release.yaml | 15 ++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 .github/workflows/u24_workflow_before_release.yaml create mode 100644 .github/workflows/u24_workflow_release_call.yaml create mode 100644 .github/workflows/u24_workflow_tag_to_release.yaml diff --git a/.github/workflows/u24_workflow_before_release.yaml b/.github/workflows/u24_workflow_before_release.yaml new file mode 100644 index 0000000..28a5ff5 --- /dev/null +++ b/.github/workflows/u24_workflow_before_release.yaml @@ -0,0 +1,18 @@ +name: u24_workflow_before_release_0.0.1 +on: + pull_request: + push: + branches: + - '**' + tags-ignore: + - '**' + workflow_dispatch: +jobs: + call_context_check: + uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main + call_u24_workflow_build_debian: + uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main + with: + jhub_ver: 1.4.2 + py_ver: 3.9 + dist: debian diff --git a/.github/workflows/u24_workflow_release_call.yaml b/.github/workflows/u24_workflow_release_call.yaml new file mode 100644 index 0000000..8196673 --- /dev/null +++ b/.github/workflows/u24_workflow_release_call.yaml @@ -0,0 +1,20 @@ +name: u24_workflow_release_call_0.0.1 +on: + workflow_run: + workflows: ["u24_workflow_tag_to_release_0.0.1"] + types: + - completed +jobs: + call_context_check: + uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main + call_u24_workflow_release_debian: + if: >- + github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' + uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_release.yaml@main + with: + jhub_ver: 1.4.2 + py_ver: 3.9 + dist: debian + secrets: + REGISTRY_USERNAME: ${{secrets.DOCKER_USERNAME}} + REGISTRY_PASSWORD: ${{secrets.DOCKER_PASSWORD}} diff --git a/.github/workflows/u24_workflow_tag_to_release.yaml b/.github/workflows/u24_workflow_tag_to_release.yaml new file mode 100644 index 0000000..3a6ce58 --- /dev/null +++ b/.github/workflows/u24_workflow_tag_to_release.yaml @@ -0,0 +1,15 @@ +name: u24_workflow_tag_to_release_0.0.1 +on: + push: + tags: + - '*.*.*' + - 'test*.*.*' +jobs: + call_context_check: + uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main + call_u24_workflow_build_debian: + uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main + with: + jhub_ver: 1.4.2 + py_ver: 3.9 + dist: debian From 1392b24d81733c8b1aecc68aa32bbe65b2183daf Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Mon, 24 Apr 2023 20:54:23 -0500 Subject: [PATCH 5/8] Update changelog --- CHANGELOG.md | 9 +++------ workflow_miniscope/version.py | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9fd93f..1a4d7b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,14 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## 0.2.1 - Unreleased +## [0.2.1] - 2023-04-24 + Add - Apply `black` formatting to codebase + Update - Move `recursive_search` function + Update - Revise Pytests/docker structure + Update - Add reference schema for supplemental tables + Add - Visualization notebook ++ Add - GitHub Actions workflows ## 0.2.0 - 2022-07-11 @@ -35,8 +36,4 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Version -[0.2.1]: https://github.com/datajoint/element-miniscope/releases/tag/0.2.1 -[0.2.0]: https://github.com/datajoint/element-miniscope/releases/tag/0.2.0 -[0.1.0]: https://github.com/datajoint/element-miniscope/releases/tag/0.1.0 -[0.0.2]: https://github.com/datajoint/element-miniscope/releases/tag/0.0.2 -[0.0.1]: https://github.com/datajoint/element-miniscope/releases/tag/0.0.1 +[0.2.1]: https://github.com/datajoint/workflow-miniscope/releases/tag/0.2.1 diff --git a/workflow_miniscope/version.py b/workflow_miniscope/version.py index 3ebc7f3..7e30021 100644 --- a/workflow_miniscope/version.py +++ b/workflow_miniscope/version.py @@ -1,3 +1,2 @@ """Package metadata.""" -# Also update in docker-compose.yaml __version__ = "0.2.1" From 4373997aa6a624fcdad5b52530dc5215184484a1 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Mon, 24 Apr 2023 20:56:10 -0500 Subject: [PATCH 6/8] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4d7b9..1a650ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Visualization notebook + Add - GitHub Actions workflows -## 0.2.0 - 2022-07-11 +## [0.2.0] - 2022-07-11 + Update - Docker and Compose files + Update - Requirements for latest releases of Elements @@ -37,3 +37,4 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Version [0.2.1]: https://github.com/datajoint/workflow-miniscope/releases/tag/0.2.1 +[0.2.0]: https://github.com/datajoint/workflow-miniscope/releases/tag/0.2.0 From 5e655db3eb716ac5a1aa7dd58abd2d15d8ff2a32 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Mon, 24 Apr 2023 20:58:31 -0500 Subject: [PATCH 7/8] Remove duplicate file --- workflow_miniscope/populate_all.py | 31 ------------------------------ 1 file changed, 31 deletions(-) diff --git a/workflow_miniscope/populate_all.py b/workflow_miniscope/populate_all.py index d424ab8..e69de29 100644 --- a/workflow_miniscope/populate_all.py +++ b/workflow_miniscope/populate_all.py @@ -1,31 +0,0 @@ -from workflow_miniscope.pipeline import miniscope -import warnings - -warnings.filterwarnings('ignore') - - -def run(display_progress=True): - - populate_settings = {'display_progress': display_progress, - 'reserve_jobs': False, - 'suppress_errors': False} - - print('\n---- Populate imported and computed tables ----') - - miniscope.RecordingInfo.populate(**populate_settings) - - miniscope.Processing.populate(**populate_settings) - - miniscope.MotionCorrection.populate(**populate_settings) - - miniscope.Segmentation.populate(**populate_settings) - - miniscope.Fluorescence.populate(**populate_settings) - - miniscope.Activity.populate(**populate_settings) - - print('\n---- Successfully completed workflow_miniscope/populate_all.py ----') - - -if __name__ == '__main__': - run() \ No newline at end of file From 3bab6110c20b10582318fef877d0af07b40ca943 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Mon, 24 Apr 2023 20:59:15 -0500 Subject: [PATCH 8/8] Delete file --- workflow_miniscope/populate_all.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 workflow_miniscope/populate_all.py diff --git a/workflow_miniscope/populate_all.py b/workflow_miniscope/populate_all.py deleted file mode 100644 index e69de29..0000000