From 0d6bea7e83b9a86a80486ff8ee766b53d51722d8 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 31 Aug 2020 16:47:39 -0400 Subject: [PATCH 1/6] CLI: Add debug argument --- fmriprep/cli/parser.py | 9 ++++++++- fmriprep/config.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/fmriprep/cli/parser.py b/fmriprep/cli/parser.py index ae4a7ff76..6a56fce66 100644 --- a/fmriprep/cli/parser.py +++ b/fmriprep/cli/parser.py @@ -533,9 +533,16 @@ def _bids_filter(value): "improve FMRIPREP and provides an indicator of real " "world usage crucial for obtaining funding.", ) + g_other.add_argument( + "--debug", + action="store", + nargs="+", + choices=config.DEBUG_MODES + ("all",), + help="Debug mode(s) to enable. 'all' is alias for all available modes.", + ) + g_other.add_argument( "--sloppy", - dest="debug", action="store_true", default=False, help="Use low-quality tools for speed - TESTING ONLY", diff --git a/fmriprep/config.py b/fmriprep/config.py index 76b9dfd59..6adb67003 100644 --- a/fmriprep/config.py +++ b/fmriprep/config.py @@ -187,6 +187,11 @@ pass +# Debug modes are names that influence the exposure of internal details to +# the user, either through additional derivatives or increased verbosity +DEBUG_MODES = ("compcor",) + + class _Config: """An abstract class forbidding instantiation.""" @@ -359,8 +364,10 @@ class execution(_Config): """A dictionary of BIDS selection filters.""" boilerplate_only = False """Only generate a boilerplate.""" - debug = False + sloppy = False """Run in sloppy mode (meaning, suboptimal parameters that minimize run-time).""" + debug = [] + """Debug mode(s).""" echo_idx = None """Select a particular echo for multi-echo EPI datasets.""" fs_license_file = _fs_license @@ -456,6 +463,9 @@ def init(cls): for k, v in filters.items() } + if "all" in cls.debug: + cls.debug = list(DEBUG_MODES) + # These variables are not necessary anymore del _fs_license From 8b55527276d56f0d7cd896f6f7b67ecb42946d82 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 1 Sep 2020 16:50:51 -0400 Subject: [PATCH 2/6] ENH: Output compcor masks if --debug compcor passed --- fmriprep/workflows/bold/base.py | 6 ++++-- fmriprep/workflows/bold/confounds.py | 4 +++- fmriprep/workflows/bold/outputs.py | 23 ++++++++++++++++++++--- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/fmriprep/workflows/bold/base.py b/fmriprep/workflows/bold/base.py index ec0b11f81..86f555a3a 100644 --- a/fmriprep/workflows/bold/base.py +++ b/fmriprep/workflows/bold/base.py @@ -303,6 +303,8 @@ def init_func_preproc_wf(bold_file): ('cifti_metadata', 'inputnode.cifti_metadata'), ('cifti_density', 'inputnode.cifti_density'), ('confounds_metadata', 'inputnode.confounds_metadata'), + ('acompcor_masks', 'inputnode.acompcor_masks'), + ('tcompcor_mask', 'inputnode.tcompcor_mask'), ]), ]) @@ -496,9 +498,9 @@ def init_func_preproc_wf(bold_file): ('outputnode.bold_mask', 'inputnode.bold_mask')]), (bold_confounds_wf, outputnode, [ ('outputnode.confounds_file', 'confounds'), - ]), - (bold_confounds_wf, outputnode, [ ('outputnode.confounds_metadata', 'confounds_metadata'), + ('outputnode.acompcor_masks', 'acompcor_masks'), + ('outputnode.tcompcor_mask', 'tcompcor_mask'), ]), # Connect bold_bold_trans_wf (bold_split, bold_bold_trans_wf, [ diff --git a/fmriprep/workflows/bold/confounds.py b/fmriprep/workflows/bold/confounds.py index bc62fd817..5672355fe 100644 --- a/fmriprep/workflows/bold/confounds.py +++ b/fmriprep/workflows/bold/confounds.py @@ -193,7 +193,7 @@ def init_bold_confs_wf( 'skip_vols', 't1w_mask', 't1w_tpms', 't1_bold_xform']), name='inputnode') outputnode = pe.Node(niu.IdentityInterface( - fields=['confounds_file', 'confounds_metadata']), + fields=['confounds_file', 'confounds_metadata', 'acompcor_masks', 'tcompcor_mask']), name='outputnode') # DVARS @@ -392,6 +392,8 @@ def _select_cols(table): # Set outputs (spike_regress, outputnode, [('confounds_file', 'confounds_file')]), (mrg_conf_metadata2, outputnode, [('out_dict', 'confounds_metadata')]), + (tcompcor, outputnode, [("high_variance_masks", "tcompcor_mask")]), + (acc_msk_bin, outputnode, [("out_file", "acompcor_masks")]), (inputnode, rois_plot, [('bold', 'in_file'), ('bold_mask', 'in_mask')]), (tcompcor, mrg_compcor, [('high_variance_masks', 'in1')]), diff --git a/fmriprep/workflows/bold/outputs.py b/fmriprep/workflows/bold/outputs.py index 358a6f47c..eae04efd5 100644 --- a/fmriprep/workflows/bold/outputs.py +++ b/fmriprep/workflows/bold/outputs.py @@ -4,8 +4,9 @@ from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu -from ...config import DEFAULT_MEMORY_MIN_GB -from ...interfaces import DerivativesDataSink +from fmriprep import config +from fmriprep.config import DEFAULT_MEMORY_MIN_GB +from fmriprep.interfaces import DerivativesDataSink def init_func_derivatives_wf( @@ -62,7 +63,7 @@ def init_func_derivatives_wf( 'bold_mask_native', 'cifti_variant', 'cifti_metadata', 'cifti_density', 'confounds', 'confounds_metadata', 'melodic_mix', 'nonaggr_denoised_file', 'source_file', 'surf_files', 'surf_refs', 'template', 'spatial_reference', - 'bold2anat_xfm', 'anat2bold_xfm']), + 'bold2anat_xfm', 'anat2bold_xfm', 'acompcor_masks', 'tcompcor_mask']), name='inputnode') raw_sources = pe.Node(niu.Function(function=_bids_relative), name='raw_sources') @@ -338,6 +339,22 @@ def init_func_derivatives_wf( (('cifti_metadata', _read_json), 'meta_dict')]) ]) + if "compcor" in config.execution.debug: + ds_acompcor_masks = pe.Node( + DerivativesDataSink( + base_directory=output_dir, desc=[f"CompCor{_}" for _ in "CWA"], suffix="mask"), + name="ds_acompcor_masks", run_without_submitting=True) + ds_tcompcor_mask = pe.Node( + DerivativesDataSink( + base_directory=output_dir, desc="CompCorT", suffix="mask"), + name="ds_tcompcor_mask", run_without_submitting=True) + workflow.connect([ + (inputnode, ds_acompcor_masks, [("acompcor_masks", "in_file"), + ("source_file", "source_file")]), + (inputnode, ds_tcompcor_mask, [("tcompcor_mask", "in_file"), + ("source_file", "source_file")]), + ]) + return workflow From 0e246780add03b0a1527ef55aa753716c35efd11 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 1 Sep 2020 16:54:41 -0400 Subject: [PATCH 3/6] CI: Check for debug outputs --- .circleci/config.yml | 2 +- .circleci/ds054_fasttrack_outputs.txt | 8 ++++++++ .circleci/ds054_outputs.txt | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb7b65ea2..bc5542780 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -601,7 +601,7 @@ jobs: ${FASTRACK_ARG} \ --fs-no-reconall --sloppy \ --output-spaces MNI152NLin2009cAsym:res-2 anat func \ - --mem_mb 4096 --nthreads 2 -vv + --mem_mb 4096 --nthreads 2 -vv --debug compcor - run: name: Checking outputs of fMRIPrep command: | diff --git a/.circleci/ds054_fasttrack_outputs.txt b/.circleci/ds054_fasttrack_outputs.txt index d1b8bb442..fdde2ab66 100644 --- a/.circleci/ds054_fasttrack_outputs.txt +++ b/.circleci/ds054_fasttrack_outputs.txt @@ -13,6 +13,10 @@ fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorW_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_from-scanner_to-T1w_mode-image_xfm.txt @@ -32,6 +36,10 @@ fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorW_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_from-scanner_to-T1w_mode-image_xfm.txt diff --git a/.circleci/ds054_outputs.txt b/.circleci/ds054_outputs.txt index ee7cdc052..f5ae26644 100644 --- a/.circleci/ds054_outputs.txt +++ b/.circleci/ds054_outputs.txt @@ -32,6 +32,10 @@ fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorW_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_from-scanner_to-T1w_mode-image_xfm.txt @@ -51,6 +55,10 @@ fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorW_mask.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_from-scanner_to-T1w_mode-image_xfm.txt From 3e3eefa7dc3e91a7f3f9984e48f001eb6e6766fa Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Sep 2020 07:20:45 -0700 Subject: [PATCH 4/6] fix: 's/config.execution.debug/config.execution.sloppy/g' but do not change the occurrence in ``fmriprep/workflows/bold/confounds.py`` because that one is intended. --- fmriprep/workflows/base.py | 2 +- fmriprep/workflows/bold/base.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index a7e2692a3..f510c3bad 100644 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -237,7 +237,7 @@ def init_single_subject_wf(subject_id): # Preprocessing of T1w (includes registration to MNI) anat_preproc_wf = init_anat_preproc_wf( bids_root=str(config.execution.bids_dir), - debug=config.execution.debug is True, + debug=config.execution.sloppy is True, existing_derivatives=anat_derivatives, freesurfer=config.workflow.run_reconall, hires=config.workflow.hires, diff --git a/fmriprep/workflows/bold/base.py b/fmriprep/workflows/bold/base.py index 86f555a3a..d77dd345e 100644 --- a/fmriprep/workflows/bold/base.py +++ b/fmriprep/workflows/bold/base.py @@ -335,7 +335,7 @@ def init_func_preproc_wf(bold_file): mem_gb=mem_gb['resampled'], name='bold_reg_wf', omp_nthreads=omp_nthreads, - sloppy=config.execution.debug, + sloppy=config.execution.sloppy, use_bbr=config.workflow.use_bbr, use_compression=False, ) @@ -406,7 +406,7 @@ def init_func_preproc_wf(bold_file): # SDC (SUSCEPTIBILITY DISTORTION CORRECTION) or bypass ########################## bold_sdc_wf = init_sdc_estimate_wf(fmaps, metadata, omp_nthreads=omp_nthreads, - debug=config.execution.debug) + debug=config.execution.sloppy) # MULTI-ECHO EPI DATA ############################################# if multiecho: # instantiate relevant interfaces, imports From 7dc91a3ce7f44d8af25d6f481cfc783423749b59 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Sep 2020 17:24:37 +0200 Subject: [PATCH 5/6] Update fmriprep/workflows/base.py [skip ci] Co-authored-by: Mathias Goncalves --- fmriprep/workflows/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index f510c3bad..8a19e074b 100644 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -237,7 +237,7 @@ def init_single_subject_wf(subject_id): # Preprocessing of T1w (includes registration to MNI) anat_preproc_wf = init_anat_preproc_wf( bids_root=str(config.execution.bids_dir), - debug=config.execution.sloppy is True, + debug=config.execution.sloppy, existing_derivatives=anat_derivatives, freesurfer=config.workflow.run_reconall, hires=config.workflow.hires, From 6b97020b060b3228ca74f52d14aa867f7bf26409 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 4 Sep 2020 08:27:56 -0700 Subject: [PATCH 6/6] fix: ensure compression of new CompCor masks --- fmriprep/workflows/bold/outputs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fmriprep/workflows/bold/outputs.py b/fmriprep/workflows/bold/outputs.py index eae04efd5..0edb97fa4 100644 --- a/fmriprep/workflows/bold/outputs.py +++ b/fmriprep/workflows/bold/outputs.py @@ -342,11 +342,12 @@ def init_func_derivatives_wf( if "compcor" in config.execution.debug: ds_acompcor_masks = pe.Node( DerivativesDataSink( - base_directory=output_dir, desc=[f"CompCor{_}" for _ in "CWA"], suffix="mask"), + base_directory=output_dir, desc=[f"CompCor{_}" for _ in "CWA"], + suffix="mask", compress=True), name="ds_acompcor_masks", run_without_submitting=True) ds_tcompcor_mask = pe.Node( DerivativesDataSink( - base_directory=output_dir, desc="CompCorT", suffix="mask"), + base_directory=output_dir, desc="CompCorT", suffix="mask", compress=True), name="ds_tcompcor_mask", run_without_submitting=True) workflow.connect([ (inputnode, ds_acompcor_masks, [("acompcor_masks", "in_file"),