diff --git a/nipype/algorithms/tests/test_auto_AddCSVRow.py b/nipype/algorithms/tests/test_auto_AddCSVRow.py index 3090c8b6a9..a8c4467cbf 100644 --- a/nipype/algorithms/tests/test_auto_AddCSVRow.py +++ b/nipype/algorithms/tests/test_auto_AddCSVRow.py @@ -6,7 +6,10 @@ def test_AddCSVRow_inputs(): input_map = dict( _outputs=dict(usedefault=True, ), - in_file=dict(mandatory=True, ), + in_file=dict( + extensions=None, + mandatory=True, + ), ) inputs = AddCSVRow.input_spec() diff --git a/nipype/interfaces/afni/preprocess.py b/nipype/interfaces/afni/preprocess.py index ee81ee7e2f..a9775fb763 100644 --- a/nipype/interfaces/afni/preprocess.py +++ b/nipype/interfaces/afni/preprocess.py @@ -1000,7 +1000,7 @@ class ClipLevel(AFNICommandBase): input_spec = ClipLevelInputSpec output_spec = ClipLevelOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() @@ -2115,21 +2115,15 @@ class Seg(AFNICommandBase): input_spec = SegInputSpec output_spec = AFNICommandOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): - - import glob - - outputs = self._outputs() + def _list_outputs(self): + from nipype.utils.filemanip import Path + prefix = 'Segsy' if isdefined(self.inputs.prefix): - outfile = os.path.join(os.getcwd(), self.inputs.prefix, - 'Classes+*.BRIK') - else: - outfile = os.path.join(os.getcwd(), 'Segsy', 'Classes+*.BRIK') - - outputs.out_file = glob.glob(outfile)[0] + prefix = self.inputs.prefix - return outputs + return {'out_file': str( + sorted(Path() / prefix).glob('Classes+*.BRIK')[0])} class SkullStripInputSpec(AFNICommandInputSpec): diff --git a/nipype/interfaces/afni/tests/test_auto_Allineate.py b/nipype/interfaces/afni/tests/test_auto_Allineate.py index 0b110e669a..e1565376bd 100644 --- a/nipype/interfaces/afni/tests/test_auto_Allineate.py +++ b/nipype/interfaces/afni/tests/test_auto_Allineate.py @@ -86,6 +86,7 @@ def test_Allineate_inputs(): ), out_weight_file=dict( argstr='-wtprefix %s', + extensions=None, xor=['allcostx'], ), outputtype=dict(), diff --git a/nipype/interfaces/afni/tests/test_auto_ClipLevel.py b/nipype/interfaces/afni/tests/test_auto_ClipLevel.py index f9b3dbf705..96f928a809 100644 --- a/nipype/interfaces/afni/tests/test_auto_ClipLevel.py +++ b/nipype/interfaces/afni/tests/test_auto_ClipLevel.py @@ -17,6 +17,7 @@ def test_ClipLevel_inputs(): ), grad=dict( argstr='-grad %s', + extensions=None, position=3, xor='doall', ), diff --git a/nipype/interfaces/afni/tests/test_auto_LocalBistat.py b/nipype/interfaces/afni/tests/test_auto_LocalBistat.py index ed3e61d74d..4632e4cf7d 100644 --- a/nipype/interfaces/afni/tests/test_auto_LocalBistat.py +++ b/nipype/interfaces/afni/tests/test_auto_LocalBistat.py @@ -26,7 +26,10 @@ def test_LocalBistat_inputs(): mandatory=True, position=-1, ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), neighborhood=dict( argstr="-nbhd '%s(%s)'", mandatory=True, @@ -37,6 +40,7 @@ def test_LocalBistat_inputs(): ), out_file=dict( argstr='-prefix %s', + extensions=None, keep_extension=True, name_source='in_file1', name_template='%s_bistat', @@ -49,6 +53,7 @@ def test_LocalBistat_inputs(): ), weight_file=dict( argstr='-weight %s', + extensions=None, xor=['automask'], ), ) diff --git a/nipype/interfaces/afni/tests/test_auto_Localstat.py b/nipype/interfaces/afni/tests/test_auto_Localstat.py index 011ce44da8..62dc800941 100644 --- a/nipype/interfaces/afni/tests/test_auto_Localstat.py +++ b/nipype/interfaces/afni/tests/test_auto_Localstat.py @@ -21,7 +21,10 @@ def test_Localstat_inputs(): mandatory=True, position=-1, ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), neighborhood=dict( argstr="-nbhd '%s(%s)'", mandatory=True, @@ -33,6 +36,7 @@ def test_Localstat_inputs(): ), out_file=dict( argstr='-prefix %s', + extensions=None, keep_extension=True, name_source='in_file', name_template='%s_localstat', diff --git a/nipype/interfaces/afni/tests/test_auto_NwarpApply.py b/nipype/interfaces/afni/tests/test_auto_NwarpApply.py index e00457f4f3..87388c65ec 100644 --- a/nipype/interfaces/afni/tests/test_auto_NwarpApply.py +++ b/nipype/interfaces/afni/tests/test_auto_NwarpApply.py @@ -20,7 +20,10 @@ def test_NwarpApply_inputs(): usedefault=True, ), inv_warp=dict(argstr='-iwarp', ), - master=dict(argstr='-master %s', ), + master=dict( + argstr='-master %s', + extensions=None, + ), out_file=dict( argstr='-prefix %s', extensions=None, diff --git a/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py b/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py index f8e664a727..114fe53fba 100644 --- a/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py +++ b/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py @@ -30,6 +30,7 @@ def test_OneDToolPy_inputs(): show_censor_count=dict(argstr='-show_censor_count', ), show_cormat_warnings=dict( argstr='-show_cormat_warnings |& tee %s', + extensions=None, position=-1, xor=['out_file'], ), diff --git a/nipype/interfaces/afni/tests/test_auto_Qwarp.py b/nipype/interfaces/afni/tests/test_auto_Qwarp.py index 3ef8c2e9b2..0b8a9e38ec 100644 --- a/nipype/interfaces/afni/tests/test_auto_Qwarp.py +++ b/nipype/interfaces/afni/tests/test_auto_Qwarp.py @@ -125,7 +125,10 @@ def test_Qwarp_inputs(): name_source=['in_file'], name_template='ppp_%s', ), - out_weight_file=dict(argstr='-wtprefix %s', ), + out_weight_file=dict( + argstr='-wtprefix %s', + extensions=None, + ), outputtype=dict(), overwrite=dict(argstr='-overwrite', ), pblur=dict(argstr='-pblur %s', ), diff --git a/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py b/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py index ca27a0d682..e282d0d0a5 100644 --- a/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py +++ b/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py @@ -125,7 +125,10 @@ def test_QwarpPlusMinus_inputs(): position=0, usedefault=True, ), - out_weight_file=dict(argstr='-wtprefix %s', ), + out_weight_file=dict( + argstr='-wtprefix %s', + extensions=None, + ), outputtype=dict(), overwrite=dict(argstr='-overwrite', ), pblur=dict(argstr='-pblur %s', ), diff --git a/nipype/interfaces/afni/tests/test_auto_ROIStats.py b/nipype/interfaces/afni/tests/test_auto_ROIStats.py index d3c956f7c5..c7fc517ccc 100644 --- a/nipype/interfaces/afni/tests/test_auto_ROIStats.py +++ b/nipype/interfaces/afni/tests/test_auto_ROIStats.py @@ -49,7 +49,10 @@ def test_ROIStats_inputs(): position=-1, ), quiet=dict(argstr='-quiet', ), - roisel=dict(argstr='-roisel %s', ), + roisel=dict( + argstr='-roisel %s', + extensions=None, + ), stat=dict(argstr='%s...', ), zerofill=dict( argstr='-zerofill %s', diff --git a/nipype/interfaces/afni/tests/test_auto_ReHo.py b/nipype/interfaces/afni/tests/test_auto_ReHo.py index 0edcedcdaf..cf3e468159 100644 --- a/nipype/interfaces/afni/tests/test_auto_ReHo.py +++ b/nipype/interfaces/afni/tests/test_auto_ReHo.py @@ -25,13 +25,17 @@ def test_ReHo_inputs(): argstr='-in_rois %s', extensions=None, ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), neighborhood=dict( argstr='-nneigh %s', xor=['sphere', 'ellipsoid'], ), out_file=dict( argstr='-prefix %s', + extensions=None, keep_extension=True, name_source='in_file', name_template='%s_reho', diff --git a/nipype/interfaces/afni/tests/test_auto_Remlfit.py b/nipype/interfaces/afni/tests/test_auto_Remlfit.py index f2e6703bf6..05ee75210e 100644 --- a/nipype/interfaces/afni/tests/test_auto_Remlfit.py +++ b/nipype/interfaces/afni/tests/test_auto_Remlfit.py @@ -58,6 +58,7 @@ def test_Remlfit_inputs(): ), matim=dict( argstr='-matim %s', + extensions=None, xor=['matrix'], ), matrix=dict( diff --git a/nipype/interfaces/afni/tests/test_auto_Resample.py b/nipype/interfaces/afni/tests/test_auto_Resample.py index 560d883d75..f63e1347f5 100644 --- a/nipype/interfaces/afni/tests/test_auto_Resample.py +++ b/nipype/interfaces/afni/tests/test_auto_Resample.py @@ -17,7 +17,10 @@ def test_Resample_inputs(): mandatory=True, position=-1, ), - master=dict(argstr='-master %s', ), + master=dict( + argstr='-master %s', + extensions=None, + ), num_threads=dict( nohash=True, usedefault=True, diff --git a/nipype/interfaces/afni/tests/test_auto_TCorrMap.py b/nipype/interfaces/afni/tests/test_auto_TCorrMap.py index 1ea4c0790e..209473fe6f 100644 --- a/nipype/interfaces/afni/tests/test_auto_TCorrMap.py +++ b/nipype/interfaces/afni/tests/test_auto_TCorrMap.py @@ -93,7 +93,10 @@ def test_TCorrMap_inputs(): name_source='in_file', suffix='_qmean', ), - regress_out_timeseries=dict(argstr='-ort %s', ), + regress_out_timeseries=dict( + argstr='-ort %s', + extensions=None, + ), seeds=dict( argstr='-seed %s', extensions=None, diff --git a/nipype/interfaces/afni/tests/test_auto_Zeropad.py b/nipype/interfaces/afni/tests/test_auto_Zeropad.py index abeceda432..1bd80cfad8 100644 --- a/nipype/interfaces/afni/tests/test_auto_Zeropad.py +++ b/nipype/interfaces/afni/tests/test_auto_Zeropad.py @@ -55,6 +55,7 @@ def test_Zeropad_inputs(): ), master=dict( argstr='-master %s', + extensions=None, xor=['I', 'S', 'A', 'P', 'L', 'R', 'z', 'RL', 'AP', 'IS', 'mm'], ), mm=dict( diff --git a/nipype/interfaces/afni/utils.py b/nipype/interfaces/afni/utils.py index e16e1bbd79..125d2fc2d1 100644 --- a/nipype/interfaces/afni/utils.py +++ b/nipype/interfaces/afni/utils.py @@ -210,9 +210,9 @@ class Autobox(AFNICommand): input_spec = AutoboxInputSpec output_spec = AutoboxOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = super(Autobox, self).aggregate_outputs( - runtime, needed_outputs) + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(Autobox, self).aggregate_outputs(runtime, needed_outputs, + rebase_cwd=rebase_cwd) pattern = 'x=(?P-?\d+)\.\.(?P-?\d+) '\ 'y=(?P-?\d+)\.\.(?P-?\d+) '\ 'z=(?P-?\d+)\.\.(?P-?\d+)' @@ -286,7 +286,7 @@ class BrickStat(AFNICommandBase): input_spec = BrickStatInputSpec output_spec = BrickStatOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() diff --git a/nipype/interfaces/ants/registration.py b/nipype/interfaces/ants/registration.py index c08c7646a9..0cb8bcba10 100644 --- a/nipype/interfaces/ants/registration.py +++ b/nipype/interfaces/ants/registration.py @@ -1487,7 +1487,7 @@ def _format_arg(self, opt, spec, val): return self._mask_constructor() return super(MeasureImageSimilarity, self)._format_arg(opt, spec, val) - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() stdout = runtime.stdout.split('\n') outputs.similarity = float(stdout[0]) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index 0011c925dd..a368986f19 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -32,13 +32,13 @@ from ... import config, logging, LooseVersion from ...utils.provenance import write_provenance from ...utils.misc import str2bool, rgetcwd -from ...utils.filemanip import (FileNotFoundError, split_filename, - which, get_dependencies) +from ...utils.filemanip import (split_filename, which, get_dependencies, + FileNotFoundError) from ...utils.subprocess import run_command from ...external.due import due -from .traits_extension import traits, isdefined +from .traits_extension import traits, isdefined, rebase_path_traits from .specs import (BaseInterfaceInputSpec, CommandLineInputSpec, StdOutCommandLineInputSpec, MpiCommandLineInputSpec, get_filecopy_info) @@ -110,7 +110,7 @@ def run(self): """Execute the command.""" raise NotImplementedError - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): """Called to populate outputs""" raise NotImplementedError @@ -204,10 +204,10 @@ def _check_requires(self, spec, name, value): ] if any(values) and isdefined(value): if len(values) > 1: - fmt = ("%s requires values for inputs %s because '%s' is set. " + fmt = ("%s requires values for inputs %s because '%s' is set. " "For a list of required inputs, see %s.help()") else: - fmt = ("%s requires a value for input %s because '%s' is set. " + fmt = ("%s requires a value for input %s because '%s' is set. " "For a list of required inputs, see %s.help()") msg = fmt % (self.__class__.__name__, ', '.join("'%s'" % req for req in spec.requires), @@ -299,7 +299,7 @@ def _duecredit_cite(self): r['path'] = self.__module__ due.cite(**r) - def run(self, cwd=None, ignore_exception=None, **inputs): + def run(self, cwd=None, ignore_exception=None, rebase_cwd=None, **inputs): """Execute this interface. This interface will not raise an exception if runtime.returncode is @@ -327,6 +327,9 @@ def run(self, cwd=None, ignore_exception=None, **inputs): if cwd is None: cwd = syscwd + if rebase_cwd: + rebase_cwd = cwd + os.chdir(cwd) # Change to the interface wd enable_rm = config.resource_monitor and self.resource_monitor @@ -442,42 +445,46 @@ def run(self, cwd=None, ignore_exception=None, **inputs): return results def _list_outputs(self): - """ List the expected outputs - """ + """List the expected outputs.""" if self.output_spec: raise NotImplementedError else: return None def aggregate_outputs(self, runtime=None, needed_outputs=None): - """ Collate expected outputs and check for existence - """ + """Collate expected outputs and check for existence.""" + outputs = self._outputs() # Generate an output spec object + if needed_outputs is not None and not needed_outputs: + return outputs - predicted_outputs = self._list_outputs() - outputs = self._outputs() - if predicted_outputs: - _unavailable_outputs = [] - if outputs: - _unavailable_outputs = \ - self._check_version_requirements(self._outputs()) - for key, val in list(predicted_outputs.items()): - if needed_outputs and key not in needed_outputs: - continue - if key in _unavailable_outputs: - raise KeyError(('Output trait %s not available in version ' - '%s of interface %s. Please inform ' - 'developers.') % (key, self.version, - self.__class__.__name__)) - try: - setattr(outputs, key, val) - except TraitError as error: - if getattr(error, 'info', - 'default').startswith('an existing'): - msg = ("File/Directory '%s' not found for %s output " - "'%s'." % (val, self.__class__.__name__, key)) - raise FileNotFoundError(msg) - raise error + predicted_outputs = self._list_outputs() # Predictions from _list_outputs + if not predicted_outputs: + return outputs + # Precalculate the list of output trait names that should be + # aggregated + aggregate_names = set(predicted_outputs.keys()) + if needed_outputs: + aggregate_names = set(needed_outputs).intersection(aggregate_names) + + if outputs and aggregate_names: + _na_outputs = self._check_version_requirements(outputs) + na_names = aggregate_names.intersection(set(_na_outputs)) + if na_names: + raise TypeError("""\ +Output trait(s) %s not available in version %s of interface %s.\ +""" % (', '.join(na_names), self.version, self.__class__.__name__)) + + for key in aggregate_names: + val = predicted_outputs[key] + try: + setattr(outputs, key, val) + except TraitError as error: + if 'an existing' in getattr(error, 'info', 'default'): + msg = "No such file or directory for output '%s' of a %s interface" % \ + (key, self.__class__.__name__) + raise FileNotFoundError(val, message=msg) + raise error return outputs @property diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py index 0fd1d27674..4aa5d4a3b8 100644 --- a/nipype/interfaces/base/support.py +++ b/nipype/interfaces/base/support.py @@ -225,13 +225,15 @@ def __init__(self, runtime, inputs=None, outputs=None, - provenance=None): + provenance=None, + error=None): self._version = 2.0 self.interface = interface self.runtime = runtime self.inputs = inputs self.outputs = outputs self.provenance = provenance + self.error = error @property def version(self): diff --git a/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py b/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py index e39dc3acaa..95afcd3216 100644 --- a/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py +++ b/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py @@ -12,6 +12,7 @@ def test_StdOutCommandLine_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/base/tests/test_core.py b/nipype/interfaces/base/tests/test_core.py index 265edc444f..e6baad36a2 100644 --- a/nipype/interfaces/base/tests/test_core.py +++ b/nipype/interfaces/base/tests/test_core.py @@ -10,6 +10,7 @@ import pytest from .... import config +from ....utils.filemanip import Path from ....testing import example_data from ... import base as nib from ..support import _inputs_help @@ -289,7 +290,7 @@ def _list_outputs(self): return {'foo': 1} obj = DerivedInterface1() - with pytest.raises(KeyError): + with pytest.raises(TypeError): obj.run() @@ -529,3 +530,75 @@ def _run_interface(self, runtime): with pytest.raises(RuntimeError): BrokenRuntime().run() + + +# def test_aggregate_outputs(tmpdir): +# tmpdir.chdir() +# b_value = '%s' % tmpdir.join('filename.txt') +# c_value = '%s' % tmpdir.join('sub') + +# class TestInterface(nib.BaseInterface): +# class input_spec(nib.TraitedSpec): +# a = nib.traits.Any() +# class output_spec(nib.TraitedSpec): +# b = nib.File() +# c = nib.Directory(exists=True) +# d = nib.File() + +# def _list_outputs(self): +# return {'b': b_value, 'c': c_value, 'd': './log.txt'} + +# # Test aggregate_outputs without needed_outputs +# outputs = TestInterface().aggregate_outputs(needed_outputs=[]) +# assert outputs.b is nib.Undefined + +# # Test that only those in needed_outputs are returned +# outputs = TestInterface().aggregate_outputs( +# needed_outputs=['b', 'd']) +# assert outputs.c is nib.Undefined +# assert outputs.b == b_value +# assert Path(outputs.b).is_absolute() +# assert not Path(outputs.d).is_absolute() + +# # Test that traits are actually validated at aggregation +# with pytest.raises(OSError): +# outputs = TestInterface().aggregate_outputs( +# needed_outputs=['b', 'c'], rebase_cwd='%s' % tmpdir) + +# # Test that rebase_cwd actually returns relative paths +# tmpdir.mkdir('sub') +# outputs = TestInterface().aggregate_outputs(rebase_cwd='%s' % tmpdir) + +# assert outputs.b == 'filename.txt' +# assert not Path(outputs.b).is_absolute() +# assert not Path(outputs.c).is_absolute() +# assert not Path(outputs.d).is_absolute() + + +# def test_aggregate_outputs_compounds(): +# outputs_dict = { +# 'b': tuple(['/some/folder/f%d.txt' % d +# for d in range(1, 3)]), +# 'c': ['/some/folder/f%d.txt' % d for d in range(1, 5)], +# 'd': 2.0, +# 'e': ['/some/folder/f%d.txt' % d for d in range(1, 4)] +# } + +# class TestInterface(nib.BaseInterface): +# class input_spec(nib.TraitedSpec): +# a = nib.traits.Any() +# class output_spec(nib.TraitedSpec): +# b = nib.traits.Tuple(nib.File(extensions=['.txt']), +# nib.File(extensions=['.txt'])) +# c = nib.traits.List(nib.File()) +# d = nib.traits.Either(nib.File(), nib.traits.Float()) +# e = nib.OutputMultiObject(nib.File()) + +# def _list_outputs(self): +# return outputs_dict + +# outputs = TestInterface().aggregate_outputs(rebase_cwd='/some/folder') +# assert outputs.d == 2.0 +# assert outputs.b == ('f1.txt', 'f2.txt') +# assert outputs.e == ['f%d.txt' % (d + 1) +# for d in range(len(outputs_dict['e']))] diff --git a/nipype/interfaces/base/tests/test_traits_extension.py b/nipype/interfaces/base/tests/test_traits_extension.py new file mode 100644 index 0000000000..4a2b884921 --- /dev/null +++ b/nipype/interfaces/base/tests/test_traits_extension.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, unicode_literals + +from ... import base as nib +from ..traits_extension import rebase_path_traits, resolve_path_traits, Path + + +class _test_spec(nib.TraitedSpec): + a = nib.traits.File() + b = nib.traits.Tuple(nib.File(), + nib.File()) + c = nib.traits.List(nib.File()) + d = nib.traits.Either(nib.File(), nib.traits.Float()) + e = nib.OutputMultiObject(nib.File()) + f = nib.traits.Dict(nib.Str, nib.File()) + g = nib.traits.Either(nib.File, nib.Str) + h = nib.Str + ee = nib.OutputMultiObject(nib.Str) + + +def test_rebase_path_traits(): + """Check rebase_path_traits.""" + spec = _test_spec() + + a = rebase_path_traits( + spec.trait('a'), '/some/path/f1.txt', '/some/path') + assert '%s' % a == 'f1.txt' + + b = rebase_path_traits( + spec.trait('b'), ('/some/path/f1.txt', '/some/path/f2.txt'), '/some/path') + assert b == (Path('f1.txt'), Path('f2.txt')) + + c = rebase_path_traits( + spec.trait('c'), ['/some/path/f1.txt', '/some/path/f2.txt', '/some/path/f3.txt'], + '/some/path') + assert c == [Path('f1.txt'), Path('f2.txt'), Path('f3.txt')] + + d = rebase_path_traits( + spec.trait('d'), 2.0, '/some/path') + assert d == 2.0 + + d = rebase_path_traits( + spec.trait('d'), '/some/path/either.txt', '/some/path') + assert '%s' % d == 'either.txt' + + e = rebase_path_traits( + spec.trait('e'), ['/some/path/f1.txt', '/some/path/f2.txt', '/some/path/f3.txt'], + '/some/path') + assert e == [Path('f1.txt'), Path('f2.txt'), Path('f3.txt')] + + e = rebase_path_traits( + spec.trait('e'), [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]], + '/some/path') + assert e == [[Path('f1.txt'), Path('f2.txt')], [[Path('f3.txt')]]] + + f = rebase_path_traits( + spec.trait('f'), {'1': '/some/path/f1.txt'}, '/some/path') + assert f == {'1': Path('f1.txt')} + + g = rebase_path_traits( + spec.trait('g'), 'some/path/either.txt', '/some/path') + assert '%s' % g == 'some/path/either.txt' + + g = rebase_path_traits( + spec.trait('g'), '/some/path/either.txt', '/some') + assert '%s' % g == 'path/either.txt' + + g = rebase_path_traits(spec.trait('g'), 'string', '/some') + assert '%s' % g == 'string' + + g = rebase_path_traits(spec.trait('g'), '2', '/some/path') + assert g == '2' # You dont want this one to be a Path + + h = rebase_path_traits(spec.trait('h'), '2', '/some/path') + assert h == '2' + + ee = rebase_path_traits( + spec.trait('ee'), [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]], + '/some/path') + assert ee == [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]] + + +def test_resolve_path_traits(): + """Check resolve_path_traits.""" + spec = _test_spec() + + a = resolve_path_traits( + spec.trait('a'), 'f1.txt', '/some/path') + assert a == Path('/some/path/f1.txt') + + b = resolve_path_traits( + spec.trait('b'), ('f1.txt', 'f2.txt'), '/some/path') + assert b == (Path('/some/path/f1.txt'), Path('/some/path/f2.txt')) + + c = resolve_path_traits( + spec.trait('c'), ['f1.txt', 'f2.txt', 'f3.txt'], + '/some/path') + assert c == [Path('/some/path/f1.txt'), Path('/some/path/f2.txt'), Path('/some/path/f3.txt')] + + d = resolve_path_traits( + spec.trait('d'), 2.0, '/some/path') + assert d == 2.0 + + d = resolve_path_traits( + spec.trait('d'), 'either.txt', '/some/path') + assert '%s' % d == '/some/path/either.txt' + + e = resolve_path_traits( + spec.trait('e'), ['f1.txt', 'f2.txt', 'f3.txt'], + '/some/path') + assert e == [Path('/some/path/f1.txt'), Path('/some/path/f2.txt'), Path('/some/path/f3.txt')] + + e = resolve_path_traits( + spec.trait('e'), [['f1.txt', 'f2.txt'], [['f3.txt']]], + '/some/path') + assert e == [[Path('/some/path/f1.txt'), Path('/some/path/f2.txt')], + [[Path('/some/path/f3.txt')]]] + + f = resolve_path_traits( + spec.trait('f'), {'1': 'path/f1.txt'}, '/some') + assert f == {'1': Path('/some/path/f1.txt')} + + g = resolve_path_traits( + spec.trait('g'), '/either.txt', '/some/path') + assert g == Path('/either.txt') + + # This is a problematic case, it is impossible to know whether this + # was meant to be a string or a file. + # Commented out because in this implementation, strings take precedence + # g = resolve_path_traits( + # spec.trait('g'), 'path/either.txt', '/some') + # assert g == Path('/some/path/either.txt') + + # This is a problematic case, it is impossible to know whether this + # was meant to be a string or a file. + g = resolve_path_traits(spec.trait('g'), 'string', '/some') + assert g == 'string' + + # This is a problematic case, it is impossible to know whether this + # was meant to be a string or a file. + g = resolve_path_traits(spec.trait('g'), '2', '/some/path') + assert g == '2' # You dont want this one to be a Path + + h = resolve_path_traits(spec.trait('h'), '2', '/some/path') + assert h == '2' + + ee = resolve_path_traits( + spec.trait('ee'), [['f1.txt', 'f2.txt'], [['f3.txt']]], + '/some/path') + assert ee == [['f1.txt', 'f2.txt'], [['f3.txt']]] diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index de215beb96..e0a8705ad1 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -30,6 +30,7 @@ import traits.api as traits from traits.trait_handlers import TraitType, NoDefaultSpecified from traits.trait_base import _Undefined +from traits.traits import _TraitMaker, trait_from from traits.api import Unicode from future import standard_library @@ -304,6 +305,11 @@ def validate(self, objekt, name, value, return_pathlike=False): return value +# Patch in traits these two new +traits.File = File +traits.Directory = Directory + + class ImageFile(File): """Defines a trait whose value must be a known neuroimaging file.""" @@ -465,3 +471,120 @@ class InputMultiObject(MultiObject): InputMultiPath = InputMultiObject OutputMultiPath = OutputMultiObject + + +class Tuple(traits.BaseTuple): + """Defines a new type of Tuple trait that reports inner types.""" + + def init_fast_validator(self, *args): + """Set up the C-level fast validator.""" + super(Tuple, self).init_fast_validator(*args) + self.fast_validate = args + + def inner_traits(self): + """Return the *inner trait* (or traits) for this trait.""" + return self.types + + +class PatchedEither(TraitType): + """Defines a trait whose value can be any of of a specified list of traits.""" + + def __init__(self, *traits, **metadata): + """Create a trait whose value can be any of of a specified list of traits.""" + metadata['alternatives'] = tuple(trait_from(t) for t in traits) + self.trait_maker = _TraitMaker( + metadata.pop("default", None), *traits, **metadata) + + def as_ctrait(self): + """Return a CTrait corresponding to the trait defined by this class.""" + return self.trait_maker.as_ctrait() + + +traits.Tuple = Tuple +traits.Either = PatchedEither + + +def _rebase_path(value, cwd): + if isinstance(value, list): + return [_rebase_path(v, cwd) for v in value] + + try: + value = Path(value) + except TypeError: + pass + else: + try: + value = Path(value).relative_to(cwd) + except ValueError: + pass + return value + + +def rebase_path_traits(thistrait, value, cwd): + """Rebase a BasePath-derived trait given an interface spec.""" + if thistrait.is_trait_type(BasePath): + value = _rebase_path(value, cwd) + elif thistrait.is_trait_type(traits.List): + innertrait, = thistrait.inner_traits + if not isinstance(value, (list, tuple)): + value = rebase_path_traits(innertrait, value, cwd) + else: + value = [rebase_path_traits(innertrait, v, cwd) + for v in value] + elif thistrait.is_trait_type(traits.Dict): + _, innertrait = thistrait.inner_traits + value = {k: rebase_path_traits(innertrait, v, cwd) + for k, v in value.items()} + elif thistrait.is_trait_type(Tuple): + value = tuple([rebase_path_traits(subtrait, v, cwd) + for subtrait, v in zip(thistrait.inner_traits, value)]) + elif thistrait.alternatives: + is_str = [f.is_trait_type((traits.String, traits.BaseStr, traits.BaseBytes, Str)) + for f in thistrait.alternatives] + if any(is_str) and isinstance(value, (bytes, str)) and not value.startswith('/'): + return value + for subtrait in thistrait.alternatives: + value = rebase_path_traits(subtrait, value, cwd) + return value + + +def _resolve_path(value, cwd): + if isinstance(value, list): + return [_resolve_path(v, cwd) for v in value] + + try: + value = Path(value) + except TypeError: + pass + else: + if not value.is_absolute(): + value = Path(cwd) / value + return value + + +def resolve_path_traits(thistrait, value, cwd): + """Resolve a BasePath-derived trait given an interface spec.""" + if thistrait.is_trait_type(BasePath): + value = _resolve_path(value, cwd) + elif thistrait.is_trait_type(traits.List): + innertrait, = thistrait.inner_traits + if not isinstance(value, (list, tuple)): + value = resolve_path_traits(innertrait, value, cwd) + else: + value = [resolve_path_traits(innertrait, v, cwd) + for v in value] + elif thistrait.is_trait_type(traits.Dict): + _, innertrait = thistrait.inner_traits + value = {k: resolve_path_traits(innertrait, v, cwd) + for k, v in value.items()} + elif thistrait.is_trait_type(Tuple): + value = tuple([resolve_path_traits(subtrait, v, cwd) + for subtrait, v in zip(thistrait.inner_traits, value)]) + elif thistrait.alternatives: + is_str = [f.is_trait_type((traits.String, traits.BaseStr, traits.BaseBytes, Str)) + for f in thistrait.alternatives] + if any(is_str) and isinstance(value, (bytes, str)) and not value.startswith('/'): + return value + for subtrait in thistrait.alternatives: + value = resolve_path_traits(subtrait, value, cwd) + return value diff --git a/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py b/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py index 28170946ac..c40082d836 100644 --- a/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py +++ b/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py @@ -50,6 +50,7 @@ def test_AnalyzeHeader_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py b/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py index 66ac282175..edf38864fa 100644 --- a/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py +++ b/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py @@ -24,6 +24,7 @@ def test_ComputeEigensystem_inputs(): maxcomponents=dict(argstr='-maxcomponents %d', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py b/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py index 9cfae77b2f..75604df01b 100644 --- a/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py +++ b/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py @@ -20,6 +20,7 @@ def test_ComputeFractionalAnisotropy_inputs(): inputmodel=dict(argstr='-inputmodel %s', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py b/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py index 1443a253bd..4d31fa884c 100644 --- a/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py +++ b/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py @@ -20,6 +20,7 @@ def test_ComputeTensorTrace_inputs(): inputmodel=dict(argstr='-inputmodel %s', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_DTIFit.py b/nipype/interfaces/camino/tests/test_auto_DTIFit.py index 26d27d57d2..0870a77752 100644 --- a/nipype/interfaces/camino/tests/test_auto_DTIFit.py +++ b/nipype/interfaces/camino/tests/test_auto_DTIFit.py @@ -26,6 +26,7 @@ def test_DTIFit_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py b/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py index f7caf77ef1..3242163c3a 100644 --- a/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py +++ b/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py @@ -28,6 +28,7 @@ def test_DTLUTGen_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py b/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py index 350c604c99..9b8a74be6a 100644 --- a/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py +++ b/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py @@ -40,6 +40,7 @@ def test_FSL2Scheme_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py b/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py index 29864bef82..2c013cf216 100644 --- a/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py +++ b/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py @@ -18,6 +18,7 @@ def test_Image2Voxel_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_LinRecon.py b/nipype/interfaces/camino/tests/test_auto_LinRecon.py index 996d8f4b99..147cfaab5e 100644 --- a/nipype/interfaces/camino/tests/test_auto_LinRecon.py +++ b/nipype/interfaces/camino/tests/test_auto_LinRecon.py @@ -24,6 +24,7 @@ def test_LinRecon_inputs(): normalize=dict(argstr='-normalize', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_MESD.py b/nipype/interfaces/camino/tests/test_auto_MESD.py index 57dbbf3b28..f7dfecfdb9 100644 --- a/nipype/interfaces/camino/tests/test_auto_MESD.py +++ b/nipype/interfaces/camino/tests/test_auto_MESD.py @@ -42,6 +42,7 @@ def test_MESD_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ModelFit.py b/nipype/interfaces/camino/tests/test_auto_ModelFit.py index e5c16ec975..6d49969eb6 100644 --- a/nipype/interfaces/camino/tests/test_auto_ModelFit.py +++ b/nipype/interfaces/camino/tests/test_auto_ModelFit.py @@ -34,6 +34,7 @@ def test_ModelFit_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py b/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py index 0a5583b03e..65e0210268 100644 --- a/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py +++ b/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py @@ -26,6 +26,7 @@ def test_NIfTIDT2Camino_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py b/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py index c918a372f1..e5cdea5cae 100644 --- a/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py +++ b/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py @@ -36,6 +36,7 @@ def test_PicoPDFs_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py b/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py index c30ef08d8c..0386e0d54f 100644 --- a/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py +++ b/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py @@ -62,6 +62,7 @@ def test_ProcStreamlines_inputs(): noresample=dict(argstr='-noresample', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_QBallMX.py b/nipype/interfaces/camino/tests/test_auto_QBallMX.py index 7f7d0bc99f..f452f26350 100644 --- a/nipype/interfaces/camino/tests/test_auto_QBallMX.py +++ b/nipype/interfaces/camino/tests/test_auto_QBallMX.py @@ -20,6 +20,7 @@ def test_QBallMX_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py b/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py index 795138ea89..220e116255 100644 --- a/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py +++ b/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py @@ -35,6 +35,7 @@ def test_SFLUTGen_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py b/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py index 80222fce4c..1b71553676 100644 --- a/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py +++ b/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py @@ -27,6 +27,7 @@ def test_SFPICOCalibData_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_SFPeaks.py b/nipype/interfaces/camino/tests/test_auto_SFPeaks.py index 8db250c58c..49ac58aa06 100644 --- a/nipype/interfaces/camino/tests/test_auto_SFPeaks.py +++ b/nipype/interfaces/camino/tests/test_auto_SFPeaks.py @@ -38,6 +38,7 @@ def test_SFPeaks_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_Shredder.py b/nipype/interfaces/camino/tests/test_auto_Shredder.py index 594b35fc60..2d8ec43589 100644 --- a/nipype/interfaces/camino/tests/test_auto_Shredder.py +++ b/nipype/interfaces/camino/tests/test_auto_Shredder.py @@ -28,6 +28,7 @@ def test_Shredder_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_TractShredder.py b/nipype/interfaces/camino/tests/test_auto_TractShredder.py index eeed244533..e4df010c60 100644 --- a/nipype/interfaces/camino/tests/test_auto_TractShredder.py +++ b/nipype/interfaces/camino/tests/test_auto_TractShredder.py @@ -28,6 +28,7 @@ def test_TractShredder_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py b/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py index 72b6f106d8..bd8d295572 100644 --- a/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py +++ b/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py @@ -25,6 +25,7 @@ def test_VtkStreamlines_inputs(): interpolatescalars=dict(argstr='-interpolatescalars', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/freesurfer/preprocess.py b/nipype/interfaces/freesurfer/preprocess.py index 2941968f85..6577b1547e 100644 --- a/nipype/interfaces/freesurfer/preprocess.py +++ b/nipype/interfaces/freesurfer/preprocess.py @@ -2815,7 +2815,7 @@ def run(self, **inputs): copy2subjdir(self, originalfile, folder='mri') return super(SegmentCC, self).run(**inputs) - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): # it is necessary to find the output files and move # them to the correct loacation predicted_outputs = self._list_outputs() @@ -2842,7 +2842,7 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None): os.makedirs(os.path.dirname(out_tmp)) shutil.move(out_tmp, out_file) return super(SegmentCC, self).aggregate_outputs( - runtime, needed_outputs) + runtime, needed_outputs, rebase_cwd) class SegmentWMInputSpec(FSTraitedSpec): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py b/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py index f3bfd5ad62..4d56d217c7 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py @@ -7,7 +7,10 @@ def test_CALabel_inputs(): input_map = dict( align=dict(argstr='-align', ), args=dict(argstr='%s', ), - aseg=dict(argstr='-aseg %s', ), + aseg=dict( + argstr='-aseg %s', + extensions=None, + ), environ=dict( nohash=True, usedefault=True, @@ -26,7 +29,10 @@ def test_CALabel_inputs(): argstr='-r %s', extensions=None, ), - label=dict(argstr='-l %s', ), + label=dict( + argstr='-l %s', + extensions=None, + ), no_big_ventricles=dict(argstr='-nobigventricles', ), num_threads=dict(), out_file=dict( diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py b/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py index 1f6546ae3a..5700103e84 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py @@ -59,7 +59,7 @@ def test_CANormalize_inputs(): def test_CANormalize_outputs(): output_map = dict( control_points=dict(extensions=None, ), - out_file=dict(), + out_file=dict(extensions=None, ), ) outputs = CANormalize.output_spec() diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py b/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py index 83f669b218..dcf5aa84a8 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py @@ -53,7 +53,7 @@ def test_CARegister_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_CARegister_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = CARegister.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py b/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py index 68a5a98e66..fe9c1a3121 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py @@ -35,7 +35,7 @@ def test_CheckTalairachAlignment_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_CheckTalairachAlignment_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = CheckTalairachAlignment.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py b/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py index 8acab945c1..ee1d3ae7f3 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py @@ -38,11 +38,13 @@ def test_ConcatenateLTA_inputs(): subjects_dir=dict(), tal_source_file=dict( argstr='-tal %s', + extensions=None, position=-5, requires=['tal_template_file'], ), tal_template_file=dict( argstr='%s', + extensions=None, position=-4, requires=['tal_source_file'], ), diff --git a/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py b/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py index 3c4e5aa484..3cd62e8ee7 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py @@ -5,10 +5,16 @@ def test_Contrast_inputs(): input_map = dict( - annotation=dict(mandatory=True, ), + annotation=dict( + extensions=None, + mandatory=True, + ), args=dict(argstr='%s', ), copy_inputs=dict(), - cortex=dict(mandatory=True, ), + cortex=dict( + extensions=None, + mandatory=True, + ), environ=dict( nohash=True, usedefault=True, diff --git a/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py b/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py index 4ba8442b14..83427dcd20 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py @@ -35,7 +35,10 @@ def test_FixTopology_inputs(): ), mgz=dict(argstr='-mgz', ), seed=dict(argstr='-seed %d', ), - sphere=dict(argstr='-sphere %s', ), + sphere=dict( + argstr='-sphere %s', + extensions=None, + ), subject_id=dict( argstr='%s', mandatory=True, diff --git a/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py b/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py index 10c7af6832..eaf91bc1e8 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py @@ -6,7 +6,10 @@ def test_MRIsCALabel_inputs(): input_map = dict( args=dict(argstr='%s', ), - aseg=dict(argstr='-aseg %s', ), + aseg=dict( + argstr='-aseg %s', + extensions=None, + ), canonsurf=dict( argstr='%s', extensions=None, @@ -33,7 +36,10 @@ def test_MRIsCALabel_inputs(): mandatory=True, position=-4, ), - label=dict(argstr='-l %s', ), + label=dict( + argstr='-l %s', + extensions=None, + ), num_threads=dict(), out_file=dict( argstr='%s', diff --git a/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py b/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py index badb3b4f0c..3d3f5cde11 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py @@ -17,22 +17,30 @@ def test_MS_LDA_inputs(): mandatory=True, position=-1, ), - label_file=dict(argstr='-label %s', ), + label_file=dict( + argstr='-label %s', + extensions=None, + ), lda_labels=dict( argstr='-lda %s', mandatory=True, sep=' ', ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), shift=dict(argstr='-shift %d', ), subjects_dir=dict(), use_weights=dict(argstr='-W', ), vol_synth_file=dict( argstr='-synth %s', + extensions=None, mandatory=True, ), weight_file=dict( argstr='-weight %s', + extensions=None, mandatory=True, ), ) diff --git a/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py b/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py index 2384adbb2b..baf03f026c 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py @@ -43,7 +43,7 @@ def test_Normalize_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_Normalize_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = Normalize.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py b/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py index be19b2bd37..8b4ae45d67 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py @@ -27,11 +27,16 @@ def test_ParcellationStats_inputs(): ), in_annotation=dict( argstr='-a %s', + extensions=None, xor=['in_label'], ), - in_cortex=dict(argstr='-cortex %s', ), + in_cortex=dict( + argstr='-cortex %s', + extensions=None, + ), in_label=dict( argstr='-l %s', + extensions=None, xor=['in_annotatoin', 'out_color'], ), lh_pial=dict( @@ -45,11 +50,13 @@ def test_ParcellationStats_inputs(): mgz=dict(argstr='-mgz', ), out_color=dict( argstr='-c %s', + extensions=None, genfile=True, xor=['in_label'], ), out_table=dict( argstr='-f %s', + extensions=None, genfile=True, requires=['tabular_output'], ), diff --git a/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py b/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py index f3406d41fc..5a609f586e 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py @@ -47,7 +47,7 @@ def test_RegisterAVItoTalairach_outputs(): extensions=None, usedefault=True, ), - out_file=dict(), + out_file=dict(extensions=None, ), ) outputs = RegisterAVItoTalairach.output_spec() diff --git a/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py b/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py index 3cf47c71ce..09da0d001d 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py @@ -100,7 +100,10 @@ def test_SegStatsReconAll_inputs(): extensions=None, mandatory=True, ), - ribbon=dict(mandatory=True, ), + ribbon=dict( + extensions=None, + mandatory=True, + ), segment_id=dict(argstr='--id %s...', ), segmentation_file=dict( argstr='--seg %s', diff --git a/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py b/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py index 26dcbe3458..968041ff67 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py @@ -59,6 +59,7 @@ def test_SurfaceSnapshots_inputs(): overlay_range_offset=dict(argstr='-foffset %.3f', ), overlay_reg=dict( argstr='-overlay-reg %s', + extensions=None, xor=['overlay_reg', 'identity_reg', 'mni152_reg'], ), patch_file=dict( diff --git a/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py b/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py index 9d248c1a7d..2a28765d67 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py @@ -30,9 +30,9 @@ def test_TalairachAVI_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_TalairachAVI_outputs(): output_map = dict( - out_file=dict(), - out_log=dict(), - out_txt=dict(), + out_file=dict(extensions=None, ), + out_log=dict(extensions=None, ), + out_txt=dict(extensions=None, ), ) outputs = TalairachAVI.output_spec() diff --git a/nipype/interfaces/freesurfer/utils.py b/nipype/interfaces/freesurfer/utils.py index 55e38576bb..8cae2cc15b 100644 --- a/nipype/interfaces/freesurfer/utils.py +++ b/nipype/interfaces/freesurfer/utils.py @@ -1031,8 +1031,9 @@ def info_regexp(self, info, field, delim="\n"): else: return None - def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = self._outputs() + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(ImageInfo, self).aggregate_outputs( + runtime, needed_outputs, rebase_cwd) info = runtime.stdout outputs.info = info diff --git a/nipype/interfaces/fsl/model.py b/nipype/interfaces/fsl/model.py index 113f785120..cb9d0c445a 100644 --- a/nipype/interfaces/fsl/model.py +++ b/nipype/interfaces/fsl/model.py @@ -1820,7 +1820,7 @@ class SmoothEstimate(FSLCommand): output_spec = SmoothEstimateOutputSpec _cmd = 'smoothest' - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() stdout = runtime.stdout.split('\n') outputs.dlh = float(stdout[0].split()[1]) diff --git a/nipype/interfaces/fsl/preprocess.py b/nipype/interfaces/fsl/preprocess.py index 66e3c5904d..6383f8fffd 100644 --- a/nipype/interfaces/fsl/preprocess.py +++ b/nipype/interfaces/fsl/preprocess.py @@ -661,9 +661,9 @@ class FLIRT(FSLCommand): output_spec = FLIRTOutputSpec _log_written = False - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = super(FLIRT, self).aggregate_outputs( - runtime=runtime, needed_outputs=needed_outputs) + runtime=runtime, needed_outputs=needed_outputs, rebase_cwd=rebase_cwd) if self.inputs.save_log and not self._log_written: with open(outputs.out_log, "a") as text_file: text_file.write(runtime.stdout + '\n') diff --git a/nipype/interfaces/fsl/tests/test_auto_Cleaner.py b/nipype/interfaces/fsl/tests/test_auto_Cleaner.py index a75df99db5..597dc88d5e 100644 --- a/nipype/interfaces/fsl/tests/test_auto_Cleaner.py +++ b/nipype/interfaces/fsl/tests/test_auto_Cleaner.py @@ -22,14 +22,17 @@ def test_Cleaner_inputs(): ), confound_file=dict( argstr='-x %s', + extensions=None, position=4, ), confound_file_1=dict( argstr='-x %s', + extensions=None, position=5, ), confound_file_2=dict( argstr='-x %s', + extensions=None, position=6, ), environ=dict( diff --git a/nipype/interfaces/fsl/tests/test_auto_Cluster.py b/nipype/interfaces/fsl/tests/test_auto_Cluster.py index 6ed34ab816..19340b9383 100644 --- a/nipype/interfaces/fsl/tests/test_auto_Cluster.py +++ b/nipype/interfaces/fsl/tests/test_auto_Cluster.py @@ -7,7 +7,10 @@ def test_Cluster_inputs(): input_map = dict( args=dict(argstr='%s', ), connectivity=dict(argstr='--connectivity=%d', ), - cope_file=dict(argstr='--cope=%s', ), + cope_file=dict( + argstr='--cope=%s', + extensions=None, + ), dlh=dict(argstr='--dlh=%.10f', ), environ=dict( nohash=True, diff --git a/nipype/interfaces/fsl/utils.py b/nipype/interfaces/fsl/utils.py index f4ef73c0e9..494aba86cc 100644 --- a/nipype/interfaces/fsl/utils.py +++ b/nipype/interfaces/fsl/utils.py @@ -777,8 +777,9 @@ def _format_arg(self, name, trait_spec, value): '-k %s option in op_string requires mask_file') return super(ImageStats, self)._format_arg(name, trait_spec, value) - def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = self._outputs() + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(ImageStats, self).aggregate_outputs( + runtime, needed_outputs, rebase_cwd) # local caching for backward compatibility outfile = os.path.join(os.getcwd(), 'stat_result.json') if runtime is None: diff --git a/nipype/interfaces/minc/tests/test_auto_Average.py b/nipype/interfaces/minc/tests/test_auto_Average.py index 678ab93a52..6f0e84d144 100644 --- a/nipype/interfaces/minc/tests/test_auto_Average.py +++ b/nipype/interfaces/minc/tests/test_auto_Average.py @@ -29,6 +29,7 @@ def test_Average_inputs(): ), filelist=dict( argstr='-filelist %s', + extensions=None, mandatory=True, xor=('input_files', 'filelist'), ), @@ -126,7 +127,10 @@ def test_Average_inputs(): argstr='-quiet', xor=('verbose', 'quiet'), ), - sdfile=dict(argstr='-sdfile %s', ), + sdfile=dict( + argstr='-sdfile %s', + extensions=None, + ), two=dict(argstr='-2', ), verbose=dict( argstr='-verbose', diff --git a/nipype/interfaces/minc/tests/test_auto_BBox.py b/nipype/interfaces/minc/tests/test_auto_BBox.py index c1b3515cea..7bdf35f03d 100644 --- a/nipype/interfaces/minc/tests/test_auto_BBox.py +++ b/nipype/interfaces/minc/tests/test_auto_BBox.py @@ -25,6 +25,7 @@ def test_BBox_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_Beast.py b/nipype/interfaces/minc/tests/test_auto_Beast.py index bc4705db2e..3a44b436b0 100644 --- a/nipype/interfaces/minc/tests/test_auto_Beast.py +++ b/nipype/interfaces/minc/tests/test_auto_Beast.py @@ -18,7 +18,10 @@ def test_Beast_inputs(): argstr='-alpha %s', usedefault=True, ), - configuration_file=dict(argstr='-configuration %s', ), + configuration_file=dict( + argstr='-configuration %s', + extensions=None, + ), environ=dict( nohash=True, usedefault=True, @@ -27,6 +30,7 @@ def test_Beast_inputs(): flip_images=dict(argstr='-flip', ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-2, ), @@ -44,6 +48,7 @@ def test_Beast_inputs(): ), output_file=dict( argstr='%s', + extensions=None, hash_files=False, name_source=['input_file'], name_template='%s_beast_mask.mnc', diff --git a/nipype/interfaces/minc/tests/test_auto_Calc.py b/nipype/interfaces/minc/tests/test_auto_Calc.py index e9df677150..d5fb316e39 100644 --- a/nipype/interfaces/minc/tests/test_auto_Calc.py +++ b/nipype/interfaces/minc/tests/test_auto_Calc.py @@ -26,6 +26,7 @@ def test_Calc_inputs(): eval_width=dict(argstr='-eval_width %s', ), expfile=dict( argstr='-expfile %s', + extensions=None, mandatory=True, xor=('expression', 'expfile'), ), @@ -36,6 +37,7 @@ def test_Calc_inputs(): ), filelist=dict( argstr='-filelist %s', + extensions=None, mandatory=True, xor=('input_files', 'filelist'), ), diff --git a/nipype/interfaces/minc/tests/test_auto_Dump.py b/nipype/interfaces/minc/tests/test_auto_Dump.py index 19c299dac8..fbd33f5a46 100644 --- a/nipype/interfaces/minc/tests/test_auto_Dump.py +++ b/nipype/interfaces/minc/tests/test_auto_Dump.py @@ -36,6 +36,7 @@ def test_Dump_inputs(): netcdf_name=dict(argstr='-n %s', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_Extract.py b/nipype/interfaces/minc/tests/test_auto_Extract.py index 35f6162c7f..fbd0a84729 100644 --- a/nipype/interfaces/minc/tests/test_auto_Extract.py +++ b/nipype/interfaces/minc/tests/test_auto_Extract.py @@ -84,6 +84,7 @@ def test_Extract_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_Math.py b/nipype/interfaces/minc/tests/test_auto_Math.py index 6bc142b15d..41d75379d4 100644 --- a/nipype/interfaces/minc/tests/test_auto_Math.py +++ b/nipype/interfaces/minc/tests/test_auto_Math.py @@ -36,6 +36,7 @@ def test_Math_inputs(): exp=dict(argstr='-exp -const2 %s %s', ), filelist=dict( argstr='-filelist %s', + extensions=None, mandatory=True, xor=('input_files', 'filelist'), ), diff --git a/nipype/interfaces/minc/tests/test_auto_Norm.py b/nipype/interfaces/minc/tests/test_auto_Norm.py index 9fb0d3c5ba..cdd4a3db7c 100644 --- a/nipype/interfaces/minc/tests/test_auto_Norm.py +++ b/nipype/interfaces/minc/tests/test_auto_Norm.py @@ -26,7 +26,10 @@ def test_Norm_inputs(): position=-2, ), lower=dict(argstr='-lower %s', ), - mask=dict(argstr='-mask %s', ), + mask=dict( + argstr='-mask %s', + extensions=None, + ), out_ceil=dict(argstr='-out_ceil %s', ), out_floor=dict(argstr='-out_floor %s', ), output_file=dict( @@ -40,6 +43,7 @@ def test_Norm_inputs(): ), output_threshold_mask=dict( argstr='-threshold_mask %s', + extensions=None, hash_files=False, name_source=['input_file'], name_template='%s_norm_threshold_mask.mnc', diff --git a/nipype/interfaces/minc/tests/test_auto_Resample.py b/nipype/interfaces/minc/tests/test_auto_Resample.py index 8d4b24ff41..da512d9f62 100644 --- a/nipype/interfaces/minc/tests/test_auto_Resample.py +++ b/nipype/interfaces/minc/tests/test_auto_Resample.py @@ -94,7 +94,10 @@ def test_Resample_inputs(): argstr='-keep_real_range', xor=('keep_real_range', 'nokeep_real_range'), ), - like=dict(argstr='-like %s', ), + like=dict( + argstr='-like %s', + extensions=None, + ), nearest_neighbour_interpolation=dict( argstr='-nearest_neighbour', xor=('trilinear_interpolation', 'tricubic_interpolation', @@ -157,7 +160,10 @@ def test_Resample_inputs(): xor=('nelements', 'nelements_x_y_or_z'), ), talairach=dict(argstr='-talairach', ), - transformation=dict(argstr='-transformation %s', ), + transformation=dict( + argstr='-transformation %s', + extensions=None, + ), transverse_slices=dict( argstr='-transverse', xor=('transverse', 'sagittal', 'coronal'), diff --git a/nipype/interfaces/minc/tests/test_auto_Reshape.py b/nipype/interfaces/minc/tests/test_auto_Reshape.py index 669425da95..d80f9a377b 100644 --- a/nipype/interfaces/minc/tests/test_auto_Reshape.py +++ b/nipype/interfaces/minc/tests/test_auto_Reshape.py @@ -16,6 +16,7 @@ def test_Reshape_inputs(): ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-2, ), diff --git a/nipype/interfaces/minc/tests/test_auto_ToRaw.py b/nipype/interfaces/minc/tests/test_auto_ToRaw.py index aeda687c97..42ba72f145 100644 --- a/nipype/interfaces/minc/tests/test_auto_ToRaw.py +++ b/nipype/interfaces/minc/tests/test_auto_ToRaw.py @@ -26,6 +26,7 @@ def test_ToRaw_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_VolSymm.py b/nipype/interfaces/minc/tests/test_auto_VolSymm.py index b710d59543..ae3332b7f2 100644 --- a/nipype/interfaces/minc/tests/test_auto_VolSymm.py +++ b/nipype/interfaces/minc/tests/test_auto_VolSymm.py @@ -22,6 +22,7 @@ def test_VolSymm_inputs(): fit_nonlinear=dict(argstr='-nonlinear', ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-3, ), @@ -38,6 +39,7 @@ def test_VolSymm_inputs(): ), trans_file=dict( argstr='%s', + extensions=None, genfile=True, hash_files=False, keep_extension=False, diff --git a/nipype/interfaces/minc/tests/test_auto_XfmInvert.py b/nipype/interfaces/minc/tests/test_auto_XfmInvert.py index 9e242300da..69c455f875 100644 --- a/nipype/interfaces/minc/tests/test_auto_XfmInvert.py +++ b/nipype/interfaces/minc/tests/test_auto_XfmInvert.py @@ -16,6 +16,7 @@ def test_XfmInvert_inputs(): ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-2, ), diff --git a/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py b/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py index 20995e806e..2a58b39d57 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py @@ -8,6 +8,7 @@ def test_DwiTool_inputs(): args=dict(argstr='%s', ), b0_file=dict( argstr='-b0 %s', + extensions=None, position=4, ), ball_flag=dict( @@ -28,11 +29,13 @@ def test_DwiTool_inputs(): ), bval_file=dict( argstr='-bval %s', + extensions=None, mandatory=True, position=2, ), bvec_file=dict( argstr='-bvec %s', + extensions=None, position=3, ), diso_val=dict(argstr='-diso %f', ), @@ -59,6 +62,7 @@ def test_DwiTool_inputs(): ), famap_file=dict( argstr='-famap %s', + extensions=None, name_source=['source_file'], name_template='%s_famap.nii.gz', ), @@ -72,20 +76,24 @@ def test_DwiTool_inputs(): ), logdti_file=dict( argstr='-logdti2 %s', + extensions=None, name_source=['source_file'], name_template='%s_logdti2.nii.gz', ), mask_file=dict( argstr='-mask %s', + extensions=None, position=5, ), mcmap_file=dict( argstr='-mcmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mcmap.nii.gz', ), mdmap_file=dict( argstr='-mdmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mdmap.nii.gz', ), @@ -115,22 +123,26 @@ def test_DwiTool_inputs(): ), rgbmap_file=dict( argstr='-rgbmap %s', + extensions=None, name_source=['source_file'], name_template='%s_rgbmap.nii.gz', ), source_file=dict( argstr='-source %s', + extensions=None, mandatory=True, position=1, ), syn_file=dict( argstr='-syn %s', + extensions=None, name_source=['source_file'], name_template='%s_syn.nii.gz', requires=['bvec_file', 'b0_file'], ), v1map_file=dict( argstr='-v1map %s', + extensions=None, name_source=['source_file'], name_template='%s_v1map.nii.gz', ), @@ -142,13 +154,13 @@ def test_DwiTool_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_DwiTool_outputs(): output_map = dict( - famap_file=dict(), - logdti_file=dict(), - mcmap_file=dict(), - mdmap_file=dict(), - rgbmap_file=dict(), - syn_file=dict(), - v1map_file=dict(), + famap_file=dict(extensions=None, ), + logdti_file=dict(extensions=None, ), + mcmap_file=dict(extensions=None, ), + mdmap_file=dict(extensions=None, ), + rgbmap_file=dict(extensions=None, ), + syn_file=dict(extensions=None, ), + v1map_file=dict(extensions=None, ), ) outputs = DwiTool.output_spec() diff --git a/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py b/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py index b2e1bef961..94a489a4ad 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py @@ -8,6 +8,7 @@ def test_FitAsl_inputs(): args=dict(argstr='%s', ), cbf_file=dict( argstr='-cbf %s', + extensions=None, name_source=['source_file'], name_template='%s_cbf.nii.gz', ), @@ -20,19 +21,33 @@ def test_FitAsl_inputs(): ), error_file=dict( argstr='-error %s', + extensions=None, name_source=['source_file'], name_template='%s_error.nii.gz', ), gm_plasma=dict(argstr='-gmL %f', ), gm_t1=dict(argstr='-gmT1 %f', ), gm_ttt=dict(argstr='-gmTTT %f', ), - ir_output=dict(argstr='-IRoutput %s', ), - ir_volume=dict(argstr='-IRvolume %s', ), + ir_output=dict( + argstr='-IRoutput %s', + extensions=None, + ), + ir_volume=dict( + argstr='-IRvolume %s', + extensions=None, + ), ldd=dict(argstr='-LDD %f', ), - m0map=dict(argstr='-m0map %s', ), - m0mape=dict(argstr='-m0mape %s', ), + m0map=dict( + argstr='-m0map %s', + extensions=None, + ), + m0mape=dict( + argstr='-m0mape %s', + extensions=None, + ), mask=dict( argstr='-mask %s', + extensions=None, position=2, ), mul=dict(argstr='-mul %f', ), @@ -46,21 +61,29 @@ def test_FitAsl_inputs(): pv2=dict(argstr='-pv2 %d', ), pv3=dict(argstr='-pv3 %d %d %d', ), pv_threshold=dict(argstr='-pvthreshold', ), - seg=dict(argstr='-seg %s', ), + seg=dict( + argstr='-seg %s', + extensions=None, + ), segstyle=dict(argstr='-segstyle', ), sig=dict(argstr='-sig', ), source_file=dict( argstr='-source %s', + extensions=None, mandatory=True, position=1, ), syn_file=dict( argstr='-syn %s', + extensions=None, name_source=['source_file'], name_template='%s_syn.nii.gz', ), t1_art_cmp=dict(argstr='-T1a %f', ), - t1map=dict(argstr='-t1map %s', ), + t1map=dict( + argstr='-t1map %s', + extensions=None, + ), t_inv1=dict(argstr='-Tinv1 %f', ), t_inv2=dict(argstr='-Tinv2 %f', ), wm_plasma=dict(argstr='-wmL %f', ), @@ -74,9 +97,9 @@ def test_FitAsl_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_FitAsl_outputs(): output_map = dict( - cbf_file=dict(), - error_file=dict(), - syn_file=dict(), + cbf_file=dict(extensions=None, ), + error_file=dict(extensions=None, ), + syn_file=dict(extensions=None, ), ) outputs = FitAsl.output_spec() diff --git a/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py b/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py index 700d9a31c4..2086d12a2b 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py @@ -25,15 +25,20 @@ def test_FitDwi_inputs(): ), bval_file=dict( argstr='-bval %s', + extensions=None, mandatory=True, position=2, ), bvec_file=dict( argstr='-bvec %s', + extensions=None, mandatory=True, position=3, ), - cov_file=dict(argstr='-cov %s', ), + cov_file=dict( + argstr='-cov %s', + extensions=None, + ), csf_t2_val=dict(argstr='-csfT2 %f', ), diso_val=dict(argstr='-diso %f', ), dpr_val=dict(argstr='-dpr %f', ), @@ -51,11 +56,13 @@ def test_FitDwi_inputs(): ), error_file=dict( argstr='-error %s', + extensions=None, name_source=['source_file'], name_template='%s_error.nii.gz', ), famap_file=dict( argstr='-famap %s', + extensions=None, name_source=['source_file'], name_template='%s_famap.nii.gz', ), @@ -75,13 +82,17 @@ def test_FitDwi_inputs(): argstr='-lm %f %f', requires=['gn_flag'], ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), maxit_val=dict( argstr='-maxit %d', requires=['gn_flag'], ), mcmap_file=dict( argstr='-mcmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mcmap.nii.gz', requires=['nodv_flag'], @@ -89,12 +100,14 @@ def test_FitDwi_inputs(): mcmaxit=dict(argstr='-mcmaxit %d', ), mcout=dict( argstr='-mcout %s', + extensions=None, name_source=['source_file'], name_template='%s_mcout.txt', ), mcsamples=dict(argstr='-mcsamples %d', ), mdmap_file=dict( argstr='-mdmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mdmap.nii.gz', ), @@ -116,6 +129,7 @@ def test_FitDwi_inputs(): ), nodiff_file=dict( argstr='-nodiff %s', + extensions=None, name_source=['source_file'], name_template='%s_no_diff.nii.gz', ), @@ -128,14 +142,19 @@ def test_FitDwi_inputs(): ], ), perf_thr=dict(argstr='-perfthreshold %f', ), - prior_file=dict(argstr='-prior %s', ), + prior_file=dict( + argstr='-prior %s', + extensions=None, + ), res_file=dict( argstr='-res %s', + extensions=None, name_source=['source_file'], name_template='%s_resmap.nii.gz', ), rgbmap_file=dict( argstr='-rgbmap %s', + extensions=None, name_source=['source_file'], name_template='%s_rgbmap.nii.gz', requires=['dti_flag'], @@ -144,38 +163,45 @@ def test_FitDwi_inputs(): slice_no=dict(argstr='-slice %d', ), source_file=dict( argstr='-source %s', + extensions=None, mandatory=True, position=1, ), swls_val=dict(argstr='-swls %f', ), syn_file=dict( argstr='-syn %s', + extensions=None, name_source=['source_file'], name_template='%s_syn.nii.gz', ), te_file=dict( argstr='-TE %s', + extensions=None, xor=['te_file'], ), te_value=dict( argstr='-TE %s', + extensions=None, xor=['te_file'], ), ten_type=dict(usedefault=True, ), tenmap2_file=dict( argstr='-tenmap2 %s', + extensions=None, name_source=['source_file'], name_template='%s_tenmap2.nii.gz', requires=['dti_flag'], ), tenmap_file=dict( argstr='-tenmap %s', + extensions=None, name_source=['source_file'], name_template='%s_tenmap.nii.gz', requires=['dti_flag'], ), v1map_file=dict( argstr='-v1map %s', + extensions=None, name_source=['source_file'], name_template='%s_v1map.nii.gz', ), @@ -194,18 +220,18 @@ def test_FitDwi_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_FitDwi_outputs(): output_map = dict( - error_file=dict(), - famap_file=dict(), - mcmap_file=dict(), - mcout=dict(), - mdmap_file=dict(), - nodiff_file=dict(), - res_file=dict(), - rgbmap_file=dict(), - syn_file=dict(), - tenmap2_file=dict(), - tenmap_file=dict(), - v1map_file=dict(), + error_file=dict(extensions=None, ), + famap_file=dict(extensions=None, ), + mcmap_file=dict(extensions=None, ), + mcout=dict(extensions=None, ), + mdmap_file=dict(extensions=None, ), + nodiff_file=dict(extensions=None, ), + res_file=dict(extensions=None, ), + rgbmap_file=dict(extensions=None, ), + syn_file=dict(extensions=None, ), + tenmap2_file=dict(extensions=None, ), + tenmap_file=dict(extensions=None, ), + v1map_file=dict(extensions=None, ), ) outputs = FitDwi.output_spec() diff --git a/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py b/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py index 392654fd5c..b184e448cd 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py @@ -7,7 +7,10 @@ def test_FitQt1_inputs(): input_map = dict( acceptance=dict(argstr='-acceptance %f', ), args=dict(argstr='%s', ), - b1map=dict(argstr='-b1map %s', ), + b1map=dict( + argstr='-b1map %s', + extensions=None, + ), comp_file=dict( argstr='-comp %s', extensions=None, @@ -28,7 +31,10 @@ def test_FitQt1_inputs(): argstr='-flips %s', sep=' ', ), - flips_list=dict(argstr='-fliplist %s', ), + flips_list=dict( + argstr='-fliplist %s', + extensions=None, + ), gn_flag=dict( argstr='-gn', position=8, @@ -63,7 +69,10 @@ def test_FitQt1_inputs(): name_template='%s_mcmap.nii.gz', ), mcmaxit=dict(argstr='-mcmaxit %d', ), - mcout=dict(argstr='-mcout %s', ), + mcout=dict( + argstr='-mcout %s', + extensions=None, + ), mcsamples=dict(argstr='-mcsamples %d', ), nb_comp=dict( argstr='-nc %d', @@ -101,7 +110,10 @@ def test_FitQt1_inputs(): name_source=['source_file'], name_template='%s_syn.nii.gz', ), - t1_list=dict(argstr='-T1list %s', ), + t1_list=dict( + argstr='-T1list %s', + extensions=None, + ), t1map_file=dict( argstr='-t1map %s', extensions=None, @@ -119,7 +131,10 @@ def test_FitQt1_inputs(): position=14, sep=' ', ), - tis_list=dict(argstr='-TIlist %s', ), + tis_list=dict( + argstr='-TIlist %s', + extensions=None, + ), tr_value=dict( argstr='-TR %f', position=5, diff --git a/nipype/interfaces/niftyseg/label_fusion.py b/nipype/interfaces/niftyseg/label_fusion.py index 1b0237d37c..9d3342657b 100644 --- a/nipype/interfaces/niftyseg/label_fusion.py +++ b/nipype/interfaces/niftyseg/label_fusion.py @@ -11,6 +11,7 @@ from ..base import (TraitedSpec, File, traits, isdefined, CommandLineInputSpec, NipypeInterfaceError) +from ..base.traits_extension import rebase_path_traits from .base import NiftySegCommand from ..niftyreg.base import get_custom_path from ...utils.filemanip import load_json, save_json, split_filename @@ -314,7 +315,7 @@ class CalcTopNCC(NiftySegCommand): input_spec = CalcTopNCCInputSpec output_spec = CalcTopNCCOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() # local caching for backward compatibility outfile = os.path.join(os.getcwd(), 'CalcTopNCC.json') @@ -335,5 +336,9 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None): if len(out_files) == 1: out_files = out_files[0] save_json(outfile, dict(files=out_files)) + + if rebase_cwd: + out_files = rebase_path_traits(outputs.trait('out_files'), + out_files, rebase_cwd) outputs.out_files = out_files return outputs diff --git a/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py b/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py index 3fe17160db..9a9b5c421c 100644 --- a/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py +++ b/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py @@ -13,7 +13,7 @@ def test_EstimateContrast_inputs(): constants=dict(mandatory=True, ), contrasts=dict(mandatory=True, ), dof=dict(mandatory=True, ), - mask=dict(), + mask=dict(extensions=None, ), nvbeta=dict(mandatory=True, ), reg_names=dict(mandatory=True, ), s2=dict( diff --git a/nipype/interfaces/nipy/tests/test_auto_FitGLM.py b/nipype/interfaces/nipy/tests/test_auto_FitGLM.py index a700c18d43..71ca221efc 100644 --- a/nipype/interfaces/nipy/tests/test_auto_FitGLM.py +++ b/nipype/interfaces/nipy/tests/test_auto_FitGLM.py @@ -8,7 +8,7 @@ def test_FitGLM_inputs(): TR=dict(mandatory=True, ), drift_model=dict(usedefault=True, ), hrf_model=dict(usedefault=True, ), - mask=dict(), + mask=dict(extensions=None, ), method=dict(usedefault=True, ), model=dict(usedefault=True, ), normalize_design_matrix=dict(usedefault=True, ), @@ -30,7 +30,7 @@ def test_FitGLM_outputs(): dof=dict(), nvbeta=dict(), reg_names=dict(), - residuals=dict(), + residuals=dict(extensions=None, ), s2=dict(extensions=None, ), ) outputs = FitGLM.output_spec() diff --git a/nipype/interfaces/spm/model.py b/nipype/interfaces/spm/model.py index 4fb1c51e08..204c016aa1 100644 --- a/nipype/interfaces/spm/model.py +++ b/nipype/interfaces/spm/model.py @@ -717,11 +717,10 @@ def _make_matlab_command(self, _): return script - def aggregate_outputs(self, runtime=None): - outputs = self._outputs() - setattr(outputs, 'thresholded_map', - self._gen_thresholded_map_filename()) - setattr(outputs, 'pre_topo_fdr_map', self._gen_pre_topo_map_filename()) + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(Threshold, self).aggregate_outputs( + runtime, needed_outputs, rebase_cwd) + for line in runtime.stdout.split('\n'): if line.startswith("activation_forced = "): setattr(outputs, 'activation_forced', @@ -842,7 +841,7 @@ def _make_matlab_command(self, _): """ return script - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() cur_output = "" for line in runtime.stdout.split('\n'): diff --git a/nipype/interfaces/utility/tests/test_auto_Rename.py b/nipype/interfaces/utility/tests/test_auto_Rename.py index 6b56badd6f..70ced0ecdd 100644 --- a/nipype/interfaces/utility/tests/test_auto_Rename.py +++ b/nipype/interfaces/utility/tests/test_auto_Rename.py @@ -20,7 +20,7 @@ def test_Rename_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_Rename_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = Rename.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 006558b296..bc81dd7890 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -194,7 +194,7 @@ def interface(self): @property def result(self): """Get result from result file (do not hold it in memory)""" - return _load_resultfile(self.output_dir(), self.name)[0] + return self._load_results() @property def inputs(self): @@ -483,6 +483,7 @@ def run(self, updatehash=False): # Tear-up after success shutil.move(hashfile_unfinished, hashfile_unfinished.replace('_unfinished', '')) + write_report( self, report_type='postexec', is_mapnode=isinstance(self, MapNode)) logger.info('[Node] Finished "%s".', self.fullname) @@ -564,14 +565,16 @@ def _run_interface(self, execute=True, updatehash=False): def _load_results(self): cwd = self.output_dir() - result, aggregate, attribute_error = _load_resultfile(cwd, self.name) + is_mapnode = isinstance(self, MapNode) + result, attribute_error = _load_resultfile( + cwd, self.name, resolve=not is_mapnode) # try aggregating first - if aggregate: + if result is None: logger.debug('aggregating results') if attribute_error: old_inputs = loadpkl(op.join(cwd, '_inputs.pklz')) self.inputs.trait_set(**old_inputs) - if not isinstance(self, MapNode): + if not is_mapnode: self._copyfiles_to_wd(linksonly=True) aggouts = self._interface.aggregate_outputs( needed_outputs=self.needed_outputs) @@ -630,6 +633,7 @@ def _run_command(self, execute, copyfiles=True): with indirectory(outdir): cmd = self._interface.cmdline except Exception as msg: + result.error = True result.runtime.stderr = '{}\n\n{}'.format( getattr(result.runtime, 'stderr', ''), msg) _save_resultfile(result, outdir, self.name) @@ -640,8 +644,9 @@ def _run_command(self, execute, copyfiles=True): message += ', a CommandLine Interface with command:\n{}'.format(cmd) logger.info(message) try: - result = self._interface.run(cwd=outdir) + result = self._interface.run(cwd=outdir, rebase_cwd=True) except Exception as msg: + result.error = True result.runtime.stderr = '%s\n\n%s'.format( getattr(result.runtime, 'stderr', ''), msg) _save_resultfile(result, outdir, self.name) @@ -651,15 +656,18 @@ def _run_command(self, execute, copyfiles=True): if isinstance(self, MapNode): dirs2keep = [op.join(outdir, 'mapflow')] - result.outputs = clean_working_directory( + unset_outputs = clean_working_directory( result.outputs, outdir, self._interface.inputs, self.needed_outputs, self.config, dirs2keep=dirs2keep) - _save_resultfile(result, outdir, self.name) + for name in unset_outputs: + setattr(result.outputs, name, Undefined) + + _save_resultfile(result, outdir, self.name) return result def _copyfiles_to_wd(self, execute=True, linksonly=False): @@ -1131,9 +1139,12 @@ def _collate_results(self, nodes): inputs=[], outputs=self.outputs) returncode = [] + error_msgs = [] for i, nresult, err in nodes: finalresult.runtime.insert(i, None) returncode.insert(i, err) + if err is not None: + error_msgs += ['Subnode %d failed with error:\n%s' % (i, err)] if nresult: if hasattr(nresult, 'runtime'): @@ -1171,14 +1182,10 @@ def _collate_results(self, nodes): self.iterfield[0]))) setattr(finalresult.outputs, key, values) - if returncode and any([code is not None for code in returncode]): - msg = [] - for i, code in enumerate(returncode): - if code is not None: - msg += ['Subnode %d failed' % i] - msg += ['Error: %s' % str(code)] - raise Exception('Subnodes of node: %s failed:\n%s' % - (self.name, '\n'.join(msg))) + if error_msgs: + error_msgs.insert(0, 'Subnodes of node: %s failed:' % self.name) + logger.critical('\n\t\t'.join(error_msgs)) + raise Exception('\n\t\t'.join(error_msgs)) return finalresult @@ -1229,7 +1236,8 @@ def _check_iterfield(self): "have the same length. %s") % str(self.inputs)) def _run_interface(self, execute=True, updatehash=False): - """Run the mapnode interface + """ + Run the mapnode interface. This is primarily intended for serial execution of mapnode. A parallel execution requires creation of new nodes that can be spawned @@ -1258,7 +1266,7 @@ def _run_interface(self, execute=True, updatehash=False): stop_first=str2bool( self.config['execution']['stop_on_first_crash']))) # And store results - _save_resultfile(result, cwd, self.name) + _save_resultfile(result, cwd, self.name, rebase=False) # remove any node directories no longer required dirs2remove = [] for path in glob(op.join(cwd, 'mapflow', '*')): diff --git a/nipype/pipeline/engine/tests/test_nodes.py b/nipype/pipeline/engine/tests/test_nodes.py index ea03fe69ae..15b464f675 100644 --- a/nipype/pipeline/engine/tests/test_nodes.py +++ b/nipype/pipeline/engine/tests/test_nodes.py @@ -153,7 +153,7 @@ def func1(in1): Function(input_names=['in1'], output_names=['out'], function=func1), iterfield=['in1'], nested=False, - name='n1') + name='n2') n2.inputs.in1 = [[1, [2]], 3, [4, 5]] with pytest.raises(Exception) as excinfo: @@ -295,13 +295,17 @@ def test_inputs_removal(tmpdir): def test_outputmultipath_collapse(tmpdir): """Test an OutputMultiPath whose initial value is ``[[x]]`` to ensure that it is returned as ``[x]``, regardless of how accessed.""" - select_if = niu.Select(inlist=[[1, 2, 3], [4]], index=1) - select_nd = pe.Node(niu.Select(inlist=[[1, 2, 3], [4]], index=1), + select_if = niu.Select(inlist=[[1, 2, 3], [4], [5]], index=[0, 1]) + select_nd = pe.Node(niu.Select(inlist=[[1, 2, 3], [4], [5]], index=[0, 1]), name='select_nd') ifres = select_if.run() + ifbase = select_if.run(rebase_cwd=True) # If flattening happens could be the rebase ndres = select_nd.run() - assert ifres.outputs.out == [4] - assert ndres.outputs.out == [4] - assert select_nd.result.outputs.out == [4] + assert ifres.outputs.out == [[1, 2, 3], [4]] + assert ifbase.outputs.out == [[1, 2, 3], [4]] + assert ndres.outputs.out == [[1, 2, 3], [4]] + assert select_nd.result.outputs.out == [[1, 2, 3], [4]] + + # Inconsistent behavior of OutputMultiObject, see #2968 diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 4f4383f169..7a6a53b300 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -107,7 +107,7 @@ class InputSpec(nib.TraitedSpec): out = clean_working_directory(outputs, tmpdir.strpath, inputs, needed_outputs, deepcopy(config._sections)) assert os.path.exists(outfiles[5]) - assert out.others == outfiles[5] + assert 'others' not in out config.set('execution', 'remove_unnecessary_outputs', True) out = clean_working_directory(outputs, tmpdir.strpath, inputs, needed_outputs, deepcopy(config._sections)) @@ -115,8 +115,8 @@ class InputSpec(nib.TraitedSpec): assert os.path.exists(outfiles[3]) assert os.path.exists(outfiles[4]) assert not os.path.exists(outfiles[5]) - assert out.others == nib.Undefined - assert len(out.files) == 2 + assert 'others' in out + assert 'files' not in out config.set_default_config() @@ -222,5 +222,67 @@ def test_mapnode_crash3(tmpdir): wf.base_dir = tmpdir.strpath # changing crashdump dir to current working directory (to avoid problems with read-only systems) wf.config["execution"]["crashdump_dir"] = os.getcwd() - with pytest.raises(RuntimeError): + with pytest.raises(Exception): wf.run(plugin='Linear') + + +class StrPathConfuserInputSpec(nib.TraitedSpec): + in_str = nib.traits.String() + + +class StrPathConfuserOutputSpec(nib.TraitedSpec): + out_tuple = nib.traits.Tuple(nib.File, nib.traits.String) + out_dict_path = nib.traits.Dict(nib.traits.String, nib.File(exists=True)) + out_dict_str = nib.traits.DictStrStr() + out_list = nib.traits.List(nib.traits.String) + out_str = nib.traits.String() + out_path = nib.File(exists=True) + + +class StrPathConfuser(nib.SimpleInterface): + input_spec = StrPathConfuserInputSpec + output_spec = StrPathConfuserOutputSpec + + def _run_interface(self, runtime): + out_path = os.path.abspath(os.path.basename(self.inputs.in_str) + '_path') + open(out_path, 'w').close() + self._results['out_str'] = self.inputs.in_str + self._results['out_path'] = out_path + self._results['out_tuple'] = (out_path, self.inputs.in_str) + self._results['out_dict_path'] = {self.inputs.in_str: out_path} + self._results['out_dict_str'] = {self.inputs.in_str: self.inputs.in_str} + self._results['out_list'] = [self.inputs.in_str] * 2 + return runtime + + +def test_modify_paths_bug(tmpdir): + """ + There was a bug in which, if the current working directory contained a file with the name + of an output String, the string would get transformed into a path, and generally wreak havoc. + + This attempts to replicate that condition, using an object with strings and paths in various + trait configurations, to ensure that the guards added resolve the issue. + + Please see https://github.com/nipy/nipype/issues/2944 for more details. + """ + tmpdir.chdir() + + spc = pe.Node(StrPathConfuser(in_str='2'), name='spc') + + open('2', 'w').close() + + outputs = spc.run().outputs + + # Basic check that string was not manipulated + out_str = outputs.out_str + assert out_str == '2' + + # Check path exists and is absolute + out_path = outputs.out_path + assert os.path.isabs(out_path) + + # Assert data structures pass through correctly + assert outputs.out_tuple == (out_path, out_str) + assert outputs.out_dict_path == {out_str: out_path} + assert outputs.out_dict_str == {out_str: out_str} + assert outputs.out_list == [out_str] * 2 diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 3d961126d5..2245c2638f 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -25,6 +25,7 @@ from ... import logging, config, LooseVersion from ...utils.filemanip import ( + indirectory, relpath, makedirs, fname_presuffix, @@ -40,6 +41,7 @@ ) from ...utils.misc import str2bool from ...utils.functions import create_function_from_source +from ...interfaces.base.traits_extension import rebase_path_traits, resolve_path_traits from ...interfaces.base import (Bunch, CommandLine, isdefined, Undefined, InterfaceResult, traits) from ...interfaces.utility import IdentityInterface @@ -100,7 +102,6 @@ def nodelist_runner(nodes, updatehash=False, stop_first=False): except Exception: if stop_first: raise - result = node.result err = [] if result.runtime and hasattr(result.runtime, 'traceback'): @@ -146,16 +147,24 @@ def write_report(node, report_type=None, is_mapnode=False): write_rst_dict(node.inputs.trait_get()), ] - result = node.result # Locally cache result - outputs = result.outputs - - if outputs is None: + # Preexec reports should stop here + if report_type == 'preexec': with open(report_file, 'at') as fp: fp.write('\n'.join(lines)) return lines.append(write_rst_header('Execution Outputs', level=1)) + result = node.result # Locally cache result + + # Premature end of report + if result is None: + logger.critical('[Node] Recovered InterfaceResult is None.') + lines += ['', 'CRITICAL - InterfaceResult is None.'] + with open(report_file, 'at') as fp: + fp.write('\n'.join(lines)) + return + outputs = result.outputs if isinstance(outputs, Bunch): lines.append(write_rst_dict(outputs.dictcopy())) elif outputs: @@ -180,7 +189,7 @@ def write_report(node, report_type=None, is_mapnode=False): # Init rst dictionary of runtime stats rst_dict = { 'hostname': result.runtime.hostname, - 'duration': result.runtime.duration, + 'duration': getattr(result.runtime, 'duration', ''), 'working_dir': result.runtime.cwd, 'prev_wd': getattr(result.runtime, 'prevcwd', ''), } @@ -284,87 +293,96 @@ def _protect_collapses(hastraits): return _uncollapse(hastraits.trait_get(), collapsed) -def save_resultfile(result, cwd, name): - """Save a result pklz file to ``cwd``""" +def save_resultfile(result, cwd, name, rebase=True): + """Save a result pklz file to ``cwd``.""" + cwd = os.path.abspath(cwd) resultsfile = os.path.join(cwd, 'result_%s.pklz' % name) - if result.outputs: - try: - collapsed = _identify_collapses(result.outputs) - outputs = _uncollapse(result.outputs.trait_get(), collapsed) - # Double-protect tosave so that the original, uncollapsed trait - # is saved in the pickle file. Thus, when the loading process - # collapses, the original correct value is loaded. - tosave = _uncollapse(outputs.copy(), collapsed) - except AttributeError: - tosave = outputs = result.outputs.dictcopy() # outputs was a bunch - for k, v in list(modify_paths(tosave, relative=True, basedir=cwd).items()): - setattr(result.outputs, k, v) - - savepkl(resultsfile, result) - logger.debug('saved results in %s', resultsfile) - - if result.outputs: - for k, v in list(outputs.items()): - setattr(result.outputs, k, v) - - -def load_resultfile(path, name): + logger.debug("Saving results file: '%s'", resultsfile) + + if result.outputs is None: + logger.warn('Storing result file without outputs') + savepkl(resultsfile, result) + return + + try: + outputs = result.outputs.trait_get() + except AttributeError: + logger.debug('Storing non-traited results, skipping rebase of paths') + savepkl(resultsfile, result) + return + + try: + with indirectory(cwd): + # All the magic to fix #2944 resides here: + for key, val in list(outputs.items()): + val = rebase_path_traits(result.outputs.trait(key), val, cwd) + setattr(result.outputs, key, val) + savepkl(resultsfile, result) + finally: + # Reset resolved paths from the outputs dict no matter what + for key, val in list(outputs.items()): + setattr(result.outputs, key, val) + + +def load_resultfile(path, name, resolve=True): """ - Load InterfaceResult file from path + Load InterfaceResult file from path. Parameter --------- - path : base_dir of node name : name of node Returns ------- - result : InterfaceResult structure - aggregate : boolean indicating whether node should aggregate_outputs attribute error : boolean indicating whether there was some mismatch in versions of traits used to store result and hence node needs to rerun + """ - aggregate = True - resultsoutputfile = os.path.join(path, 'result_%s.pklz' % name) result = None attribute_error = False - if os.path.exists(resultsoutputfile): - pkl_file = gzip.open(resultsoutputfile, 'rb') + + if not os.path.exists(os.path.join(path, 'result_%s.pklz' % name)): + logger.warning("Error loading results file, no such file: '%s'", + os.path.join(path, 'result_%s.pklz' % name)) + return result, False + + with indirectory(path): + pkl_file = gzip.open('result_%s.pklz' % name, 'rb') try: result = pickle.load(pkl_file) except UnicodeDecodeError: # Was this pickle created with Python 2.x? - pickle.load(pkl_file, fix_imports=True, encoding='utf-8') - logger.warning('Successfully loaded pkl in compatibility mode') - except (traits.TraitError, AttributeError, ImportError, - EOFError) as err: - if isinstance(err, (AttributeError, ImportError)): - attribute_error = True - logger.debug('attribute error: %s probably using ' - 'different trait pickled file', str(err)) - else: - logger.debug( - 'some file does not exist. hence trait cannot be set') - else: - if result.outputs: - try: - outputs = _protect_collapses(result.outputs) - except AttributeError: - outputs = result.outputs.dictcopy() # outputs == Bunch - try: - for k, v in list(modify_paths(outputs, relative=False, - basedir=path).items()): - setattr(result.outputs, k, v) - except FileNotFoundError: - logger.debug('conversion to full path results in ' - 'non existent file') - aggregate = False - pkl_file.close() - logger.debug('Aggregate: %s', aggregate) - return result, aggregate, attribute_error + result = pickle.load(pkl_file, fix_imports=True, encoding='utf-8') + logger.warning('Successfully loaded pkl in compatibility mode.') + except traits.TraitError as err: + logger.warning('Error restoring outputs while loading results.\n%s.', + err) + except (AttributeError, ImportError, EOFError) as err: + attribute_error = True + logger.warning( + 'Error loading results file (probably using different trait pickled file).\n%s', + err) + finally: + pkl_file.close() + + if result is None: + return result, attribute_error + + if result.error: + return result, attribute_error + + if result is not None and resolve: + outputs = result.outputs + for name in list(result.outputs.get().keys()): + old_value = getattr(outputs, name) + value = resolve_path_traits(outputs.trait(name), old_value, path) + setattr(outputs, name, value) + result.outputs = outputs + + return result, attribute_error def strip_temp(files, wd): @@ -1518,10 +1536,9 @@ def clean_working_directory(outputs, logger.debug('Removing files: %s', ';'.join(files2remove)) for f in files2remove: os.remove(f) - for key in outputs.copyable_trait_names(): - if key not in outputs_to_keep: - setattr(outputs, key, Undefined) - return outputs + + unset_outputs = set(outputs.copyable_trait_names()) - set(outputs_to_keep) + return unset_outputs def merge_dict(d1, d2, merge=lambda x, y: y): diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index e19563ed54..4e649ca805 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -43,10 +43,11 @@ class FileNotFoundError(OSError): # noqa """Defines the exception for Python 2.""" - def __init__(self, path): + def __init__(self, path, errno=2, message=None): """Initialize the exception.""" - super(FileNotFoundError, self).__init__( - 2, 'No such file or directory', '%s' % path) + message = message or 'No such file or directory' + + super(FileNotFoundError, self).__init__(errno, message, '%s' % path) USING_PATHLIB2 = False diff --git a/tools/checkspecs.py b/tools/checkspecs.py index e282728c8e..da5dad1854 100644 --- a/tools/checkspecs.py +++ b/tools/checkspecs.py @@ -198,7 +198,7 @@ def test_specs(self, uri): ] in_built = [ 'type', 'copy', 'parent', 'instance_handler', 'comparison_mode', - 'array', 'default', 'editor' + 'array', 'default', 'editor', 'alternatives' ] bad_specs = [] for c in classes: