From 60a878cc02b701acfeea498f4d3ff072c47eb8fa Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Tue, 14 Feb 2017 13:43:57 +0000 Subject: [PATCH 1/9] LoadVesuvio will not assume it uses RAW files It also bases the instrument prefix on the run number being loaded. Refs #18843 --- .../plugins/algorithms/LoadVesuvio.py | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py index 8e087d58bbf9..c0419eac3230 100644 --- a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py +++ b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py @@ -37,7 +37,6 @@ class LoadVesuvio(LoadEmptyVesuvio): _ws_index = None _spectrum_no = None foil_map = None - _inst_prefix = None _back_scattering = None _load_common_called = False _load_monitors = False @@ -324,12 +323,9 @@ def _exec_single_foil_state_mode(self): self._setup_raw(all_spectra) else: isis = config.getFacility("ISIS") - inst_prefix = isis.instrument("VESUVIO").shortName() - - try: - run_str = inst_prefix + runs[0] +'.raw' - except ValueError: - run_str = runs[0] + vesuvio = isis.instrument("VESUVIO") + run_no = runs[0] + run_str = vesuvio.filePrefix(int(run_no)) + run_no self._raise_error_period_scatter(run_str, self._back_scattering) all_spectra = [item for sublist in self._spectra for item in sublist] @@ -423,13 +419,13 @@ def _load_single_run_spec_and_mon(self, all_spectra, run_str): logger.warning("LoadMonitors is true while monitor spectra are defined in the spectra list.") logger.warning("Monitors have been loaded into the data workspace not separately.") if mons_in_ws: - ms.LoadRaw(Filename=run_str, OutputWorkspace=SUMMED_WS, SpectrumList=all_spectra, - EnableLogging=_LOGGING_) + ms.Load(Filename=run_str, OutputWorkspace=SUMMED_WS, SpectrumList=all_spectra, + EnableLogging=_LOGGING_) else: all_spec_inc_mon = self._mon_spectra all_spec_inc_mon.extend(all_spectra) - ms.LoadRaw(Filename=run_str, OutputWorkspace=SUMMED_WS, SpectrumList=all_spec_inc_mon, - LoadMonitors='Separate', EnableLogging=_LOGGING_) + ms.Load(Filename=run_str, OutputWorkspace=SUMMED_WS, SpectrumList=all_spec_inc_mon, + LoadMonitors='Separate', EnableLogging=_LOGGING_) if self._load_monitors: monitor_group = mtd[SUMMED_WS +'_monitors'] mon_out_name = self.getPropertyValue(WKSP_PROP) + "_monitors" @@ -441,6 +437,7 @@ def _load_single_run_spec_and_mon(self, all_spectra, run_str): self._load_monitors_workspace = self._sum_monitors_in_group(monitor_group, self._load_monitors_workspace) self._raw_monitors = mtd[SUMMED_WS +'_monitors'] + #---------------------------------------------------------------------------------------- def _load_common_inst_parameters(self): @@ -461,7 +458,6 @@ def to_int_list(str_param): return list(range(int(elements[0]),int(elements[1]) + 1)) # range goes x_l,x_h-1 # Attach parameters as attributes - self._inst_prefix = isis.instrument("VESUVIO").shortName() parnames = empty_vesuvio.getParameterNames(False) for name in parnames: # Irritating parameter access doesn't let you query the type @@ -585,28 +581,26 @@ def _load_and_sum_runs(self, spectra): @returns a tuple of length 2 containing (main_detector_ws, monitor_ws) """ isis = config.getFacility("ISIS") - inst_prefix = isis.instrument("VESUVIO").shortName() - + vesuvio = isis.instrument("VESUVIO") runs = self._get_runs() self.summed_ws, self.summed_mon = "__loadraw_evs", "__loadraw_evs_monitors" spec_inc_mon = self._mon_spectra spec_inc_mon.extend(spectra) for index, run in enumerate(runs): - run = inst_prefix + str(run) - self._raise_error_period_scatter(run, self._back_scattering) + filename = vesuvio.filePrefix(int(run)) + str(run) + self._raise_error_period_scatter(filename, self._back_scattering) if index == 0: out_name, out_mon = SUMMED_WS, SUMMED_WS + '_monitors' else: out_name, out_mon = SUMMED_WS + 'tmp', SUMMED_WS + 'tmp_monitors' # Load data - raw_filepath = FileFinder.findRuns(run)[0] - ms.LoadRaw(Filename=raw_filepath, - SpectrumList=spec_inc_mon, - OutputWorkspace=out_name, - LoadMonitors='Separate', - EnableLogging=_LOGGING_) + ms.Load(Filename=filename, + SpectrumList=spec_inc_mon, + OutputWorkspace=out_name, + LoadMonitors='Separate', + EnableLogging=_LOGGING_) # Sum if index > 0: @@ -685,7 +679,7 @@ def _get_runs(self): runs = list(range(int(lower), int(upper)+1)) elif "," in run_str: - runs = run_str.split(",") + runs = run_str.split(",") else: runs = [run_str] From 540f5613aea5a45d8a349a628a540344ae0b996d Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Tue, 14 Feb 2017 13:44:29 +0000 Subject: [PATCH 2/9] Disable logging of delete workspace coperations Refs #18843 --- Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py index c0419eac3230..4bc8b68f76aa 100644 --- a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py +++ b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py @@ -402,7 +402,7 @@ def _exec_single_foil_state_mode(self): if self._sumspectra: self._sum_all_spectra() - ms.DeleteWorkspace(Workspace=SUMMED_WS) + ms.DeleteWorkspace(Workspace=SUMMED_WS, EnableLogging=_LOGGING_) self._store_results() self._cleanup_raw() From a394e3bbdb95884c74a9b33555ef9eee8d5309aa Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Thu, 16 Feb 2017 08:26:43 +0000 Subject: [PATCH 3/9] Remove unused variable Refs #18843 --- Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py index 4bc8b68f76aa..021cc4d09eed 100644 --- a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py +++ b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py @@ -448,7 +448,6 @@ def _load_common_inst_parameters(self): if self._load_common_called: return - isis = config.getFacility("ISIS") empty_vesuvio_ws = self._load_empty_evs() empty_vesuvio = empty_vesuvio_ws.getInstrument() From cdb86e79bd0e96e105e1349f5db1a9ba012fb841 Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Thu, 16 Feb 2017 08:27:02 +0000 Subject: [PATCH 4/9] Add release note to indirect section Refs #18843 --- docs/source/release/v3.10.0/indirect_inelastic.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/release/v3.10.0/indirect_inelastic.rst b/docs/source/release/v3.10.0/indirect_inelastic.rst index e45f25b1b5f7..f0d331f40f18 100644 --- a/docs/source/release/v3.10.0/indirect_inelastic.rst +++ b/docs/source/release/v3.10.0/indirect_inelastic.rst @@ -12,6 +12,7 @@ Algorithms ########## - A new input property *RebinCanToSample* was added to :ref:`ApplyPaalmanPingsCorrection ` which enables or disables the rebinning of the empty container workspace. +- :ref:`LoadVesuvio can now load NeXus files as well as raw files Data Analysis ############# From ce56023623aa96f2708533d95696b02d6c14e45a Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Mon, 20 Feb 2017 14:12:39 +0000 Subject: [PATCH 5/9] Fix LoadVesuvio run finder when extension is provided. Refs #18843 --- .../plugins/algorithms/LoadVesuvio.py | 39 ++++++++++++------- .../tests/analysis/LoadVesuvioTest.py | 21 +++++++++- scripts/Inelastic/IndirectReductionCommon.py | 4 +- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py index 021cc4d09eed..ef6f09d5f241 100644 --- a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py +++ b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py @@ -7,6 +7,8 @@ import copy import numpy as np +import re +import six RUN_PROP = "Filename" WKSP_PROP = "OutputWorkspace" @@ -18,6 +20,8 @@ LOAD_MON = "LoadMonitors" WKSP_PROP_LOAD_MON= "OutputMonitorWorkspace" +FILENAME_RE = re.compile(r'^([0-9]+)(\.[a-zA-z]+)?$') + # Raw workspace names which are necessary at the moment SUMMED_WS = "__loadraw_evs" # Enumerate the indexes for the different foil state sums @@ -315,22 +319,13 @@ def _exec_single_foil_state_mode(self): """ runs = self._get_runs() - all_spectra = [item for sublist in self._spectra for item in sublist] if len(runs) > 1: self._set_spectra_type(all_spectra[0]) self._setup_raw(all_spectra) else: - isis = config.getFacility("ISIS") - vesuvio = isis.instrument("VESUVIO") - run_no = runs[0] - run_str = vesuvio.filePrefix(int(run_no)) + run_no - - self._raise_error_period_scatter(run_str, self._back_scattering) - all_spectra = [item for sublist in self._spectra for item in sublist] - - self._load_single_run_spec_and_mon(all_spectra, run_str) + self._load_single_run_spec_and_mon(all_spectra, self._get_filename(runs[0])) raw_group = mtd[SUMMED_WS] self._nperiods = raw_group.size() @@ -408,7 +403,27 @@ def _exec_single_foil_state_mode(self): #---------------------------------------------------------------------------------------- + def _get_filename(self, run_or_filename): + """Given a string containing either a filename/partial filename or run number find the correct + file prefix""" + isis = config.getFacility("ISIS") + vesuvio = isis.instrument("VESUVIO") + if isinstance(run_or_filename, six.integer_types): + run_no = run_or_filename + return vesuvio.filePrefix(int(run_no)) + str(run_or_filename) + else: + match = FILENAME_RE.match(run_or_filename) + if match: + run_no = match.group(1) + return vesuvio.filePrefix(int(run_no)) + str(run_or_filename) + else: + # Assume file is okay and give it a go with Load + return run_or_filename + + #---------------------------------------------------------------------------------------- + def _load_single_run_spec_and_mon(self, all_spectra, run_str): + self._raise_error_period_scatter(run_str, self._back_scattering) # check if the monitor spectra are already in the spectra list filtered_spectra = sorted([i for i in all_spectra if i <= self._mon_spectra[-1]]) mons_in_ws = False @@ -579,15 +594,13 @@ def _load_and_sum_runs(self, spectra): @param spectra :: The list of spectra to load @returns a tuple of length 2 containing (main_detector_ws, monitor_ws) """ - isis = config.getFacility("ISIS") - vesuvio = isis.instrument("VESUVIO") runs = self._get_runs() self.summed_ws, self.summed_mon = "__loadraw_evs", "__loadraw_evs_monitors" spec_inc_mon = self._mon_spectra spec_inc_mon.extend(spectra) for index, run in enumerate(runs): - filename = vesuvio.filePrefix(int(run)) + str(run) + filename = self._get_filename(run) self._raise_error_period_scatter(filename, self._back_scattering) if index == 0: out_name, out_mon = SUMMED_WS, SUMMED_WS + '_monitors' diff --git a/Testing/SystemTests/tests/analysis/LoadVesuvioTest.py b/Testing/SystemTests/tests/analysis/LoadVesuvioTest.py index 6e8179aaa2cd..79b760a18f90 100644 --- a/Testing/SystemTests/tests/analysis/LoadVesuvioTest.py +++ b/Testing/SystemTests/tests/analysis/LoadVesuvioTest.py @@ -1,7 +1,7 @@ #pylint: disable=invalid-name,no-init,too-many-public-methods,too-many-arguments import stresstesting -from mantid.api import MatrixWorkspace, mtd +from mantid.api import FileFinder, MatrixWorkspace, mtd import mantid.simpleapi as ms import math @@ -46,6 +46,25 @@ def test_spectrum_list_comma_separated_ranges(self): #================== Success cases ================================ + def test_filename_accepts_full_filepath(self): + diff_mode = "FoilOut" + rawfile = FileFinder.getFullPath("EVS14188.raw") + self._run_load(rawfile, "3", diff_mode) + self.assertTrue(mtd.doesExist('evs_raw')) + self.assertEquals(mtd['evs_raw'].getNumberHistograms(), 1) + + def test_filename_accepts_filename_no_path(self): + diff_mode = "FoilOut" + self._run_load("EVS14188.raw", "3", diff_mode) + self.assertTrue(mtd.doesExist('evs_raw')) + self.assertEquals(mtd['evs_raw'].getNumberHistograms(), 1) + + def test_filename_accepts_run_and_ext(self): + diff_mode = "FoilOut" + self._run_load("14188.raw", "3", diff_mode) + self.assertTrue(mtd.doesExist('evs_raw')) + self.assertEquals(mtd['evs_raw'].getNumberHistograms(), 1) + def test_load_with_back_scattering_spectra_produces_correct_workspace_using_double_difference(self): diff_mode = "DoubleDifference" self._run_load("14188", "3-134", diff_mode) diff --git a/scripts/Inelastic/IndirectReductionCommon.py b/scripts/Inelastic/IndirectReductionCommon.py index ef51a0eeb388..477682401f31 100644 --- a/scripts/Inelastic/IndirectReductionCommon.py +++ b/scripts/Inelastic/IndirectReductionCommon.py @@ -37,8 +37,8 @@ def load_files(data_files, ipf_filename, spec_min, spec_max, sum_files=False, lo logger.debug('Loading file %s as workspace %s' % (filename, ws_name)) if 'VESUVIO' in ipf_filename: - evs_filename = os.path.basename(str(filename)).replace('EVS', '') - LoadVesuvio(Filename=evs_filename, + # Load all spectra. They are cropped later + LoadVesuvio(Filename=filename, OutputWorkspace=ws_name, SpectrumList='1-198', **load_opts) From 339d13f1e07ddf0b8b3bd595847fad1736d7ceab Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Tue, 21 Feb 2017 09:15:15 +0000 Subject: [PATCH 6/9] Only pass strings to LoadVesuvio Refs #18843 --- scripts/Inelastic/IndirectReductionCommon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Inelastic/IndirectReductionCommon.py b/scripts/Inelastic/IndirectReductionCommon.py index 477682401f31..84ce8f7e5685 100644 --- a/scripts/Inelastic/IndirectReductionCommon.py +++ b/scripts/Inelastic/IndirectReductionCommon.py @@ -38,7 +38,7 @@ def load_files(data_files, ipf_filename, spec_min, spec_max, sum_files=False, lo if 'VESUVIO' in ipf_filename: # Load all spectra. They are cropped later - LoadVesuvio(Filename=filename, + LoadVesuvio(Filename=str(filename), OutputWorkspace=ws_name, SpectrumList='1-198', **load_opts) From 6f4cee41096807906be85589a83956ca59599fb9 Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Tue, 21 Feb 2017 13:45:07 +0000 Subject: [PATCH 7/9] Revert config changes after each documentation test Refs #18843 --- docs/conf.py.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py.in b/docs/conf.py.in index 233dcc744b36..3e30942ac4e4 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -92,7 +92,9 @@ except NameError: # Run this after each test group has executed doctest_global_cleanup = """ from mantid.api import FrameworkManager +from mantid.kernel import ConfigService FrameworkManager.Instance().clear() +ConfigService.Instance().reset() """ # -- Options for pngmath -------------------------------------------------- From 41f8add375b9eb68fb6f92616501ec2533dac6a4 Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Tue, 21 Feb 2017 17:23:31 +0000 Subject: [PATCH 8/9] Fix loader in case path contains "-" character Refs #18843 --- Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py index ef6f09d5f241..439a024ebbe2 100644 --- a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py +++ b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py @@ -7,6 +7,7 @@ import copy import numpy as np +import os import re import six @@ -158,7 +159,8 @@ def validateInputs(self): # Validate run number ranges run_str = self.getProperty(RUN_PROP).value - if "-" in run_str: + # String could be a full file path + if "-" in os.path.basename(run_str): lower, upper = run_str.split("-") issues = self._validate_range_formatting(lower, upper, RUN_PROP, issues) From 6c508da3e9cffaa6dabbf7bd43007acfe08a73d7 Mon Sep 17 00:00:00 2001 From: Martyn Gigg Date: Wed, 22 Feb 2017 08:49:19 +0000 Subject: [PATCH 9/9] Fix full path handling properly Refs #18843 --- .../plugins/algorithms/LoadVesuvio.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py index 439a024ebbe2..2930a4b2a583 100644 --- a/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py +++ b/Framework/PythonInterface/plugins/algorithms/LoadVesuvio.py @@ -158,9 +158,10 @@ def validateInputs(self): issues = {} # Validate run number ranges - run_str = self.getProperty(RUN_PROP).value + user_input = self.getProperty(RUN_PROP).value + run_str = os.path.basename(user_input) # String could be a full file path - if "-" in os.path.basename(run_str): + if "-" in run_str: lower, upper = run_str.split("-") issues = self._validate_range_formatting(lower, upper, RUN_PROP, issues) @@ -685,17 +686,19 @@ def _get_runs(self): """ Returns the runs as a list of strings """ - run_str = self.getProperty(RUN_PROP).value + # String could be a full file path + user_input = self.getProperty(RUN_PROP).value + run_str = os.path.basename(user_input) # Load is not doing the right thing when summing. The numbers don't look correct if "-" in run_str: lower, upper = run_str.split("-") # Range goes lower to up-1 but we want to include the last number runs = list(range(int(lower), int(upper)+1)) - elif "," in run_str: runs = run_str.split(",") else: - runs = [run_str] + # Leave it as it is + runs = [user_input] return runs