New IDAaaS data cache index file loading

Framework/API/CMakeLists.txt

@@ -80,6 +80,7 @@ set(SRC_FILES
     src/IPeakFunction.cpp
     src/IPeaksWorkspace.cpp
     src/IPowderDiffPeakFunction.cpp
+    src/ISISInstrDataCache.cpp
     src/ISpectrum.cpp
     src/ITableWorkspace.cpp
     src/ImmutableCompositeFunction.cpp
@@ -263,6 +264,7 @@ set(INC_FILES
     inc/MantidAPI/IPeaksWorkspace.h
     inc/MantidAPI/IPeaksWorkspace_fwd.h
     inc/MantidAPI/IPowderDiffPeakFunction.h
+    inc/MantidAPI/ISISInstrDataCache.h
     inc/MantidAPI/IPreview.h
     inc/MantidAPI/ISpectrum.h
     inc/MantidAPI/ISplittersWorkspace.h
@@ -423,6 +425,7 @@ set(TEST_FILES
     ILatticeFunctionTest.h
     IMDWorkspaceTest.h
     IPreviewTest.h
+    ISISInstrDataCacheTest.h
     ISpectrumTest.h
     ImmutableCompositeFunctionTest.h
     ImplicitFunctionParserFactoryTest.h

Framework/API/inc/MantidAPI/FileFinder.h

@@ -36,6 +36,7 @@ number
 class MANTID_API_DLL FileFinderImpl {
 public:
   std::string getFullPath(const std::string &filename, const bool ignoreDirs = false) const;
+  std::string extractAllowedSuffix(std::string &userString) const;
   const API::Result<std::string> getPath(const std::vector<IArchiveSearch_sptr> &archs,
                                          const std::set<std::string> &filenames,
                                          const std::vector<std::string> &exts) const;
@@ -49,10 +50,11 @@ class MANTID_API_DLL FileFinderImpl {
   std::vector<std::string> findRuns(const std::string &hintstr, const std::vector<std::string> &exts = {},
                                     const bool useExtsOnly = false) const;
   /// DO NOT USE! MADE PUBLIC FOR TESTING ONLY.
-  const Kernel::InstrumentInfo getInstrument(const std::string &hint) const;
+  const Kernel::InstrumentInfo getInstrument(const std::string &hint, const bool &throwOnThrow = false) const;
   /// DO NOT USE! MADE PUBLIC FOR TESTING ONLY.
   std::string getExtension(const std::string &filename, const std::vector<std::string> &exts) const;
   void getUniqueExtensions(const std::vector<std::string> &extensionsToAdd, std::vector<std::string> &uniqueExts) const;
+  std::pair<std::string, std::string> toInstrumentAndNumber(const std::string &hint) const;
 
 private:
   friend struct Mantid::Kernel::CreateUsingNew<FileFinderImpl>;
@@ -67,8 +69,9 @@ class MANTID_API_DLL FileFinderImpl {
   FileFinderImpl &operator=(const FileFinderImpl &);
   /// A method that returns error messages if the provided runs are invalid
   std::string validateRuns(const std::string &searchText) const;
-  std::string extractAllowedSuffix(std::string &userString) const;
-  std::pair<std::string, std::string> toInstrumentAndNumber(const std::string &hint) const;
+  const API::Result<std::string> getDataCachePath(const std::string &cachePathToSearch,
+                                                  const std::set<std::string> &filenames,
+                                                  const std::vector<std::string> &exts) const;
   const API::Result<std::string> getArchivePath(const std::vector<IArchiveSearch_sptr> &archs,
                                                 const std::set<std::string> &filenames,
                                                 const std::vector<std::string> &exts) const;

Framework/API/inc/MantidAPI/ISISInstrDataCache.h

@@ -0,0 +1,30 @@
+// Mantid Repository : https://github.com/mantidproject/mantid
+//
+// Copyright © 2010 ISIS Rutherford Appleton Laboratory UKRI,
+//   NScD Oak Ridge National Laboratory, European Spallation Source,
+//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
+// SPDX - License - Identifier: GPL - 3.0 +
+#pragma once
+
+//----------------------------------------------------------------------
+// Includes
+//----------------------------------------------------------------------
+#include "MantidAPI/DllConfig.h"
+#include "MantidAPI/IArchiveSearch.h"
+
+#include <sstream>
+#include <string>
+
+namespace Mantid {
+namespace API {
+
+class MANTID_API_DLL ISISInstrDataCache {
+public:
+  ISISInstrDataCache(const std::string &path) : m_dataCachePath(path){};
+  std::string getFileParentDirPath(std::string filename);
+
+private:
+  std::string m_dataCachePath;
+};
+} // namespace API
+} // namespace Mantid

Framework/API/src/FileFinder.cpp

@@ -11,6 +11,7 @@
 #include "MantidAPI/ArchiveSearchFactory.h"
 #include "MantidAPI/FrameworkManager.h"
 #include "MantidAPI/IArchiveSearch.h"
+#include "MantidAPI/ISISInstrDataCache.h"
 #include "MantidKernel/ConfigService.h"
 #include "MantidKernel/Exception.h"
 #include "MantidKernel/FacilityInfo.h"
@@ -30,6 +31,8 @@
 
 #include <boost/algorithm/string.hpp>
 
+#include <filesystem>
+
 namespace {
 /// static logger object
 Mantid::Kernel::Logger g_log("FileFinder");
@@ -134,7 +137,7 @@ std::string FileFinderImpl::extractAllowedSuffix(std::string &userString) const
  * @param hint :: The name hint.
  * @return This will return the default instrument if it cannot be determined.
  */
-const Kernel::InstrumentInfo FileFinderImpl::getInstrument(const string &hint) const {
+const Kernel::InstrumentInfo FileFinderImpl::getInstrument(const string &hint, const bool &throwOnThrow) const {
   if ((!hint.empty()) && (!isdigit(hint[0]))) {
     string instrName(hint);
     Poco::Path path(instrName);
@@ -158,22 +161,25 @@ const Kernel::InstrumentInfo FileFinderImpl::getInstrument(const string &hint) c
       {
         const auto it = std::find_if(instrName.begin(), instrName.end(), isdigit);
         const auto nChars = std::distance(instrName.begin(), it);
-        instrName = instrName.substr(0, nChars);
+        instrName.resize(nChars);
       }
 
       // go backwards looking for the instrument name to end - gets around
       // delimiters
       if (!instrName.empty()) {
         const auto it = std::find_if(instrName.rbegin(), instrName.rend(), isalpha);
         const auto nChars = std::distance(it, instrName.rend());
-        instrName = instrName.substr(0, nChars);
+        instrName.resize(nChars);
       }
     }
     try {
       const Kernel::InstrumentInfo instrument = Kernel::ConfigService::Instance().getInstrument(instrName);
       return instrument;
     } catch (Kernel::Exception::NotFoundError &e) {
       g_log.debug() << e.what() << "\n";
+      if (throwOnThrow) {
+        throw e;
+      }
     }
   }
   return Kernel::ConfigService::Instance().getInstrument();
@@ -185,7 +191,7 @@ const Kernel::InstrumentInfo FileFinderImpl::getInstrument(const string &hint) c
  * @return A pair of instrument name and run number
  */
 std::pair<std::string, std::string> FileFinderImpl::toInstrumentAndNumber(const std::string &hint) const {
-  // g_log.debug() << "toInstrumentAndNumber(" << hint << ")\n";
+  g_log.debug() << "toInstrumentAndNumber(" << hint << ")\n";
   std::string instrPart;
   std::string runPart;
 
@@ -254,8 +260,6 @@ std::pair<std::string, std::string> FileFinderImpl::toInstrumentAndNumber(const
  * too long
  */
 std::string FileFinderImpl::makeFileName(const std::string &hint, const Kernel::InstrumentInfo &instrument) const {
-  // g_log.debug() << "makeFileName(" << hint << ", " << instrument.shortName()
-  // << ")\n";
   if (hint.empty())
     return "";
 
@@ -305,7 +309,7 @@ std::string FileFinderImpl::getExtension(const std::string &filename, const std:
     std::string extension = toUpper(ext);
     if (extension.rfind('*') == extension.size() - 1) // there is a wildcard at play
     {
-      extension = extension.substr(0, extension.rfind('*'));
+      extension.resize(extension.rfind('*'));
     }
 
     std::size_t found = toUpper(filename).rfind(extension);
@@ -645,6 +649,48 @@ std::vector<std::string> FileFinderImpl::findRuns(const std::string &hintstr, co
   return res;
 }
 
+const API::Result<std::string> FileFinderImpl::getDataCachePath(const std::string &cachePathToSearch,
+                                                                const std::set<std::string> &filenames,
+                                                                const std::vector<std::string> &exts) const {
+  std::string errors;
+  auto dataCache = API::ISISInstrDataCache(cachePathToSearch);
+
+  for (const auto &filename : filenames) {
+
+    std::string parentDirPath;
+
+    try {
+      parentDirPath = dataCache.getFileParentDirPath(filename);
+    } catch (const std::invalid_argument &e) {
+      errors += "Data cache: " + std::string(e.what());
+      return API::Result<std::string>("", errors);
+    }
+
+    try {
+      // Check for permission by trying to access first file in folder
+      std::filesystem::exists(*begin(std::filesystem::directory_iterator(parentDirPath)));
+    } catch (const std::filesystem::filesystem_error &e) {
+      errors += "Data cache: Could not access " + e.path1().parent_path().string() + ". Permission denied.";
+      return API::Result<std::string>("", errors);
+    }
+
+    g_log.debug() << "Searching for file in data cache directory " << parentDirPath << std::endl;
+
+    if (!parentDirPath.empty()) {
+
+      for (const auto &ext : exts) {
+        std::filesystem::path filePath(parentDirPath + '/' + filename + ext);
+
+        if (std::filesystem::exists(filePath)) {
+          return API::Result<std::string>(filePath.string());
+        }
+      }
+      errors += "Data cache: Could not find matching extension for " + filename;
+    }
+  }
+  return API::Result<std::string>("", errors);
+}
+
 /**
  * Return the path to the file found in archive
  * @param archs :: A list of archives to search
@@ -741,8 +787,24 @@ const API::Result<std::string> FileFinderImpl::getPath(const std::vector<IArchiv
     }
   }
 
+  // Search data cache
+  string errors;
+  std::filesystem::path cachePathToSearch(
+      Mantid::Kernel::ConfigService::Instance().getString("datacachesearch.directory"));
+  if (std::filesystem::exists(cachePathToSearch)) {
+
+    API::Result<std::string> cacheFilePath = getDataCachePath(cachePathToSearch.string(), filenames, exts);
+
+    if (cacheFilePath) {
+      return cacheFilePath;
+    }
+    errors += cacheFilePath.errors();
+
+  } else {
+    errors += "Could not find data cache directory: " + cachePathToSearch.string();
+  }
+
   // Search the archive
-  string errors = "";
   if (!archs.empty()) {
     g_log.debug() << "Search the archives\n";
     const auto archivePath = getArchivePath(archs, filenames, exts);
@@ -758,7 +820,6 @@ const API::Result<std::string> FileFinderImpl::getPath(const std::vector<IArchiv
       errors += archivePath.errors();
 
   } // archs
-
   return API::Result<std::string>("", errors);
 }
 

Framework/API/src/ISISInstrDataCache.cpp

@@ -0,0 +1,69 @@
+
+#include "MantidAPI/ISISInstrDataCache.h"
+#include "MantidAPI/FileFinder.h"
+#include "MantidKernel/ConfigService.h"
+#include "MantidKernel/Exception.h"
+#include "MantidKernel/InstrumentInfo.h"
+#include "MantidKernel/Logger.h"
+#include <algorithm>
+#include <boost/lexical_cast.hpp>
+#include <fstream>
+#include <iterator>
+#include <json/reader.h>
+#include <json/value.h>
+#include <string>
+
+namespace {
+Mantid::Kernel::Logger g_log("ISISInstrDataCache");
+} // namespace
+
+std::string Mantid::API::ISISInstrDataCache::getFileParentDirPath(std::string fileName) {
+  g_log.debug() << "ISISInstrDataCache::getFileParentDirPath(" << fileName << ")" << std::endl;
+
+  // Check if suffix eg. -add is present in filename
+  std::string suffix = FileFinder::Instance().extractAllowedSuffix(fileName);
+  if (!suffix.empty()) {
+    throw std::invalid_argument("Unsuported format: Suffix detected.");
+  }
+
+  // Find the last non-digit as the instrument name can contain numbers
+  std::string::reverse_iterator itRev = std::find_if(fileName.rbegin(), fileName.rend(), std::not_fn(isdigit));
+  std::string::size_type nChars = std::distance(itRev, fileName.rend());
+
+  // Check run number
+  std::string runNumber = fileName.substr(nChars);
+  if (runNumber.empty() || !std::all_of(runNumber.begin(), runNumber.end(), ::isdigit)) { // Check for wrong format
+    throw std::invalid_argument("Filename not in correct format.");
+  };
+  runNumber.erase(0, runNumber.find_first_not_of('0')); // Remove padding zeros
+
+  // Get instrument full name from short name
+  std::transform(fileName.begin(), fileName.end(), fileName.begin(), toupper);
+  std::string instrName = fileName.substr(0, nChars);
+  try {
+    auto instrInfo = FileFinder::Instance().getInstrument(instrName, true);
+    instrName = instrInfo.name();
+  } catch (const Kernel::Exception::NotFoundError &) {
+    throw std::invalid_argument("Instrument name not recognized.");
+  }
+
+  // Build path to index file
+  std::string jsonPath = m_dataCachePath + "/" + instrName + "/" + instrName + "_index.json";
+  g_log.debug() << "Opening instrument index file at " << jsonPath << std::endl;
+  std::ifstream ifstrm{jsonPath};
+  if (!ifstrm) {
+    throw std::invalid_argument("Error opennig instrument index file: " + jsonPath);
+  }
+
+  // Read directory path from json file
+  Json::Value json;
+  ifstrm >> json;
+  std::string relativePath = json[runNumber].asString();
+
+  if (relativePath.empty()) {
+    throw std::invalid_argument("Run number " + runNumber + " not found for instrument " + instrName + ".");
+  }
+
+  std::string dirPath = m_dataCachePath + "/" + instrName + "/" + relativePath;
+  return dirPath;
+}