dird: create restore tree caches on backup

core/src/dird/backup.cc

@@ -56,6 +56,11 @@
 #include "lib/util.h"
 #include "lib/version.h"
 #include "lib/bpipe.h"
+#include "lib/tree.h"
+#include "lib/tree_save.h"
+
+#include <string_view>
+#include <filesystem>
 
 namespace directordaemon {
 
@@ -173,6 +178,8 @@ bool DoNativeBackupInit(JobControlRecord* jcr)
 
   CreateClones(jcr); /* run any clone jobs */
 
+  jcr->dir_impl->backup_tree_root = new_tree(1);
+
   return true;
 }
 
@@ -727,6 +734,24 @@ int WaitForJobTermination(JobControlRecord* jcr, int timeout)
   return jcr->dir_impl->SDJobStatus;
 }
 
+static bool EnsurePathExists(const std::string& path)
+{
+  try {
+    std::filesystem::create_directories(path);
+    return true;
+  } catch (const std::system_error& e) {
+    Dmsg1(100, "Caught system error while creating path %s: [%s:%d] ERR=%s\n",
+          path.c_str(), e.code().category().name(), e.code().value(), e.what());
+  } catch (const std::exception& e) {
+    Dmsg1(100, "Caught exception while creating path %s: %s\n", path.c_str(),
+          e.what());
+  } catch (...) {
+    Dmsg1(30, "Caught unknown exception while creating path %s\n",
+          path.c_str());
+  }
+  return false;
+}
+
 // Release resources allocated during backup.
 void NativeBackupCleanup(JobControlRecord* jcr, int TermCode)
 {
@@ -760,15 +785,19 @@ void NativeBackupCleanup(JobControlRecord* jcr, int TermCode)
 
   UpdateBootstrapFile(jcr);
 
+  bool create_tree = false;
+
   switch (jcr->getJobStatus()) {
     case JS_Terminated:
       TermMsg = T_("Backup OK");
+      create_tree = true;
       break;
     case JS_Incomplete:
       TermMsg = T_("Backup failed -- incomplete");
       break;
     case JS_Warnings:
       TermMsg = T_("Backup OK -- with warnings");
+      create_tree = true;
       break;
     case JS_FatalError:
     case JS_ErrorTerminated:
@@ -799,6 +828,19 @@ void NativeBackupCleanup(JobControlRecord* jcr, int TermCode)
 
   GenerateBackupSummary(jcr, &cr, msg_type, TermMsg);
 
+  if (jcr->dir_impl->backup_tree_root) {
+    if (create_tree && !jcr->dir_impl->cache_dir.empty()) {
+      if (EnsurePathExists(jcr->dir_impl->cache_dir)) {
+        std::string path = jcr->dir_impl->cache_dir + std::string{"/"}
+                           + std::to_string(jcr->JobId) + ".tree";
+        SaveTree(path.c_str(), jcr->dir_impl->backup_tree_root);
+      }
+    }
+    FreeTree(jcr->dir_impl->backup_tree_root);
+    jcr->dir_impl->backup_tree_root = nullptr;
+  }
+
+
   Dmsg0(100, "Leave backup_cleanup()\n");
 }
 

core/src/dird/catreq.cc

@@ -46,6 +46,10 @@
 #include "lib/util.h"
 #include "lib/serial.h"
 
+#include "lib/tree.h"
+#include "lib/attribs.h"
+#include "lib/scan.h"
+
 namespace directordaemon {
 
 /*
@@ -408,6 +412,114 @@ void CatalogRequest(JobControlRecord* jcr, BareosSocket* bs)
   return;
 }
 
+static bool InsertNode(JobControlRecord* jcr, AttributesDbRecord* ar)
+{
+  if (!jcr->dir_impl->backup_tree_root) { return true; }
+  struct stat statp;
+  int32_t LinkFI;
+  DecodeStat(ar->attr, &statp, sizeof(statp), &LinkFI);
+
+  int FileIndex = ar->FileIndex;
+  bool hard_link = (LinkFI != 0);
+  int32_t delta_seq = ar->DeltaSeq;
+  JobId_t JobId = ar->JobId;
+
+  PoolMem Path, File;
+  if (S_ISDIR(statp.st_mode)) {
+    Path.strcpy(ar->fname);
+    File.strcpy("");
+  } else {
+    int fsize, psize;  // not used
+    SplitPathAndFilename(ar->fname, Path.addr(), &psize, File.addr(), &fsize);
+  }
+
+  int type = [&] {
+    if (*File.c_str() != 0) {
+      return TN_FILE;
+    } else if (!IsPathSeparator(*Path.c_str())) {
+      return TN_DIR_NLS;
+    } else {
+      return TN_DIR;
+    }
+  }();
+
+  auto* root = jcr->dir_impl->backup_tree_root;
+  auto* node = insert_tree_node(Path.c_str(), File.c_str(), type, root, NULL);
+
+  node->fhinfo = ar->Fhinfo;
+  node->fhnode = ar->Fhnode;
+
+  if (delta_seq > 0) {
+    if (delta_seq == (node->delta_seq + 1)) {
+      TreeAddDeltaPart(root, node, node->JobId, node->FileIndex);
+
+    } else {
+      /* File looks to be deleted */
+      if (node->delta_seq == -1) { /* just created */
+        TreeRemoveNode(root, node);
+
+      } else {
+        Dmsg3(0,
+              "Something is wrong with Delta, skip it "
+              "fname=%s d1=%d d2=%d\n",
+              File.c_str(), node->delta_seq, delta_seq);
+      }
+      return false;
+    }
+  }
+
+  /* - The first time we see a file (node->inserted==true), we accept it.
+   * - In the same JobId, we accept only the first copy of a
+   *   hard linked file (the others are simply pointers).
+   * - In the same JobId, we accept the last copy of any other
+   *   file -- in particular directories.
+   *
+   * All the code to set ok could be condensed to a single
+   * line, but it would be even harder to read. */
+  bool ok = true;
+  if (!node->inserted && JobId == node->JobId) {
+    if ((hard_link && FileIndex > node->FileIndex)
+        || (!hard_link && FileIndex < node->FileIndex)) {
+      ok = false;
+    }
+  }
+  if (ok) {
+    node->hard_link = hard_link;
+    node->FileIndex = FileIndex;
+    node->JobId = JobId;
+    node->type = type;
+    node->soft_link = S_ISLNK(statp.st_mode) != 0;
+    node->delta_seq = delta_seq;
+
+    // Insert file having hardlinks into hardlink hashtable.
+    if (statp.st_nlink > 1 && type != TN_DIR && type != TN_DIR_NLS) {
+      if (!LinkFI) {
+        // First occurence - file hardlinked to
+        auto* entry = (HL_ENTRY*)root->hardlinks.hash_malloc(sizeof(HL_ENTRY));
+        entry->key = (((uint64_t)JobId) << 32) + FileIndex;
+        entry->node = node;
+        root->hardlinks.insert(entry->key, entry);
+      } else {
+        // See if we are optimizing for speed or size.
+        // Hardlink to known file index: lookup original file
+        uint64_t file_key = (((uint64_t)JobId) << 32) + LinkFI;
+        HL_ENTRY* first_hl = (HL_ENTRY*)root->hardlinks.lookup(file_key);
+
+        if (first_hl && first_hl->node) {
+          // Then add hardlink entry to linked node.
+          auto* entry
+              = (HL_ENTRY*)root->hardlinks.hash_malloc(sizeof(HL_ENTRY));
+          entry->key = (((uint64_t)JobId) << 32) + FileIndex;
+          entry->node = first_hl->node;
+          root->hardlinks.insert(entry->key, entry);
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
 /**
  * Note, we receive the whole attribute record, but we select out only the
  * stat packet, VolSessionId, VolSessionTime, FileIndex, file type, and file
@@ -546,6 +658,7 @@ static void UpdateAttribute(JobControlRecord* jcr,
       ar->Fhinfo = 0;
       ar->Fhnode = 0;
 
+      InsertNode(jcr, ar);
 
       Dmsg2(400, "dird<filed: stream=%d %s\n", Stream, fname);
       Dmsg1(400, "dird<filed: attr=%s\n", attr);

core/src/dird/dird_conf.cc

@@ -140,6 +140,8 @@ static ResourceItem dir_items[] = {
   { "VerId", CFG_TYPE_STR, ITEM(res_dir, verid), 0, 0, NULL, NULL, NULL },
   { "OptimizeForSize", CFG_TYPE_BOOL, ITEM(res_dir, optimize_for_size), 0, CFG_ITEM_DEFAULT, "false", NULL, NULL },
   { "OptimizeForSpeed", CFG_TYPE_BOOL, ITEM(res_dir, optimize_for_speed), 0, CFG_ITEM_DEFAULT, "false", NULL, NULL },
+  { "CacheDirectory", CFG_TYPE_DIR, ITEM(res_dir, cache_directory), 0, CFG_ITEM_DEFAULT | CFG_ITEM_PLATFORM_SPECIFIC, "trees", "23.1.0-",
+    "path name to where restore tree caches should be stored. If the path is relative, then it is used relative to Working Directory." },
   { "KeyEncryptionKey", CFG_TYPE_AUTOPASSWORD, ITEM(res_dir, keyencrkey), 1, 0, NULL, NULL, NULL },
   { "NdmpSnooping", CFG_TYPE_BOOL, ITEM(res_dir, ndmp_snooping), 0, 0, NULL, "13.2.0-", NULL },
   { "NdmpLogLevel", CFG_TYPE_PINT32, ITEM(res_dir, ndmp_loglevel), 0, CFG_ITEM_DEFAULT, "4", "13.2.0-", NULL },

core/src/dird/dird_conf.h

@@ -36,6 +36,7 @@
 #include "lib/messages_resource.h"
 #include "lib/resource_item.h"
 #include "lib/tls_conf.h"
+#include "lib/parse_conf.h"
 
 template <typename T> class dlist;
 struct json_t;
@@ -115,6 +116,7 @@ class DirectorResource
   dlist<IPADDR>* DIRsrc_addr = nullptr; /* Address to source connections from */
   char* query_file = nullptr;           /* SQL query file */
   char* working_directory = nullptr;    /* WorkingDirectory */
+  char* cache_directory = nullptr;      /* directory for caches */
   char* scripts_directory = nullptr;    /* ScriptsDirectory */
   char* plugin_directory = nullptr;     /* Plugin Directory */
   alist<const char*>* plugin_names = nullptr; /* Plugin names to load */
@@ -686,5 +688,25 @@ void DestroyConfigureUsageString();
 bool PopulateDefs();
 std::vector<JobResource*> GetAllJobResourcesByClientName(std::string name);
 
+
+template <typename T> struct resource_type_type;
+
+template <> struct resource_type_type<DirectorResource> {
+  static constexpr auto value = R_DIRECTOR;
+};
+
+template <> struct resource_type_type<FilesetResource> {
+  static constexpr auto value = R_FILESET;
+};
+
+template <typename T>
+static inline constexpr auto resource_type_v = resource_type_type<T>::value;
+
+template <typename T>
+inline T* NextRes(ConfigurationParser* conf, T* cur = nullptr)
+{
+  return static_cast<T*>(conf->GetNextRes(resource_type_v<T>, cur));
+}
+
 } /* namespace directordaemon */
 #endif  // BAREOS_DIRD_DIRD_CONF_H_

core/src/dird/director_jcr_impl.h

@@ -28,6 +28,9 @@
 #include "dird/client_connection_handshake_mode.h"
 #include "dird/job_trigger.h"
 
+#include <string_view>
+#include <string>
+
 typedef struct s_tree_root TREE_ROOT;
 
 class ConfigResourcesContainer;
@@ -99,7 +102,11 @@ struct Resources {
 };
 
 struct DirectorJcrImpl {
-  DirectorJcrImpl( std::shared_ptr<ConfigResourcesContainer> configuration_resources_container) : job_config_resources_container_(configuration_resources_container) {
+  DirectorJcrImpl(std::shared_ptr<ConfigResourcesContainer> configuration_resources_container,
+		  std::string_view cache_dir)
+    : job_config_resources_container_(configuration_resources_container)
+    , cache_dir{cache_dir}
+  {
     RestoreJobId = 0; MigrateJobId = 0; VerifyJobId = 0;
   }
   std::shared_ptr<ConfigResourcesContainer> job_config_resources_container_;
@@ -109,6 +116,7 @@ struct DirectorJcrImpl {
   pthread_cond_t nextrun_ready = PTHREAD_COND_INITIALIZER;  /**< Wait for job next run to become ready */
   Resources res;                  /**< Resources assigned */
   TREE_ROOT* restore_tree_root{}; /**< Selected files to restore (some protocols need this info) */
+  TREE_ROOT* backup_tree_root{}; /**< files that are getting backed up */
   storagedaemon::BootStrapRecord* bsr{}; /**< Bootstrap record -- has everything */
   char* backup_format{};          /**< Backup format used when doing a NDMP backup */
   char* plugin_options{};         /**< User set options for plugin */
@@ -169,6 +177,8 @@ struct DirectorJcrImpl {
   directordaemon::ClientConnectionHandshakeMode connection_handshake_try_{
     directordaemon::ClientConnectionHandshakeMode::kUndefined};
   JobTrigger job_trigger{JobTrigger::kUndefined};
+
+  std::string cache_dir;
 };
 /* clang-format on */
 

core/src/dird/jcr_util.cc

@@ -21,20 +21,39 @@
 
 #include "dird/dird_globals.h"
 #include "dird/director_jcr_impl.h"
+#include "dird/dird_conf.h"
 #include "lib/parse_bsr.h"
 #include "lib/parse_conf.h"
 
+#include <filesystem>
+#include <string>
 
 namespace directordaemon {
 
 JobControlRecord* NewDirectorJcr(JCR_free_HANDLER* DirdFreeJcr)
 {
   JobControlRecord* jcr = new_jcr(DirdFreeJcr);
-  jcr->dir_impl = new DirectorJcrImpl(my_config->config_resources_container_);
+
+  {
+    ResLocker _{my_config};
+
+    std::string cache_dir{};
+    if (auto* me = NextRes<DirectorResource>(my_config); me != nullptr) {
+      std::filesystem::path cache_path(me->cache_directory);
+      if (cache_path.is_absolute()) {
+        cache_dir = cache_path.string();
+      } else {
+        std::filesystem::path working_path(me->working_directory);
+        cache_dir = (working_path / cache_path).string();
+      }
+    }
+    jcr->dir_impl = new DirectorJcrImpl(my_config->config_resources_container_,
+                                        cache_dir);
+    Dmsg1(10, "NewDirectorJcr: configuration_resources_ is at %p %s\n",
+          my_config->config_resources_container_->configuration_resources_,
+          my_config->config_resources_container_->TimeStampAsString().c_str());
+  }
   register_jcr(jcr);
-  Dmsg1(10, "NewDirectorJcr: configuration_resources_ is at %p %s\n",
-        my_config->config_resources_container_->configuration_resources_,
-        my_config->config_resources_container_->TimeStampAsString().c_str());
   return jcr;
 }
 

core/src/dird/msgchan.cc

@@ -475,6 +475,7 @@ extern "C" void* msg_thread(void* arg)
   // Read the Storage daemon's output.
   Dmsg0(100, "Start msg_thread loop\n");
   n = 0;
+
   while (!jcr->IsJobCanceled() && (n = BgetDirmsg(sd)) >= 0) {
     Dmsg1(400, "<stored: %s", sd->msg);
     /* Check for "3000 OK Job Authorization="
@@ -510,8 +511,10 @@ extern "C" void* msg_thread(void* arg)
      * but still end as JS_Warnings (OK -- with warnings). */
     Qmsg(jcr, M_FATAL, 0, T_("Director's comm line to SD dropped.\n"));
   }
+
   if (IsBnetError(sd)) { jcr->dir_impl->SDJobStatus = JS_ErrorTerminated; }
   pthread_cleanup_pop(1); /* remove and execute the handler */
+
   return NULL;
 }