From bdcecfd1aee6ae88030d58ad3a0ab2b988672e91 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 25 Mar 2024 19:11:30 +0100 Subject: [PATCH 1/3] Added temp directory in use detector --- Node/Workers/FlowWorker.cs | 25 ++++++++++++++++------ ServerShared/ITempDirectoryInUseService.cs | 7 ++++++ ServerShared/Workers/TempFileCleaner.cs | 9 ++++++-- 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 ServerShared/ITempDirectoryInUseService.cs diff --git a/Node/Workers/FlowWorker.cs b/Node/Workers/FlowWorker.cs index ea382e875..422a03e6d 100644 --- a/Node/Workers/FlowWorker.cs +++ b/Node/Workers/FlowWorker.cs @@ -8,6 +8,7 @@ using FileFlows.ServerShared.Workers; using FileFlows.Shared.Models; using Jint.Native.Json; +using System.Collections.Concurrent; namespace FileFlows.Node.Workers; @@ -15,7 +16,7 @@ namespace FileFlows.Node.Workers; /// /// A flow worker executes a flow and start a flow runner /// -public class FlowWorker : Worker +public class FlowWorker : Worker, ITempDirectoryInUseService { /// /// A unique identifier to identify the flow worker @@ -25,7 +26,13 @@ public class FlowWorker : Worker public readonly Guid Uid = Guid.NewGuid(); private readonly string _configKeyDefault = Guid.NewGuid().ToString(); - + + public bool IsTempDirectoryInUse(string directory) + { + var tempDirectoryInUse = ExecutingRunners.Keys.Any(t => directory.EndsWith(t.ToString(), StringComparison.OrdinalIgnoreCase)); + return tempDirectoryInUse; + } + /// /// Gets if the config encryption key /// @@ -78,7 +85,10 @@ private bool GetConfigNoEncrypt(ProcessingNode node) private static FlowWorker? Instance; private readonly Mutex mutex = new Mutex(); - private readonly List ExecutingRunners = new (); + /// + /// Concurrent because we have the TempFileCleaner running in parallel + /// + private readonly ConcurrentDictionary ExecutingRunners = new (); private const int DEFAULT_INTERVAL = 10; @@ -506,7 +516,7 @@ private void AddExecutingRunner(Guid uid) mutex.WaitOne(); try { - ExecutingRunners.Add(uid); + ExecutingRunners.TryAdd(uid, null); } finally { @@ -525,8 +535,10 @@ private void RemoveExecutingRunner(Guid uid) mutex.WaitOne(); try { - if (ExecutingRunners.Contains(uid)) - ExecutingRunners.Remove(uid); + if (ExecutingRunners.TryRemove(uid, out var _)) + { + + } else { Logger.Instance?.ILog("Executing runner not in list: " + uid +" => " + string.Join(",", ExecutingRunners.Select(x => x.ToString()))); @@ -742,5 +754,4 @@ private async Task UpdateConfiguration(ProcessingNode node) return true; } - } diff --git a/ServerShared/ITempDirectoryInUseService.cs b/ServerShared/ITempDirectoryInUseService.cs new file mode 100644 index 000000000..3ba618be4 --- /dev/null +++ b/ServerShared/ITempDirectoryInUseService.cs @@ -0,0 +1,7 @@ +namespace FileFlows.ServerShared +{ + public interface ITempDirectoryInUseService + { + public bool IsTempDirectoryInUse(string directory); + } +} diff --git a/ServerShared/Workers/TempFileCleaner.cs b/ServerShared/Workers/TempFileCleaner.cs index 32136fcd3..b964eaf09 100644 --- a/ServerShared/Workers/TempFileCleaner.cs +++ b/ServerShared/Workers/TempFileCleaner.cs @@ -8,14 +8,16 @@ namespace FileFlows.ServerShared.Workers; public class TempFileCleaner:Worker { private string nodeAddress; - + private readonly ITempDirectoryInUseService tempDirectoryInUseService; + /// /// Constructs a temp file cleaner /// The name of the node /// - public TempFileCleaner(string nodeAddress) : base(ScheduleType.Daily, 5) + public TempFileCleaner(string nodeAddress, ITempDirectoryInUseService tempDirectoryInUseService) : base(ScheduleType.Daily, 5) { this.nodeAddress = nodeAddress; + this.tempDirectoryInUseService = tempDirectoryInUseService; Trigger(); } @@ -37,6 +39,9 @@ protected sealed override void Execute() Logger.Instance?.ILog("About to clean temporary directory: " + tempDir.FullName); foreach (var dir in tempDir.GetDirectories()) { + if (tempDirectoryInUseService.IsTempDirectoryInUse(dir.Name)) + continue; + if (dir.CreationTimeUtc < DateTime.UtcNow.AddDays(-1)) { try From 7e95ac074af441ea13926bb5f46035bf49524e82 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 25 Mar 2024 19:14:24 +0100 Subject: [PATCH 2/3] naming --- Node/Workers/FlowWorker.cs | 2 +- ...yInUseService.cs => IWorkerThatUsesTempDirectories.cs} | 2 +- ServerShared/Workers/TempFileCleaner.cs | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) rename ServerShared/{ITempDirectoryInUseService.cs => IWorkerThatUsesTempDirectories.cs} (68%) diff --git a/Node/Workers/FlowWorker.cs b/Node/Workers/FlowWorker.cs index 422a03e6d..4e460e6ce 100644 --- a/Node/Workers/FlowWorker.cs +++ b/Node/Workers/FlowWorker.cs @@ -16,7 +16,7 @@ namespace FileFlows.Node.Workers; /// /// A flow worker executes a flow and start a flow runner /// -public class FlowWorker : Worker, ITempDirectoryInUseService +public class FlowWorker : Worker, IWorkerThatUsesTempDirectories { /// /// A unique identifier to identify the flow worker diff --git a/ServerShared/ITempDirectoryInUseService.cs b/ServerShared/IWorkerThatUsesTempDirectories.cs similarity index 68% rename from ServerShared/ITempDirectoryInUseService.cs rename to ServerShared/IWorkerThatUsesTempDirectories.cs index 3ba618be4..48dbdb8b9 100644 --- a/ServerShared/ITempDirectoryInUseService.cs +++ b/ServerShared/IWorkerThatUsesTempDirectories.cs @@ -1,6 +1,6 @@ namespace FileFlows.ServerShared { - public interface ITempDirectoryInUseService + public interface IWorkerThatUsesTempDirectories { public bool IsTempDirectoryInUse(string directory); } diff --git a/ServerShared/Workers/TempFileCleaner.cs b/ServerShared/Workers/TempFileCleaner.cs index b964eaf09..2482c8d68 100644 --- a/ServerShared/Workers/TempFileCleaner.cs +++ b/ServerShared/Workers/TempFileCleaner.cs @@ -8,16 +8,16 @@ namespace FileFlows.ServerShared.Workers; public class TempFileCleaner:Worker { private string nodeAddress; - private readonly ITempDirectoryInUseService tempDirectoryInUseService; + private readonly IWorkerThatUsesTempDirectories workerThatUsesTempDirectories; /// /// Constructs a temp file cleaner /// The name of the node /// - public TempFileCleaner(string nodeAddress, ITempDirectoryInUseService tempDirectoryInUseService) : base(ScheduleType.Daily, 5) + public TempFileCleaner(string nodeAddress, IWorkerThatUsesTempDirectories workerThatUsesTempDirectories) : base(ScheduleType.Daily, 5) { this.nodeAddress = nodeAddress; - this.tempDirectoryInUseService = tempDirectoryInUseService; + this.workerThatUsesTempDirectories = workerThatUsesTempDirectories; Trigger(); } @@ -39,7 +39,7 @@ protected sealed override void Execute() Logger.Instance?.ILog("About to clean temporary directory: " + tempDir.FullName); foreach (var dir in tempDir.GetDirectories()) { - if (tempDirectoryInUseService.IsTempDirectoryInUse(dir.Name)) + if (workerThatUsesTempDirectories.IsTempDirectoryInUse(dir.Name)) continue; if (dir.CreationTimeUtc < DateTime.UtcNow.AddDays(-1)) From e7f03406bf80a51a68a47bf5524723b1b218ef47 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 25 Mar 2024 19:15:05 +0100 Subject: [PATCH 3/3] comments --- ServerShared/Workers/TempFileCleaner.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ServerShared/Workers/TempFileCleaner.cs b/ServerShared/Workers/TempFileCleaner.cs index 2482c8d68..428f2eaea 100644 --- a/ServerShared/Workers/TempFileCleaner.cs +++ b/ServerShared/Workers/TempFileCleaner.cs @@ -13,6 +13,7 @@ public class TempFileCleaner:Worker /// /// Constructs a temp file cleaner /// The name of the node + /// The worker that uses temp directories, this is used to verify that a temp directory should not be removed while still in use" /// public TempFileCleaner(string nodeAddress, IWorkerThatUsesTempDirectories workerThatUsesTempDirectories) : base(ScheduleType.Daily, 5) {