From 9fd7a6f55e195bb8f8af18481edd01bc0290e956 Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 17 Mar 2024 21:05:23 +0100 Subject: [PATCH 1/5] Added file open helper which fixes opening files with too strict FileShare configurations --- Server/Controllers/DownloadController.cs | 2 +- Server/Controllers/PluginController.cs | 2 +- Server/Helpers/Gzipper.cs | 6 +- Server/Server.csproj | Bin 4192 -> 8510 bytes Server/Workers/WatchedLibrary.cs | 96 ++++++++++---------- ServerShared/FileServices/FileDownloader.cs | 2 +- ServerShared/FileServices/FileHasher.cs | 10 +- ServerShared/FileServices/FileUploader.cs | 4 +- ServerShared/Helpers/FileHelper.cs | 3 +- Shared/Helpers/FileOpenHelper.cs | 34 +++++++ Shared/Helpers/HttpHelper.cs | 3 +- 11 files changed, 99 insertions(+), 63 deletions(-) create mode 100644 Shared/Helpers/FileOpenHelper.cs diff --git a/Server/Controllers/DownloadController.cs b/Server/Controllers/DownloadController.cs index c173f70ee..c0f7a2655 100644 --- a/Server/Controllers/DownloadController.cs +++ b/Server/Controllers/DownloadController.cs @@ -20,7 +20,7 @@ public IActionResult Download() if (System.IO.File.Exists(file) == false) return NotFound(); - return new FileStreamResult(System.IO.File.OpenRead(file), "application/octet-stream") + return new FileStreamResult(FileOpenHelper.OpenRead_NoLocks(file), "application/octet-stream") { FileDownloadName = zipName }; diff --git a/Server/Controllers/PluginController.cs b/Server/Controllers/PluginController.cs index 749e70066..83715094a 100644 --- a/Server/Controllers/PluginController.cs +++ b/Server/Controllers/PluginController.cs @@ -218,7 +218,7 @@ public FileStreamResult DownloadPackage([FromRoute] string package) try { - return File(System.IO.File.OpenRead(file), "application/octet-stream"); + return File(FileOpenHelper.OpenRead_NoLocks(file), "application/octet-stream"); } catch(Exception ex) { diff --git a/Server/Helpers/Gzipper.cs b/Server/Helpers/Gzipper.cs index 6d1aa54a7..8605d1a59 100644 --- a/Server/Helpers/Gzipper.cs +++ b/Server/Helpers/Gzipper.cs @@ -22,7 +22,7 @@ public static bool CompressFile(string inputFile, string outputFile, bool delete if (File.Exists(outputFile)) File.Delete(outputFile); - using (FileStream originalFileStream = File.Open(inputFile, FileMode.Open)) + using (FileStream originalFileStream = FileOpenHelper.OpenRead_NoLocks(inputFile)) { using FileStream compressedFileStream = File.Create(outputFile); using var compressor = new GZipStream(compressedFileStream, CompressionMode.Compress); @@ -55,7 +55,7 @@ public static bool DecompressFile(string inputFile, string outputFile, bool dele if (File.Exists(outputFile)) File.Delete(outputFile); - using FileStream compressedFileStream = File.Open(inputFile, FileMode.Open); + using FileStream compressedFileStream = FileOpenHelper.OpenRead_NoLocks(inputFile); using FileStream outputFileStream = File.Create(outputFile); using var decompressor = new GZipStream(compressedFileStream, CompressionMode.Decompress); decompressor.CopyTo(outputFileStream); @@ -96,7 +96,7 @@ public static void CompressToFile(string file, string contents) /// the decompresses file contents public static string DecompressFileToString(string inputFile, int lines = 0, int bytes = 0) { - using FileStream compressedFileStream = File.Open(inputFile, FileMode.Open); + using FileStream compressedFileStream = FileOpenHelper.OpenRead_NoLocks(inputFile); using MemoryStream outputStream = new MemoryStream(); using var decompressor = new GZipStream(compressedFileStream, CompressionMode.Decompress); diff --git a/Server/Server.csproj b/Server/Server.csproj index a85aa98a5e2e3742b60484a7fa9d9aa1b9b6f272..c04cb1598d196d6102e1c19f9e46b1cfb40b461b 100644 GIT binary patch literal 8510 zcmeHMZBH9V5T4JK`X5|WiGa$1Atj0GI3ghsqK2aIk}pV=@f%LCv3%zMQGa{W=b2%9 zd*?mdxHX|th48)X?#wfj2bDHZvPzRzT!;eMs}6fQ2MvC-As3_fN~9U+E|gu4(@7@wJBHN zJ^-dFWZ}pYm9rwN6X*MQQZD;4ga$7_{S3cN(51fI6X}l52vaiMH9(KL+rPzI1@?|o z-ofuB@W-lG+RLm)9z|=zoKAo-wRRPDj$|GGwD|@sx(Q3JN`6+DR+e*qO_=idHDNxl zAbTHnA3z&zIfsU-iZb**Okof9??Z?5!w%LTtH0Bum8i}Bplu`5M!j`!zy*Tu{p-h5Hpl$4Uif%7$Yt%&id$R1XF#yaYRs}n>8 zedG*Jl=BO&39gLkD((i#o3_2%N=j9P|FlV0WA^56>|w-*YLPV7 zidToZ?P5K14Kq-J>o)XNSBvv~;@QaQtBPEd+G>{dVI=hUrffm0eR%^EF}P=Wg=d7R z86AtK)X(-HoB12#jCT?2%sbj@(wZ}>^Yj+xT&S$%dDqoN z7|eR)#_Z8l|Kwga1RnbPui(mTGQ^eGTvjm*v#j=!kc#Ea;yo}se@{_%r`Rc}MS7z7 zV@W0J;vIdSwE(Th+-ds|rRDq8$H5vrRtZ&VrelpA{0%tqCHNr}7T=0A?fe z0q*D_tBjLmRtc=8aUBCjS_}UJ{Ifm?Xa5L(dWus7>n*Fbc$yu36M+KHBi54$N#ykJ z`U|t*U?yZXJHZ}9-VdRB6${!SKqt;IR>j4V@Q&~&-=>p+haO{ z=d%j5y6zBGw=Pffo8WtcteWlQge_;?w#Ue0 zo)I`g{c)5%5<5{B%Ro7-OI&>NV?}&d|383L*;Qf;UxJ5qii>e<&IyOrtvrSCG(suZ zYltMI%`iG5`}qN&Bj%oJ)9*OhK0g}84+nMjUb1hnZ$Y&u;YK6@0N+!`F(ZbI70UO|iSc`qDZEVU47?IG)dd^Qa~A-JWCQ>IQJ~ zyueN!`wF)I**mM~mr(~TV>H_paMgjkRy6zb8wI$_wvCzGscyjO0N>~q3;njD=dBN*$1YwKmDCetgtGi{d$*3eS2znj#rMoI ze$ywL<;K>KN&K;m%+q7cs-fP&nyCxhNcta4ojMpoej$XX>XY62>SqPWXU7RBy{>_d$EtS zudv59{(;6>lD4y4*jPH}lg_83qfh_*^|$l(A>h^$lx9-2?Ivm176?!JOw8MMWM;Ky z%YN(bIRhm>gJ-Off7iBen5U$cGei3C#sr(lN0{2~-kycO*fa!XR@X{Kp_{ZN?PIFu zV6GGm;9jabSAcm<8cuz?z3}KqoYN@>7exGW>Ul$9gQNM8XZnchUNH_=T;6MI1PX(4 z>gC?T=R+E=+aM>TNlsfWr=B$xMr+rsqh!0)Q@C_xxKeDsFs@TC{ulkdAmq$r=2j!U zI*)3rxv>AGR@N1&~HW){{W3aDUTKv}g~iDYzfdSXcxqrzNsIi;L^ zp$1piH7Y_1D>^qohEgVk>X3RRR>}V1&FHs?aUU6ovmqv(3?bxbc4%Anua^DEK00#h zn*c>X9jzgl^3~>Qk!-a#x(lys5TK}mOX)=c2$O^pueCc)LaOdBFm5$;!tlM+X6;mJ zD6G%~7bwB+c}N+<2f>F^-|i`V(#c;_HKlXtO3sm(nC=_}$e%TYKrW$edol=-Mukz%*qnUUQrK3# zluemKPk21?(L+U|rjxxzrqt|itQVWMeJLQHM!s#o#jLQ~hRf!e1xCYAgE8?gBbTrQ zF2gu+HY3EnP;gXDT9%X@V-%Z42CSTo(Mlr%8Aie|+zH(^o$-=04>ai^-AZ5{d|;Yk z0UOI41gm9S6tkc6X+Yz`i1y9AjEzAX?bOR%xf5pOdN1XWDdLV%Z4jF!o}P^E>3j~V zCwzDy3ls2s*>=Qz7fiwT!OtKIva*&RFi2-fxfG0Mki{U^5u2pFlVkEgOY@JJcAj?3 zc6%Q4u0DD>-}I?w;hQlwc#Q|F&ctfeOPJA!n~|qjXf%fYS?3el7@!(_UDAgz1BD8p zl;wm>Zm7U|q;&p=csj&0ARi=Q>Pd*c^kEV|p6{BL(9wb_@F(Su7eK8+lGpRy2RxcH zF-c$B?z<9Ay*niH5u&r~w5_%1KtYCzjJ(;R`9A@$=IJ{l!|w+gFCf4|He#q;;Kgw7 zGp$lWK;`v{GS2^ElG# /// A watched library is a folder that imports files into FileFlows /// -public class WatchedLibrary:IDisposable +public class WatchedLibrary : IDisposable { private FileSystemWatcher Watcher; - public Library Library { get;private set; } + public Library Library { get; private set; } - public bool ScanComplete { get;private set; } = false; + public bool ScanComplete { get; private set; } = false; /// /// Gets if the scanner should be used instead of file watched for this library /// @@ -48,10 +48,10 @@ public WatchedLibrary(Library library) this.UseScanner = true; } - if(UseScanner == false) + if (UseScanner == false) SetupWatcher(); - - + + QueueTimer = new(); QueueTimer.Elapsed += QueueTimerOnElapsed; //QueueTimer.AutoReset = false; @@ -69,7 +69,7 @@ private void LogQueueMessage(string message, Settings? settings = null) if (settings?.LogQueueMessages != true) return; - + Logger.Instance.DLog(message); } @@ -81,8 +81,8 @@ private void LogQueueMessage(string message, Settings? settings = null) private void QueueTimerOnElapsed(object? sender, ElapsedEventArgs e) => ProcessQueue(); - private SemaphoreSlim processorLock= new (1); - + private SemaphoreSlim processorLock = new(1); + /// /// Processes the queue /// @@ -115,7 +115,7 @@ private void Worker_DoWork(object? sender, DoWorkEventArgs e) while (Disposed == false) { ProcessQueuedItem(); - if(QueuedHasItems() != true) + if (QueuedHasItems() != true) { LogQueueMessage($"{Library.Name} nothing queued"); Thread.Sleep(1000); @@ -209,7 +209,7 @@ private void ProcessQueuedItem() if (Library.SkipFileAccessTests == false && Library.Folders == false && CanAccess((FileInfo)fsInfo, Library.FileSizeDetectionInterval).Result == false) { - Logger.Instance.WLog($"Failed access checks for file: " + fullpath +"\n" + + Logger.Instance.WLog($"Failed access checks for file: " + fullpath + "\n" + "These checks can be disabled in library settings, but ensure the flow can read and write to the library."); return; } @@ -254,7 +254,7 @@ private void ProcessQueuedItem() SystemEvents.TriggerFileAdded(result, Library); Logger.Instance.DLog( $"Time taken \"{(DateTime.UtcNow.Subtract(dtTotal))}\" to successfully add new library file: \"{fullpath}\""); - + if (new SettingsService().Get()?.Result?.ShowFileAddedNotifications == true) ClientServiceManager.Instance.SendToast(LogType.Info, "New File: " + result.RelativePath); } @@ -296,24 +296,24 @@ private bool MatchesDetection(string fullpath) /// true if matches detection, otherwise false public static bool MatchesDetection(Library library, FileSystemInfo info, long size) { - if(MatchesValue((int)DateTime.UtcNow.Subtract(info.CreationTimeUtc).TotalMinutes, library.DetectFileCreation, library.DetectFileCreationLower, library.DetectFileCreationUpper) == false) + if (MatchesValue((int)DateTime.UtcNow.Subtract(info.CreationTimeUtc).TotalMinutes, library.DetectFileCreation, library.DetectFileCreationLower, library.DetectFileCreationUpper) == false) return false; - if(MatchesValue((int)DateTime.UtcNow.Subtract(info.LastWriteTimeUtc).TotalMinutes, library.DetectFileLastWritten, library.DetectFileLastWrittenLower, library.DetectFileLastWrittenUpper) == false) + if (MatchesValue((int)DateTime.UtcNow.Subtract(info.LastWriteTimeUtc).TotalMinutes, library.DetectFileLastWritten, library.DetectFileLastWrittenLower, library.DetectFileLastWrittenUpper) == false) return false; - - if(MatchesValue(size, library.DetectFileSize, library.DetectFileSizeLower, library.DetectFileSizeUpper) == false) + + if (MatchesValue(size, library.DetectFileSize, library.DetectFileSizeLower, library.DetectFileSizeUpper) == false) return false; - + return true; - + } - + private static bool MatchesValue(long value, MatchRange range, long low, long high) { if (range == MatchRange.Any) return true; - + if (range == MatchRange.GreaterThan) return value > low; if (range == MatchRange.LessThan) @@ -453,7 +453,7 @@ private bool FileIsHidden(string fullpath) // recursively search the directories to see if its hidden var dir = new FileInfo(fullpath).Directory; int count = 0; - while(dir.Parent != null) + while (dir.Parent != null) { if (dir.Attributes.HasFlag(FileAttributes.Hidden)) return true; @@ -463,13 +463,13 @@ private bool FileIsHidden(string fullpath) } return false; } - + /// /// Disposes of the watched library /// public void Dispose() { - Disposed = true; + Disposed = true; DisposeWatcher(); //worker.Dispose(); //QueueTimer?.Dispose(); @@ -535,7 +535,7 @@ private bool IsMatch(string input) } catch (Exception) { } } - + if (string.IsNullOrWhiteSpace(Library.Filter) == false) { try @@ -582,7 +582,7 @@ private void Watcher_Changed(object sender, FileSystemEventArgs e) var file = new FileInfo(e.FullPath); if (file.Exists == false) return; - + long size = file.Length; Thread.Sleep(20_000); if (size < file.Length) @@ -600,7 +600,7 @@ private void Watcher_Changed(object sender, FileSystemEventArgs e) } private void FileChangeEvent(string fullPath) - { + { if (IsMatch(fullPath) == false) { if (fullPath.Contains("_UNPACK_")) @@ -628,17 +628,17 @@ internal void UpdateLibrary(Library library) UseScanner = true; SetupWatcher(); } - else if(UseScanner == false && library.Scan == true) + else if (UseScanner == false && library.Scan == true) { Logger.Instance.ILog($"WatchedLibrary: Library '{library.Name}' switched to scan mode, disposing watcher"); UseScanner = false; DisposeWatcher(); } - else if(UseScanner == false && Watcher != null && Watcher.Path != library.Path) + else if (UseScanner == false && Watcher != null && Watcher.Path != library.Path) { // library path changed, need to change watcher Logger.Instance.ILog($"WatchedLibrary: Library '{library.Name}' path changed, updating watched path"); - SetupWatcher(); + SetupWatcher(); } if (library.Enabled && library.LastScanned < new DateTime(2020, 1, 1) && Directory.Exists(library.Path)) @@ -672,9 +672,9 @@ public void Scan() Logger.Instance?.WLog($"WatchedLibrary: Library '{Library.Name}' path not found: {Library.Path}"); return; } - + Logger.Instance.ILog($"WatchedLibrary: Scan started on '{Library.Name}': {Library.Path}"); - + int count = 0; if (Library.Folders) { @@ -702,13 +702,14 @@ public void Scan() if (MatchesDetection(file.FullName) == false) continue; - + if (knownFiles.TryGetValue(file.FullName.ToLowerInvariant(), out KnownFileInfo info)) { if (Library.DownloadsDirectory && info.Status == FileStatus.Processed) { - - }else if (DatesAreSame(file, info)) + + } + else if (DatesAreSame(file, info)) continue; } @@ -724,13 +725,13 @@ public void Scan() Logger.Instance.ILog($"WatchedLibrary: Files queued for '{Library.Name}': {count} / {QueueCount()}"); ScanComplete = true; - + Library.LastScanned = DateTime.UtcNow; ServiceLoader.Load().UpdateLastScanned(Library.Uid).Wait(); } - catch(Exception ex) + catch (Exception ex) { - while(ex.InnerException != null) + while (ex.InnerException != null) ex = ex.InnerException; Logger.Instance.ELog("WatchedLibrary: Failed scanning for files: " + ex.Message + Environment.NewLine + ex.StackTrace); @@ -749,20 +750,20 @@ private bool DatesAreSame(FileInfo file, KnownFileInfo knownFile) file.LastWriteTime, knownFile.LastWriteTime )) return true; - + if (datesSame( file.CreationTimeUtc, knownFile.CreationTime, file.LastWriteTimeUtc, knownFile.LastWriteTime )) return true; - + return false; bool datesSame(DateTime create1, DateTime create2, DateTime write1, DateTime write2) { - var createDiff = (int) Math.Abs(create1.Subtract(create2).TotalSeconds); + var createDiff = (int)Math.Abs(create1.Subtract(create2).TotalSeconds); var writeDiff = (int)Math.Abs(write1.Subtract(write2).TotalSeconds); - + bool create = createDiff < 5; bool write = writeDiff < 5; return create && write; @@ -791,9 +792,10 @@ private async Task CanAccess(FileInfo file, int fileSizeDetectionInterval) checkedAccess = true; - using (var fs = File.Open(file.FullName, FileMode.Open)) + //Since we don't actually write, we don't need to lock the file + using (var fs = FileOpenHelper.OpenForCheckingReadWriteAccess(file.FullName)) { - if(fs.CanRead == false) + if (fs.CanRead == false) { Logger.Instance.ILog("Cannot read file: " + file.FullName); return false; @@ -888,9 +890,9 @@ private bool QueuedHasItems() lock (QueuedFiles) { return QueuedFiles.Any(); - } + } } - + /// /// Safely adds an item to the queue /// @@ -902,10 +904,10 @@ public void QueueItem(string fullPath) Logger.Instance.DLog($"{Library.Name} file failed file detection: {fullPath}"); return; } - + lock (QueuedFiles) { - if(QueuedFiles.Contains(fullPath) == false) + if (QueuedFiles.Contains(fullPath) == false) QueuedFiles.Enqueue(fullPath); } diff --git a/ServerShared/FileServices/FileDownloader.cs b/ServerShared/FileServices/FileDownloader.cs index 1792e3714..feab2d515 100644 --- a/ServerShared/FileServices/FileDownloader.cs +++ b/ServerShared/FileServices/FileDownloader.cs @@ -113,7 +113,7 @@ public async Task> DownloadFile(string path, string destinationPath logger?.ILog("Content-Length header is not present in the response."); } - using FileStream fileStream = File.OpenWrite(destinationPath); + using FileStream fileStream = FileOpenHelper.OpenRead_NoWriteLock(destinationPath); int bufferSize; diff --git a/ServerShared/FileServices/FileHasher.cs b/ServerShared/FileServices/FileHasher.cs index e05a6598b..7be3e5852 100644 --- a/ServerShared/FileServices/FileHasher.cs +++ b/ServerShared/FileServices/FileHasher.cs @@ -19,21 +19,21 @@ public static async Task Hash(string filePath) var fileInfo = new FileInfo(filePath); if (!fileInfo.Exists) return string.Empty; - - using(var stream = new BufferedStream(File.OpenRead(filePath), 1200000)) + + using (var stream = new BufferedStream(FileOpenHelper.OpenRead_NoLocks(filePath), 1200000)) { // The rest remains the same var sha = SHA256.Create(); byte[] checksum = await sha.ComputeHashAsync(stream); return BitConverter.ToString(checksum).Replace("-", string.Empty); } - + using var hasher = SHA256.Create(); byte[]? hash; if (fileInfo.Length > 100_000_000) { - using var stream = File.OpenRead(filePath); + using var stream = FileOpenHelper.OpenRead_NoLocks(filePath); const int chunkSize = 100_000_000; // 100MB chunks var buffer = new byte[chunkSize]; @@ -51,7 +51,7 @@ public static async Task Hash(string filePath) } else { - await using var stream = File.OpenRead(filePath); + await using var stream = FileOpenHelper.OpenRead_NoLocks(filePath); hash = await hasher.ComputeHashAsync(stream); } diff --git a/ServerShared/FileServices/FileUploader.cs b/ServerShared/FileServices/FileUploader.cs index e1317cba6..a66f6498a 100644 --- a/ServerShared/FileServices/FileUploader.cs +++ b/ServerShared/FileServices/FileUploader.cs @@ -87,7 +87,7 @@ public async Task<(bool Success, string Error)> UploadFile(string filePath, stri url += "?path=" + HttpUtility.UrlEncode(serverPath); logger.ILog("Uploading file to URL: " + url); - await using var fileStream = File.OpenRead(filePath); + await using var fileStream = FileOpenHelper.OpenRead_NoLocks(filePath); var content = new StreamContent(fileStream); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, url); request.Headers.Add("x-executor", executorUid.ToString()); @@ -116,7 +116,7 @@ public async Task<(bool Success, string Error)> UploadFile(string filePath, stri /// The computed SHA256 hash of the file as a byte array. private static async Task CalculateFileHash(string filePath) { - await using var stream = File.OpenRead(filePath); + await using var stream = FileOpenHelper.OpenRead_NoLocks(filePath); return await SHA256.Create().ComputeHashAsync(stream); } diff --git a/ServerShared/Helpers/FileHelper.cs b/ServerShared/Helpers/FileHelper.cs index f42242b25..d5ea79c4d 100644 --- a/ServerShared/Helpers/FileHelper.cs +++ b/ServerShared/Helpers/FileHelper.cs @@ -7,7 +7,6 @@ namespace FileFlows.ServerShared.Helpers; /// public class FileHelper { - /// /// Removes illegal file/path characters from a string /// @@ -46,7 +45,7 @@ public static string CalculateFingerprint(string file) } else { - using var stream = File.OpenRead(file); + using var stream = FileOpenHelper.OpenRead_NoLocks(file); hash = hasher.ComputeHash(stream); } diff --git a/Shared/Helpers/FileOpenHelper.cs b/Shared/Helpers/FileOpenHelper.cs new file mode 100644 index 000000000..2b97c6699 --- /dev/null +++ b/Shared/Helpers/FileOpenHelper.cs @@ -0,0 +1,34 @@ +using System.IO; + +/// +/// Helper class for file open operations +/// +public static class FileOpenHelper +{ + /// + /// Opens a file for reading without a read/write lock + /// This allows other applications to read/write to the file while it is being read + /// + /// the file to open + /// the file stream + public static FileStream OpenRead_NoLocks(string file) + => File.Open(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + + /// + /// Opens a file for reading and writing without a read/write lock + /// This allows other applications to keep reading/writing the file while it's being opened + /// This is useful to check if CanRead/CanWrite is true on a file + /// Actually writing to the file with this stream might have undesired effects + /// + /// the file to open + /// the file stream + public static FileStream OpenForCheckingReadWriteAccess(string file) + => File.Open(file, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + + /// + /// Opens a file for writing without a write lock + /// This allows other applications to read the file while it is being written to + /// + public static FileStream OpenRead_NoWriteLock(string file) + => File.Open(file, FileMode.Create, FileAccess.Write, FileShare.Read); +} diff --git a/Shared/Helpers/HttpHelper.cs b/Shared/Helpers/HttpHelper.cs index 85df7db99..fc4c73951 100644 --- a/Shared/Helpers/HttpHelper.cs +++ b/Shared/Helpers/HttpHelper.cs @@ -10,6 +10,7 @@ namespace FileFlows.Shared.Helpers; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using FileFlows.Plugin.Helpers; using FileFlows.Shared.Models; /// @@ -143,7 +144,7 @@ public static async Task DownloadFile(string url, string destination) using HttpResponseMessage response = await Client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); await using var streamToReadFrom = await response.Content.ReadAsStreamAsync(); - await using var streamToWriteTo = File.Open(destination, FileMode.Create); + await using var streamToWriteTo = FileOpenHelper.OpenRead_NoWriteLock(destination); await streamToReadFrom.CopyToAsync(streamToWriteTo); } catch (Exception ex) From 2636083cd536ad7a27cdf1f8bb14625f7bf68a14 Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 17 Mar 2024 21:10:16 +0100 Subject: [PATCH 2/5] Undo formatting --- Server/Workers/WatchedLibrary.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Server/Workers/WatchedLibrary.cs b/Server/Workers/WatchedLibrary.cs index d2b65d4e7..88baf03ed 100644 --- a/Server/Workers/WatchedLibrary.cs +++ b/Server/Workers/WatchedLibrary.cs @@ -910,7 +910,7 @@ public void QueueItem(string fullPath) if (QueuedFiles.Contains(fullPath) == false) QueuedFiles.Enqueue(fullPath); } - + ProcessQueue(); } @@ -926,4 +926,4 @@ private bool QueueContains(string item) return QueuedFiles.Contains(item); } } -} +} \ No newline at end of file From bc893f726664e62384a67cb4a0b50cace5d509f1 Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 17 Mar 2024 21:15:00 +0100 Subject: [PATCH 3/5] Fixes --- Server/Workers/WatchedLibrary.cs | 3 +-- ServerShared/FileServices/FileDownloader.cs | 2 +- Shared/Helpers/FileOpenHelper.cs | 4 ++-- Shared/Helpers/HttpHelper.cs | 3 +-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Server/Workers/WatchedLibrary.cs b/Server/Workers/WatchedLibrary.cs index 88baf03ed..6383aaff7 100644 --- a/Server/Workers/WatchedLibrary.cs +++ b/Server/Workers/WatchedLibrary.cs @@ -792,7 +792,6 @@ private async Task CanAccess(FileInfo file, int fileSizeDetectionInterval) checkedAccess = true; - //Since we don't actually write, we don't need to lock the file using (var fs = FileOpenHelper.OpenForCheckingReadWriteAccess(file.FullName)) { if (fs.CanRead == false) @@ -910,7 +909,7 @@ public void QueueItem(string fullPath) if (QueuedFiles.Contains(fullPath) == false) QueuedFiles.Enqueue(fullPath); } - + ProcessQueue(); } diff --git a/ServerShared/FileServices/FileDownloader.cs b/ServerShared/FileServices/FileDownloader.cs index feab2d515..fee18e99c 100644 --- a/ServerShared/FileServices/FileDownloader.cs +++ b/ServerShared/FileServices/FileDownloader.cs @@ -113,7 +113,7 @@ public async Task> DownloadFile(string path, string destinationPath logger?.ILog("Content-Length header is not present in the response."); } - using FileStream fileStream = FileOpenHelper.OpenRead_NoWriteLock(destinationPath); + using FileStream fileStream = FileOpenHelper.OpenWrite_NoReadLock(destinationPath, FileMode.Create); int bufferSize; diff --git a/Shared/Helpers/FileOpenHelper.cs b/Shared/Helpers/FileOpenHelper.cs index 2b97c6699..463339013 100644 --- a/Shared/Helpers/FileOpenHelper.cs +++ b/Shared/Helpers/FileOpenHelper.cs @@ -29,6 +29,6 @@ public static FileStream OpenForCheckingReadWriteAccess(string file) /// Opens a file for writing without a write lock /// This allows other applications to read the file while it is being written to /// - public static FileStream OpenRead_NoWriteLock(string file) - => File.Open(file, FileMode.Create, FileAccess.Write, FileShare.Read); + public static FileStream OpenWrite_NoReadLock(string file, FileMode fileMode) + => File.Open(file, fileMode, FileAccess.Write, FileShare.Read); } diff --git a/Shared/Helpers/HttpHelper.cs b/Shared/Helpers/HttpHelper.cs index fc4c73951..d078329a8 100644 --- a/Shared/Helpers/HttpHelper.cs +++ b/Shared/Helpers/HttpHelper.cs @@ -10,7 +10,6 @@ namespace FileFlows.Shared.Helpers; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using FileFlows.Plugin.Helpers; using FileFlows.Shared.Models; /// @@ -144,7 +143,7 @@ public static async Task DownloadFile(string url, string destination) using HttpResponseMessage response = await Client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); await using var streamToReadFrom = await response.Content.ReadAsStreamAsync(); - await using var streamToWriteTo = FileOpenHelper.OpenRead_NoWriteLock(destination); + await using var streamToWriteTo = FileOpenHelper.OpenWrite_NoReadLock(destination, FileMode.Create); await streamToReadFrom.CopyToAsync(streamToWriteTo); } catch (Exception ex) From be9c62ba1ffd4b3d4f92c8650491b11204382d0a Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 17 Mar 2024 21:16:14 +0100 Subject: [PATCH 4/5] Fix --- Server/Workers/WatchedLibrary.cs | 95 ++++++++++++++++---------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/Server/Workers/WatchedLibrary.cs b/Server/Workers/WatchedLibrary.cs index 6383aaff7..c2da8cc34 100644 --- a/Server/Workers/WatchedLibrary.cs +++ b/Server/Workers/WatchedLibrary.cs @@ -14,12 +14,12 @@ namespace FileFlows.Server.Workers; /// /// A watched library is a folder that imports files into FileFlows /// -public class WatchedLibrary : IDisposable +public class WatchedLibrary:IDisposable { private FileSystemWatcher Watcher; - public Library Library { get; private set; } + public Library Library { get;private set; } - public bool ScanComplete { get; private set; } = false; + public bool ScanComplete { get;private set; } = false; /// /// Gets if the scanner should be used instead of file watched for this library /// @@ -48,10 +48,10 @@ public WatchedLibrary(Library library) this.UseScanner = true; } - if (UseScanner == false) + if(UseScanner == false) SetupWatcher(); - - + + QueueTimer = new(); QueueTimer.Elapsed += QueueTimerOnElapsed; //QueueTimer.AutoReset = false; @@ -69,7 +69,7 @@ private void LogQueueMessage(string message, Settings? settings = null) if (settings?.LogQueueMessages != true) return; - + Logger.Instance.DLog(message); } @@ -81,8 +81,8 @@ private void LogQueueMessage(string message, Settings? settings = null) private void QueueTimerOnElapsed(object? sender, ElapsedEventArgs e) => ProcessQueue(); - private SemaphoreSlim processorLock = new(1); - + private SemaphoreSlim processorLock= new (1); + /// /// Processes the queue /// @@ -115,7 +115,7 @@ private void Worker_DoWork(object? sender, DoWorkEventArgs e) while (Disposed == false) { ProcessQueuedItem(); - if (QueuedHasItems() != true) + if(QueuedHasItems() != true) { LogQueueMessage($"{Library.Name} nothing queued"); Thread.Sleep(1000); @@ -209,7 +209,7 @@ private void ProcessQueuedItem() if (Library.SkipFileAccessTests == false && Library.Folders == false && CanAccess((FileInfo)fsInfo, Library.FileSizeDetectionInterval).Result == false) { - Logger.Instance.WLog($"Failed access checks for file: " + fullpath + "\n" + + Logger.Instance.WLog($"Failed access checks for file: " + fullpath +"\n" + "These checks can be disabled in library settings, but ensure the flow can read and write to the library."); return; } @@ -254,7 +254,7 @@ private void ProcessQueuedItem() SystemEvents.TriggerFileAdded(result, Library); Logger.Instance.DLog( $"Time taken \"{(DateTime.UtcNow.Subtract(dtTotal))}\" to successfully add new library file: \"{fullpath}\""); - + if (new SettingsService().Get()?.Result?.ShowFileAddedNotifications == true) ClientServiceManager.Instance.SendToast(LogType.Info, "New File: " + result.RelativePath); } @@ -296,24 +296,24 @@ private bool MatchesDetection(string fullpath) /// true if matches detection, otherwise false public static bool MatchesDetection(Library library, FileSystemInfo info, long size) { - if (MatchesValue((int)DateTime.UtcNow.Subtract(info.CreationTimeUtc).TotalMinutes, library.DetectFileCreation, library.DetectFileCreationLower, library.DetectFileCreationUpper) == false) + if(MatchesValue((int)DateTime.UtcNow.Subtract(info.CreationTimeUtc).TotalMinutes, library.DetectFileCreation, library.DetectFileCreationLower, library.DetectFileCreationUpper) == false) return false; - if (MatchesValue((int)DateTime.UtcNow.Subtract(info.LastWriteTimeUtc).TotalMinutes, library.DetectFileLastWritten, library.DetectFileLastWrittenLower, library.DetectFileLastWrittenUpper) == false) + if(MatchesValue((int)DateTime.UtcNow.Subtract(info.LastWriteTimeUtc).TotalMinutes, library.DetectFileLastWritten, library.DetectFileLastWrittenLower, library.DetectFileLastWrittenUpper) == false) return false; - - if (MatchesValue(size, library.DetectFileSize, library.DetectFileSizeLower, library.DetectFileSizeUpper) == false) + + if(MatchesValue(size, library.DetectFileSize, library.DetectFileSizeLower, library.DetectFileSizeUpper) == false) return false; - + return true; - + } - + private static bool MatchesValue(long value, MatchRange range, long low, long high) { if (range == MatchRange.Any) return true; - + if (range == MatchRange.GreaterThan) return value > low; if (range == MatchRange.LessThan) @@ -453,7 +453,7 @@ private bool FileIsHidden(string fullpath) // recursively search the directories to see if its hidden var dir = new FileInfo(fullpath).Directory; int count = 0; - while (dir.Parent != null) + while(dir.Parent != null) { if (dir.Attributes.HasFlag(FileAttributes.Hidden)) return true; @@ -463,13 +463,13 @@ private bool FileIsHidden(string fullpath) } return false; } - + /// /// Disposes of the watched library /// public void Dispose() { - Disposed = true; + Disposed = true; DisposeWatcher(); //worker.Dispose(); //QueueTimer?.Dispose(); @@ -535,7 +535,7 @@ private bool IsMatch(string input) } catch (Exception) { } } - + if (string.IsNullOrWhiteSpace(Library.Filter) == false) { try @@ -582,7 +582,7 @@ private void Watcher_Changed(object sender, FileSystemEventArgs e) var file = new FileInfo(e.FullPath); if (file.Exists == false) return; - + long size = file.Length; Thread.Sleep(20_000); if (size < file.Length) @@ -600,7 +600,7 @@ private void Watcher_Changed(object sender, FileSystemEventArgs e) } private void FileChangeEvent(string fullPath) - { + { if (IsMatch(fullPath) == false) { if (fullPath.Contains("_UNPACK_")) @@ -628,17 +628,17 @@ internal void UpdateLibrary(Library library) UseScanner = true; SetupWatcher(); } - else if (UseScanner == false && library.Scan == true) + else if(UseScanner == false && library.Scan == true) { Logger.Instance.ILog($"WatchedLibrary: Library '{library.Name}' switched to scan mode, disposing watcher"); UseScanner = false; DisposeWatcher(); } - else if (UseScanner == false && Watcher != null && Watcher.Path != library.Path) + else if(UseScanner == false && Watcher != null && Watcher.Path != library.Path) { // library path changed, need to change watcher Logger.Instance.ILog($"WatchedLibrary: Library '{library.Name}' path changed, updating watched path"); - SetupWatcher(); + SetupWatcher(); } if (library.Enabled && library.LastScanned < new DateTime(2020, 1, 1) && Directory.Exists(library.Path)) @@ -672,9 +672,9 @@ public void Scan() Logger.Instance?.WLog($"WatchedLibrary: Library '{Library.Name}' path not found: {Library.Path}"); return; } - + Logger.Instance.ILog($"WatchedLibrary: Scan started on '{Library.Name}': {Library.Path}"); - + int count = 0; if (Library.Folders) { @@ -702,14 +702,13 @@ public void Scan() if (MatchesDetection(file.FullName) == false) continue; - + if (knownFiles.TryGetValue(file.FullName.ToLowerInvariant(), out KnownFileInfo info)) { if (Library.DownloadsDirectory && info.Status == FileStatus.Processed) { - - } - else if (DatesAreSame(file, info)) + + }else if (DatesAreSame(file, info)) continue; } @@ -725,13 +724,13 @@ public void Scan() Logger.Instance.ILog($"WatchedLibrary: Files queued for '{Library.Name}': {count} / {QueueCount()}"); ScanComplete = true; - + Library.LastScanned = DateTime.UtcNow; ServiceLoader.Load().UpdateLastScanned(Library.Uid).Wait(); } - catch (Exception ex) + catch(Exception ex) { - while (ex.InnerException != null) + while(ex.InnerException != null) ex = ex.InnerException; Logger.Instance.ELog("WatchedLibrary: Failed scanning for files: " + ex.Message + Environment.NewLine + ex.StackTrace); @@ -750,20 +749,20 @@ private bool DatesAreSame(FileInfo file, KnownFileInfo knownFile) file.LastWriteTime, knownFile.LastWriteTime )) return true; - + if (datesSame( file.CreationTimeUtc, knownFile.CreationTime, file.LastWriteTimeUtc, knownFile.LastWriteTime )) return true; - + return false; bool datesSame(DateTime create1, DateTime create2, DateTime write1, DateTime write2) { - var createDiff = (int)Math.Abs(create1.Subtract(create2).TotalSeconds); + var createDiff = (int) Math.Abs(create1.Subtract(create2).TotalSeconds); var writeDiff = (int)Math.Abs(write1.Subtract(write2).TotalSeconds); - + bool create = createDiff < 5; bool write = writeDiff < 5; return create && write; @@ -792,9 +791,9 @@ private async Task CanAccess(FileInfo file, int fileSizeDetectionInterval) checkedAccess = true; - using (var fs = FileOpenHelper.OpenForCheckingReadWriteAccess(file.FullName)) + using (var fs = FileOpenHelper.OpenRead_NoLocks(file.FullName)) { - if (fs.CanRead == false) + if(fs.CanRead == false) { Logger.Instance.ILog("Cannot read file: " + file.FullName); return false; @@ -889,9 +888,9 @@ private bool QueuedHasItems() lock (QueuedFiles) { return QueuedFiles.Any(); - } + } } - + /// /// Safely adds an item to the queue /// @@ -903,10 +902,10 @@ public void QueueItem(string fullPath) Logger.Instance.DLog($"{Library.Name} file failed file detection: {fullPath}"); return; } - + lock (QueuedFiles) { - if (QueuedFiles.Contains(fullPath) == false) + if(QueuedFiles.Contains(fullPath) == false) QueuedFiles.Enqueue(fullPath); } From a28e726fdb51d045fe7a22f38edef2324acef4cc Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 18 Mar 2024 10:29:30 +0100 Subject: [PATCH 5/5] Fixed finding --- Server/Workers/WatchedLibrary.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Server/Workers/WatchedLibrary.cs b/Server/Workers/WatchedLibrary.cs index c2da8cc34..16193a55f 100644 --- a/Server/Workers/WatchedLibrary.cs +++ b/Server/Workers/WatchedLibrary.cs @@ -791,7 +791,7 @@ private async Task CanAccess(FileInfo file, int fileSizeDetectionInterval) checkedAccess = true; - using (var fs = FileOpenHelper.OpenRead_NoLocks(file.FullName)) + using (var fs = FileOpenHelper.OpenForCheckingReadWriteAccess(file.FullName)) { if(fs.CanRead == false) {