CheckFreeSpace CustomCheck moved into persister and some aligment (#4026

)

* Align MinimumStorageRequiredForIngestion

* Move CheckFreeDiskSpace into persistence layer

* Align custom checks for ingestion and free space with main instance plus get rid of double validation (things are still messy but better than before)

* Keep sane thresholds

* Fix custom check tests by resolving them from the container and introduce the same test in the main instance

* Change to direct service provider usage instead

* Aligh custom checks and make sure they pass for external ravendb

* Further small alignment

* Formatting

* Remove unneeded default setting in app.config

* Use database path in exception messages

* Register audit custom checks manually in tests only to avoid double registration

* Adjust custom check to make it work with ATT hacks

* Cleanup and unify implementations

---------

Co-authored-by: danielmarbach <danielmarbach@users.noreply.github.com>
Co-authored-by: Brandon Ording <bording@gmail.com>

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckFreeDiskSpace.cs

@@ -0,0 +1,85 @@
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using NServiceBus.CustomChecks;
+    using NServiceBus.Logging;
+    using RavenDB;
+
+    class CheckFreeDiskSpace(DatabaseConfiguration databaseConfiguration) : CustomCheck("ServiceControl.Audit database", "Storage space", TimeSpan.FromMinutes(5))
+    {
+        public override Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+        {
+            if (Logger.IsDebugEnabled)
+            {
+                Logger.Debug($"Check ServiceControl data drive space remaining custom check starting. Threshold {percentageThreshold:P0}");
+            }
+
+            if (!databaseConfiguration.ServerConfiguration.UseEmbeddedServer)
+            {
+                return CheckResult.Pass;
+            }
+
+            if (dataPathRoot is null)
+            {
+                throw new Exception($"Unable to find the root of the data path {databaseConfiguration.ServerConfiguration.DbPath}");
+            }
+
+            var dataDriveInfo = new DriveInfo(dataPathRoot);
+            var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace;
+            var totalSpace = (decimal)dataDriveInfo.TotalSize;
+
+            var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize;
+
+            if (Logger.IsDebugEnabled)
+            {
+                Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}");
+            }
+
+            return percentRemaining > percentageThreshold
+                ? CheckResult.Pass
+                : CheckResult.Failed($"{percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'.");
+        }
+
+        public static int Parse(IDictionary<string, string> settings)
+        {
+            if (!settings.TryGetValue(RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey, out var thresholdValue))
+            {
+                thresholdValue = $"{DataSpaceRemainingThresholdDefault}";
+            }
+
+            string message;
+            if (!int.TryParse(thresholdValue, out var threshold))
+            {
+                message = $"{RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey} must be an integer.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            if (threshold < 0)
+            {
+                message = $"{RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, minimum value is 0.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            if (threshold > 100)
+            {
+                message = $"{RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, maximum value is 100.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            return threshold;
+        }
+
+        readonly string dataPathRoot = Path.GetPathRoot(databaseConfiguration.ServerConfiguration.DbPath);
+        readonly decimal percentageThreshold = databaseConfiguration.DataSpaceRemainingThreshold / 100m;
+
+        public const int DataSpaceRemainingThresholdDefault = 20;
+        static readonly ILog Logger = LogManager.GetLogger(typeof(CheckFreeDiskSpace));
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs

@@ -0,0 +1,90 @@
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using NServiceBus.CustomChecks;
+    using NServiceBus.Logging;
+    using RavenDB;
+
+    class CheckMinimumStorageRequiredForIngestion(MinimumRequiredStorageState stateHolder, DatabaseConfiguration databaseConfiguration) : CustomCheck("Audit Message Ingestion Process", "ServiceControl.Audit Health", TimeSpan.FromSeconds(5))
+    {
+        public override Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+        {
+            var percentageThreshold = databaseConfiguration.MinimumStorageLeftRequiredForIngestion / 100m;
+
+            if (Logger.IsDebugEnabled)
+            {
+                Logger.Debug($"Check ServiceControl data drive space starting. Threshold {percentageThreshold:P0}");
+            }
+
+            // Should be checking UseEmbeddedServer but need to check DbPath instead for the ATT hack to work
+            if (string.IsNullOrEmpty(databaseConfiguration.ServerConfiguration.DbPath))
+            {
+                stateHolder.CanIngestMore = true;
+                return SuccessResult;
+            }
+
+            var dataPathRoot = Path.GetPathRoot(databaseConfiguration.ServerConfiguration.DbPath) ?? throw new Exception($"Unable to find the root of the data path {databaseConfiguration.ServerConfiguration.DbPath}");
+
+            var dataDriveInfo = new DriveInfo(dataPathRoot);
+            var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace;
+            var totalSpace = (decimal)dataDriveInfo.TotalSize;
+
+            var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize;
+
+            if (Logger.IsDebugEnabled)
+            {
+                Logger.Debug($"Free space: {availableFreeSpace} | Total: {totalSpace} | Percent remaining {percentRemaining:P0}");
+            }
+
+            if (percentRemaining > percentageThreshold)
+            {
+                stateHolder.CanIngestMore = true;
+                return SuccessResult;
+            }
+
+            var message = $"Audit message ingestion stopped! {percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'. This is less than {percentageThreshold}% - the minimal required space configured. The threshold can be set using the {RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} configuration setting.";
+            Logger.Warn(message);
+            stateHolder.CanIngestMore = false;
+            return CheckResult.Failed(message);
+        }
+
+        public static int Parse(IDictionary<string, string> settings)
+        {
+            if (!settings.TryGetValue(RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey, out var thresholdValue))
+            {
+                thresholdValue = $"{MinimumStorageLeftRequiredForIngestionDefault}";
+            }
+
+            if (!int.TryParse(thresholdValue, out var threshold))
+            {
+                var message = $"{RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} must be an integer.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            if (threshold < 0)
+            {
+                var message = $"{RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} is invalid, minimum value is 0.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            if (threshold > 100)
+            {
+                var message = $"{RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} is invalid, maximum value is 100.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            return threshold;
+        }
+
+        public const int MinimumStorageLeftRequiredForIngestionDefault = 5;
+        static readonly Task<CheckResult> SuccessResult = Task.FromResult(CheckResult.Pass);
+        static readonly ILog Logger = LogManager.GetLogger(typeof(CheckMinimumStorageRequiredForIngestion));
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckRavenDBIndexLag.cs

@@ -1,23 +1,17 @@
-namespace ServiceControl
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
 {
     using System;
     using System.Linq;
     using System.Text;
     using System.Threading;
     using System.Threading.Tasks;
-    using Audit.Persistence.RavenDB;
     using NServiceBus.CustomChecks;
     using NServiceBus.Logging;
     using Raven.Client.Documents.Operations;
+    using ServiceControl.Audit.Persistence.RavenDB;
 
-    class CheckRavenDBIndexLag : CustomCheck
+    class CheckRavenDBIndexLag(IRavenDocumentStoreProvider documentStoreProvider) : CustomCheck("Audit Database Index Lag", "ServiceControl.Audit Health", TimeSpan.FromMinutes(5))
     {
-        public CheckRavenDBIndexLag(IRavenDocumentStoreProvider documentStoreProvider)
-            : base("Audit Database Index Lag", "ServiceControl.Audit Health", TimeSpan.FromMinutes(5))
-        {
-            this.documentStoreProvider = documentStoreProvider;
-        }
-
         public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
         {
             var store = documentStoreProvider.GetDocumentStore();
@@ -84,7 +78,5 @@ static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexInform
         static readonly TimeSpan IndexLagThresholdWarning = TimeSpan.FromMinutes(1);
         static readonly TimeSpan IndexLagThresholdError = TimeSpan.FromMinutes(10);
         static readonly ILog Log = LogManager.GetLogger<CheckRavenDBIndexLag>();
-
-        readonly IRavenDocumentStoreProvider documentStoreProvider;
     }
 }

src/ServiceControl.Audit.Persistence.RavenDB/DatabaseConfiguration.cs

@@ -3,43 +3,35 @@
     using System;
     using Sparrow.Json;
 
-    public class DatabaseConfiguration
+    public class DatabaseConfiguration(
+        string name,
+        int expirationProcessTimerInSeconds,
+        bool enableFullTextSearch,
+        TimeSpan auditRetentionPeriod,
+        int maxBodySizeToStore,
+        int dataSpaceRemainingThreshold,
+        int minimumStorageLeftRequiredForIngestion,
+        ServerConfiguration serverConfiguration,
+        TimeSpan bulkInsertCommitTimeout)
     {
-        public DatabaseConfiguration(string name,
-            int expirationProcessTimerInSeconds,
-            bool enableFullTextSearch,
-            TimeSpan auditRetentionPeriod,
-            int maxBodySizeToStore,
-            int minimumStorageLeftRequiredForIngestion,
-            ServerConfiguration serverConfiguration,
-            TimeSpan bulkInsertCommitTimeout)
-        {
-            Name = name;
-            ExpirationProcessTimerInSeconds = expirationProcessTimerInSeconds;
-            EnableFullTextSearch = enableFullTextSearch;
-            AuditRetentionPeriod = auditRetentionPeriod;
-            MaxBodySizeToStore = maxBodySizeToStore;
-            ServerConfiguration = serverConfiguration;
-            BulkInsertCommitTimeout = bulkInsertCommitTimeout;
-            MinimumStorageLeftRequiredForIngestion = minimumStorageLeftRequiredForIngestion;
-        }
-
-        public string Name { get; }
-
-        public int ExpirationProcessTimerInSeconds { get; }
-
-        public bool EnableFullTextSearch { get; }
+        public string Name { get; } = name;
+
+        public int ExpirationProcessTimerInSeconds { get; } = expirationProcessTimerInSeconds;
+
+        public bool EnableFullTextSearch { get; } = enableFullTextSearch;
 
         public Func<string, BlittableJsonReaderObject, string> FindClrType { get; }
 
-        public ServerConfiguration ServerConfiguration { get; }
+        public ServerConfiguration ServerConfiguration { get; } = serverConfiguration;
+
+        public TimeSpan AuditRetentionPeriod { get; } = auditRetentionPeriod;
 
-        public TimeSpan AuditRetentionPeriod { get; }
+        public int MaxBodySizeToStore { get; } = maxBodySizeToStore;
 
-        public int MaxBodySizeToStore { get; }
+        public int DataSpaceRemainingThreshold { get; } = dataSpaceRemainingThreshold;
 
-        public int MinimumStorageLeftRequiredForIngestion { get; internal set; } //Setting for ATT only
+        public int MinimumStorageLeftRequiredForIngestion { get; internal set; } = minimumStorageLeftRequiredForIngestion; //Setting for ATT only
 
-        public TimeSpan BulkInsertCommitTimeout { get; }
+        public TimeSpan BulkInsertCommitTimeout { get; } = bulkInsertCommitTimeout;
     }
 }

src/ServiceControl.Audit.Persistence.RavenDB/MinimumRequiredStorageState.cs

@@ -0,0 +1,7 @@
+namespace ServiceControl.Audit.Persistence.RavenDB
+{
+    public class MinimumRequiredStorageState
+    {
+        public bool CanIngestMore { get; set; } = true;
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs

@@ -2,7 +2,6 @@
 {
     using Microsoft.Extensions.DependencyInjection;
     using Persistence.UnitOfWork;
-    using RavenDB.CustomChecks;
     using UnitOfWork;
 
     class RavenPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence
@@ -14,7 +13,7 @@ public void AddPersistence(IServiceCollection services)
             services.AddSingleton<IAuditDataStore, RavenAuditDataStore>();
             services.AddSingleton<IAuditIngestionUnitOfWorkFactory, RavenAuditIngestionUnitOfWorkFactory>();
             services.AddSingleton<IFailedAuditStorage, RavenFailedAuditStorage>();
-            services.AddSingleton<CheckMinimumStorageRequiredForAuditIngestion.State>();
+            services.AddSingleton<MinimumRequiredStorageState>();
         }
 
         public void AddInstaller(IServiceCollection services) => ConfigureLifecycle(services, databaseConfiguration);