Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CheckFreeSpace CustomCheck moved into persister and some aligment #4026

Merged
merged 14 commits into from Apr 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -0,0 +1,85 @@
namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using NServiceBus.CustomChecks;
using NServiceBus.Logging;
using RavenDB;

class CheckFreeDiskSpace(DatabaseConfiguration databaseConfiguration) : CustomCheck("ServiceControl.Audit database", "Storage space", TimeSpan.FromMinutes(5))
{
public override Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug($"Check ServiceControl data drive space remaining custom check starting. Threshold {percentageThreshold:P0}");
}

if (!databaseConfiguration.ServerConfiguration.UseEmbeddedServer)
{
return CheckResult.Pass;
}

if (dataPathRoot is null)
{
throw new Exception($"Unable to find the root of the data path {databaseConfiguration.ServerConfiguration.DbPath}");
}

var dataDriveInfo = new DriveInfo(dataPathRoot);
var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace;
var totalSpace = (decimal)dataDriveInfo.TotalSize;

var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize;

if (Logger.IsDebugEnabled)
{
Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}");
}

return percentRemaining > percentageThreshold
? CheckResult.Pass
: CheckResult.Failed($"{percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'.");
}

public static int Parse(IDictionary<string, string> settings)
{
if (!settings.TryGetValue(RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey, out var thresholdValue))
{
thresholdValue = $"{DataSpaceRemainingThresholdDefault}";
}

string message;
if (!int.TryParse(thresholdValue, out var threshold))
{
message = $"{RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey} must be an integer.";
Logger.Fatal(message);
throw new Exception(message);
}

if (threshold < 0)
{
message = $"{RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, minimum value is 0.";
Logger.Fatal(message);
throw new Exception(message);
}

if (threshold > 100)
{
message = $"{RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, maximum value is 100.";
Logger.Fatal(message);
throw new Exception(message);
}

return threshold;
}

readonly string dataPathRoot = Path.GetPathRoot(databaseConfiguration.ServerConfiguration.DbPath);
readonly decimal percentageThreshold = databaseConfiguration.DataSpaceRemainingThreshold / 100m;

public const int DataSpaceRemainingThresholdDefault = 20;
static readonly ILog Logger = LogManager.GetLogger(typeof(CheckFreeDiskSpace));
}
}

This file was deleted.

@@ -0,0 +1,90 @@
namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using NServiceBus.CustomChecks;
using NServiceBus.Logging;
using RavenDB;

class CheckMinimumStorageRequiredForIngestion(MinimumRequiredStorageState stateHolder, DatabaseConfiguration databaseConfiguration) : CustomCheck("Audit Message Ingestion Process", "ServiceControl.Audit Health", TimeSpan.FromSeconds(5))
{
public override Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
{
var percentageThreshold = databaseConfiguration.MinimumStorageLeftRequiredForIngestion / 100m;

if (Logger.IsDebugEnabled)
{
Logger.Debug($"Check ServiceControl data drive space starting. Threshold {percentageThreshold:P0}");
}

// Should be checking UseEmbeddedServer but need to check DbPath instead for the ATT hack to work
if (string.IsNullOrEmpty(databaseConfiguration.ServerConfiguration.DbPath))
{
stateHolder.CanIngestMore = true;
return SuccessResult;
}

var dataPathRoot = Path.GetPathRoot(databaseConfiguration.ServerConfiguration.DbPath) ?? throw new Exception($"Unable to find the root of the data path {databaseConfiguration.ServerConfiguration.DbPath}");

var dataDriveInfo = new DriveInfo(dataPathRoot);
var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace;
var totalSpace = (decimal)dataDriveInfo.TotalSize;

var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize;

if (Logger.IsDebugEnabled)
{
Logger.Debug($"Free space: {availableFreeSpace} | Total: {totalSpace} | Percent remaining {percentRemaining:P0}");
}

if (percentRemaining > percentageThreshold)
{
stateHolder.CanIngestMore = true;
return SuccessResult;
}

var message = $"Audit message ingestion stopped! {percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'. This is less than {percentageThreshold}% - the minimal required space configured. The threshold can be set using the {RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} configuration setting.";
Logger.Warn(message);
stateHolder.CanIngestMore = false;
return CheckResult.Failed(message);
}

public static int Parse(IDictionary<string, string> settings)
{
if (!settings.TryGetValue(RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey, out var thresholdValue))
{
thresholdValue = $"{MinimumStorageLeftRequiredForIngestionDefault}";
}

if (!int.TryParse(thresholdValue, out var threshold))
{
var message = $"{RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} must be an integer.";
Logger.Fatal(message);
throw new Exception(message);
}

if (threshold < 0)
{
var message = $"{RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} is invalid, minimum value is 0.";
Logger.Fatal(message);
throw new Exception(message);
}

if (threshold > 100)
{
var message = $"{RavenPersistenceConfiguration.MinimumStorageLeftRequiredForIngestionKey} is invalid, maximum value is 100.";
Logger.Fatal(message);
throw new Exception(message);
}

return threshold;
}

public const int MinimumStorageLeftRequiredForIngestionDefault = 5;
static readonly Task<CheckResult> SuccessResult = Task.FromResult(CheckResult.Pass);
static readonly ILog Logger = LogManager.GetLogger(typeof(CheckMinimumStorageRequiredForIngestion));
}
}
@@ -1,23 +1,17 @@
namespace ServiceControl
namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
{
using System;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Audit.Persistence.RavenDB;
using NServiceBus.CustomChecks;
using NServiceBus.Logging;
using Raven.Client.Documents.Operations;
using ServiceControl.Audit.Persistence.RavenDB;

class CheckRavenDBIndexLag : CustomCheck
class CheckRavenDBIndexLag(IRavenDocumentStoreProvider documentStoreProvider) : CustomCheck("Audit Database Index Lag", "ServiceControl.Audit Health", TimeSpan.FromMinutes(5))
{
public CheckRavenDBIndexLag(IRavenDocumentStoreProvider documentStoreProvider)
: base("Audit Database Index Lag", "ServiceControl.Audit Health", TimeSpan.FromMinutes(5))
{
this.documentStoreProvider = documentStoreProvider;
}

public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
{
var store = documentStoreProvider.GetDocumentStore();
Expand Down Expand Up @@ -84,7 +78,5 @@ static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexInform
static readonly TimeSpan IndexLagThresholdWarning = TimeSpan.FromMinutes(1);
static readonly TimeSpan IndexLagThresholdError = TimeSpan.FromMinutes(10);
static readonly ILog Log = LogManager.GetLogger<CheckRavenDBIndexLag>();

readonly IRavenDocumentStoreProvider documentStoreProvider;
}
}
Expand Up @@ -3,43 +3,35 @@
using System;
using Sparrow.Json;

public class DatabaseConfiguration
public class DatabaseConfiguration(
string name,
int expirationProcessTimerInSeconds,
bool enableFullTextSearch,
TimeSpan auditRetentionPeriod,
int maxBodySizeToStore,
int dataSpaceRemainingThreshold,
int minimumStorageLeftRequiredForIngestion,
ServerConfiguration serverConfiguration,
TimeSpan bulkInsertCommitTimeout)
{
public DatabaseConfiguration(string name,
int expirationProcessTimerInSeconds,
bool enableFullTextSearch,
TimeSpan auditRetentionPeriod,
int maxBodySizeToStore,
int minimumStorageLeftRequiredForIngestion,
ServerConfiguration serverConfiguration,
TimeSpan bulkInsertCommitTimeout)
{
Name = name;
ExpirationProcessTimerInSeconds = expirationProcessTimerInSeconds;
EnableFullTextSearch = enableFullTextSearch;
AuditRetentionPeriod = auditRetentionPeriod;
MaxBodySizeToStore = maxBodySizeToStore;
ServerConfiguration = serverConfiguration;
BulkInsertCommitTimeout = bulkInsertCommitTimeout;
MinimumStorageLeftRequiredForIngestion = minimumStorageLeftRequiredForIngestion;
}

public string Name { get; }

public int ExpirationProcessTimerInSeconds { get; }

public bool EnableFullTextSearch { get; }
public string Name { get; } = name;

public int ExpirationProcessTimerInSeconds { get; } = expirationProcessTimerInSeconds;

public bool EnableFullTextSearch { get; } = enableFullTextSearch;

public Func<string, BlittableJsonReaderObject, string> FindClrType { get; }

public ServerConfiguration ServerConfiguration { get; }
public ServerConfiguration ServerConfiguration { get; } = serverConfiguration;

public TimeSpan AuditRetentionPeriod { get; } = auditRetentionPeriod;

public TimeSpan AuditRetentionPeriod { get; }
public int MaxBodySizeToStore { get; } = maxBodySizeToStore;

public int MaxBodySizeToStore { get; }
public int DataSpaceRemainingThreshold { get; } = dataSpaceRemainingThreshold;

public int MinimumStorageLeftRequiredForIngestion { get; internal set; } //Setting for ATT only
public int MinimumStorageLeftRequiredForIngestion { get; internal set; } = minimumStorageLeftRequiredForIngestion; //Setting for ATT only

public TimeSpan BulkInsertCommitTimeout { get; }
public TimeSpan BulkInsertCommitTimeout { get; } = bulkInsertCommitTimeout;
}
}
@@ -0,0 +1,7 @@
namespace ServiceControl.Audit.Persistence.RavenDB
{
public class MinimumRequiredStorageState
{
public bool CanIngestMore { get; set; } = true;
}
}
Expand Up @@ -2,7 +2,6 @@
{
using Microsoft.Extensions.DependencyInjection;
using Persistence.UnitOfWork;
using RavenDB.CustomChecks;
using UnitOfWork;

class RavenPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence
Expand All @@ -14,7 +13,7 @@ public void AddPersistence(IServiceCollection services)
services.AddSingleton<IAuditDataStore, RavenAuditDataStore>();
services.AddSingleton<IAuditIngestionUnitOfWorkFactory, RavenAuditIngestionUnitOfWorkFactory>();
services.AddSingleton<IFailedAuditStorage, RavenFailedAuditStorage>();
services.AddSingleton<CheckMinimumStorageRequiredForAuditIngestion.State>();
services.AddSingleton<MinimumRequiredStorageState>();
}

public void AddInstaller(IServiceCollection services) => ConfigureLifecycle(services, databaseConfiguration);
Expand Down