-
Notifications
You must be signed in to change notification settings - Fork 46
/
CheckRavenDBIndexLag.cs
82 lines (69 loc) · 3.7 KB
/
CheckRavenDBIndexLag.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks
{
using System;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using NServiceBus.CustomChecks;
using NServiceBus.Logging;
using Raven.Client.Documents.Operations;
using ServiceControl.Audit.Persistence.RavenDB;
class CheckRavenDBIndexLag(IRavenDocumentStoreProvider documentStoreProvider) : CustomCheck("Audit Database Index Lag", "ServiceControl.Audit Health", TimeSpan.FromMinutes(5))
{
public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
{
var store = documentStoreProvider.GetDocumentStore();
var statistics = await store.Maintenance.SendAsync(new GetStatisticsOperation(), cancellationToken);
var indexes = statistics.Indexes.OrderBy(x => x.Name).ToArray();
CreateDiagnosticsLogEntry(statistics, indexes);
var indexCountWithTooMuchLag = CheckAndReportIndexesWithTooMuchIndexLag(indexes);
if (indexCountWithTooMuchLag > 0)
{
return CheckResult.Failed($"At least one index significantly stale. Please run maintenance mode if this custom check persists to ensure index(es) can recover. See log file for more details. Visit https://docs.particular.net/search?q=servicecontrol+troubleshooting for more information.");
}
return CheckResult.Pass;
}
static int CheckAndReportIndexesWithTooMuchIndexLag(IndexInformation[] indexes)
{
int indexCountWithTooMuchLag = 0;
foreach (var indexStats in indexes)
{
if (indexStats.IsStale && indexStats.LastIndexingTime.HasValue)
{
var indexLag = DateTime.UtcNow - indexStats.LastIndexingTime.Value;
if (indexLag > IndexLagThresholdError)
{
indexCountWithTooMuchLag++;
Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag} is above error threshold ({IndexLagThresholdError}). Launch in maintenance mode to let indexes catch up.");
}
else if (indexLag > IndexLagThresholdWarning)
{
indexCountWithTooMuchLag++;
Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag} is above warning threshold ({IndexLagThresholdWarning}). Launch in maintenance mode to let indexes catch up.");
}
}
}
return indexCountWithTooMuchLag;
}
static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexInformation[] indexes)
{
if (!Log.IsDebugEnabled)
{
return;
}
var report = new StringBuilder();
report.AppendLine("Internal RavenDB index health report:");
report.AppendLine($"- DB Size: {statistics.SizeOnDisk.HumaneSize}");
report.AppendLine($"- LastIndexingTime {statistics.LastIndexingTime:u}");
foreach (var indexStats in indexes)
{
report.AppendLine($"- Index [{indexStats.Name,-44}] State: {indexStats.State}, Stale: {indexStats.IsStale,-5}, Priority: {indexStats.Priority,-6}, LastIndexingTime: {indexStats.LastIndexingTime:u}");
}
Log.Debug(report.ToString());
}
static readonly TimeSpan IndexLagThresholdWarning = TimeSpan.FromMinutes(1);
static readonly TimeSpan IndexLagThresholdError = TimeSpan.FromMinutes(10);
static readonly ILog Log = LogManager.GetLogger<CheckRavenDBIndexLag>();
}
}