Skip to content

Commit

Permalink
Cherry pick #11309 Raise visibility of gray failure actions (#11314)
Browse files Browse the repository at this point in the history
* Cherry pick #11308 Raise visibility of gray failure actions

* format change

---------

Co-authored-by: Dan Lambright <hlambright@apple.com>
  • Loading branch information
dlambrig and Dan Lambright committed Apr 19, 2024
1 parent b6e9433 commit 684422e
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions fdbserver/ClusterController.actor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2613,19 +2613,19 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
self->excludedDegradedServers = self->degradationInfo.degradedServers;
self->excludedDegradedServers.insert(self->degradationInfo.disconnectedServers.begin(),
self->degradationInfo.disconnectedServers.end());
TraceEvent("DegradedServerDetectedAndTriggerRecovery")
TraceEvent(SevWarnAlways, "DegradedServerDetectedAndTriggerRecovery")
.detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth());
self->db.forceMasterFailure.trigger();
}
} else {
self->excludedDegradedServers.clear();
TraceEvent("DegradedServerDetectedAndSuggestRecovery").log();
TraceEvent(SevWarnAlways, "DegradedServerDetectedAndSuggestRecovery").log();
}
} else if (self->shouldTriggerFailoverDueToDegradedServers()) {
double ccUpTime = now() - machineStartTime();
if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER &&
ccUpTime > SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
TraceEvent("DegradedServerDetectedAndTriggerFailover").log();
TraceEvent(SevWarnAlways, "DegradedServerDetectedAndTriggerFailover").log();
std::vector<Optional<Key>> dcPriority;
auto remoteDcId = self->db.config.regions[0].dcId == self->clusterControllerDcId.get()
? self->db.config.regions[1].dcId
Expand All @@ -2637,7 +2637,8 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
dcPriority.push_back(self->clusterControllerDcId);
self->desiredDcIds.set(dcPriority);
} else {
TraceEvent("DegradedServerDetectedAndSuggestFailover").detail("CCUpTime", ccUpTime);
TraceEvent(SevWarnAlways, "DegradedServerDetectedAndSuggestFailover")
.detail("CCUpTime", ccUpTime);
}
}
}
Expand Down

0 comments on commit 684422e

Please sign in to comment.