Skip to content
This repository has been archived by the owner on Feb 12, 2022. It is now read-only.

Commit

Permalink
Merge pull request #797 from sundeepsf/develop
Browse files Browse the repository at this point in the history 
 Publishing metrics to better track alert evaluation/failure and notification trigger status. Added more logging for exceptions
  • Loading branch information
@sundeepsf
sundeepsf committed Jun 29, 2018
2 parents 5cce24d + 480c113 commit d8d3211
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 114 deletions.
32 changes: 20 additions & 12 deletions ArgusCore/src/main/java/com/salesforce/dva/argus/service/alert/DefaultAlertService.java
View file
Expand Up @@ -59,6 +59,7 @@
import com.salesforce.dva.argus.system.SystemConfiguration;
import com.salesforce.dva.argus.util.Cron;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -458,39 +459,32 @@ public List<History> executeScheduledAlerts(int alertCount, int timeout) {
long evalLatency = jobEndTime - jobStartTime;
_appendMessageNUpdateHistory(history, "Alert was evaluated successfully.", JobStatus.SUCCESS, evalLatency);

// publishing evaluation latency as a metric
Map<Long, Double> datapoints = new HashMap<>();
datapoints.put(1000 * 60 * (System.currentTimeMillis()/(1000 *60)), Double.valueOf(evalLatency));
Metric metric = new Metric("alerts.evaluated", "alert-evaluation-latency-" + alert.getId().toString());
metric.addDatapoints(datapoints);
try {
_tsdbService.putMetrics(Arrays.asList(new Metric[] {metric}));
} catch (Exception ex) {
_logger.error("Exception occurred while pushing alert evaluation latency metric to tsdb - {}", ex.getMessage());
}
publishAlertTrackingMetric(Counter.ALERTS_EVALUATED.getMetric(), alert.getId(), 1.0/*success*/);
Map<String, String> tags = new HashMap<>();
tags.put(USERTAG, alert.getOwner().getUserName());
_monitorService.modifyCounter(Counter.ALERTS_EVALUATION_LATENCY, evalLatency, tags);
} catch (MissingDataException mde) {
jobEndTime = System.currentTimeMillis();
logMessage = MessageFormat.format("Failed to evaluate alert : {0}. Reason: {1}", alert.getId().intValue(), mde.getMessage());
logMessage = MessageFormat.format("Failed to evaluate alert : {0} due to missing data exception. Full stack trace of exception - {1}", alert.getId().intValue(), ExceptionUtils.getFullStackTrace(mde));
_logger.warn(logMessage);
_appendMessageNUpdateHistory(history, logMessage, JobStatus.FAILURE, jobEndTime - jobStartTime);
if (alert.isMissingDataNotificationEnabled()) {
_sendNotificationForMissingData(alert);
}
publishAlertTrackingMetric(Counter.ALERTS_EVALUATED.getMetric(), alert.getId(), -1.0/*failure*/);
Map<String, String> tags = new HashMap<>();
tags.put(USERTAG, alert.getOwner().getUserName());
_monitorService.modifyCounter(Counter.ALERTS_FAILED, 1, tags);
} catch (Exception ex) {
jobEndTime = System.currentTimeMillis();
logMessage = MessageFormat.format("Failed to evaluate alert : {0}. Reason: {1}", alert.getId().intValue(), ex.getMessage());
logMessage = MessageFormat.format("Failed to evaluate alert : {0} due to an exception. Full stack trace of exception - {1}", alert.getId().intValue(), ExceptionUtils.getFullStackTrace(ex));
_logger.warn(logMessage);
_appendMessageNUpdateHistory(history, logMessage, JobStatus.FAILURE, jobEndTime - jobStartTime);

if (Boolean.valueOf(_configuration.getValue(SystemConfiguration.Property.EMAIL_EXCEPTIONS))) {
_sendEmailToAdmin(alert, alert.getId(), ex);
}
publishAlertTrackingMetric(Counter.ALERTS_EVALUATED.getMetric(), alert.getId(), -1.0/*failure*/);
Map<String, String> tags = new HashMap<>();
tags.put(USERTAG, alert.getOwner().getUserName());
_monitorService.modifyCounter(Counter.ALERTS_FAILED, 1, tags);
Expand Down Expand Up @@ -602,6 +596,7 @@ public void sendNotification(Trigger trigger, Metric metric, History history, No
tags.put("status", "active");
tags.put("type", SupportedNotifier.fromClassName(notification.getNotifierName()).name());
_monitorService.modifyCounter(Counter.NOTIFICATIONS_SENT, 1, tags);
publishAlertTrackingMetric(Counter.NOTIFICATIONS_SENT.getMetric(), trigger.getAlert().getId(), 1.0/*notification sent*/);

String logMessage = MessageFormat.format("Sent alert notification and updated the cooldown: {0}",
getDateMMDDYYYY(notification.getCooldownExpirationByTriggerAndMetric(trigger, metric)));
Expand All @@ -619,11 +614,24 @@ public void sendClearNotification(Trigger trigger, Metric metric, History histor
tags.put("status", "clear");
tags.put("type", SupportedNotifier.fromClassName(notification.getNotifierName()).name());
_monitorService.modifyCounter(Counter.NOTIFICATIONS_SENT, 1, tags);
publishAlertTrackingMetric(Counter.NOTIFICATIONS_SENT.getMetric(), trigger.getAlert().getId(), -1.0/*notification cleared*/);

String logMessage = MessageFormat.format("The notification {0} was cleared.", notification.getName());
_logger.info(logMessage);
_appendMessageNUpdateHistory(history, logMessage, null, 0);
}

private void publishAlertTrackingMetric(String scope, BigInteger alertId, double value) {
Map<Long, Double> datapoints = new HashMap<>();
datapoints.put(1000 * 60 * (System.currentTimeMillis()/(1000 *60)), value);
Metric trackingMetric = new Metric(scope, "alert-" + alertId.intValue());
trackingMetric.addDatapoints(datapoints);
try {
_tsdbService.putMetrics(Arrays.asList(new Metric[] {trackingMetric}));
} catch (Exception ex) {
_logger.error("Exception occurred while adding alerts evaluated metric to tsdb - {}", ex.getMessage());
}
}

private void _updateNotificationSetActiveStatus(Trigger trigger, Metric metric, History history, Notification notification) {
notification.setCooldownExpirationByTriggerAndMetric(trigger, metric, System.currentTimeMillis() + notification.getCooldownPeriod());
Expand Down
212 changes: 110 additions & 102 deletions ArgusCore/src/main/java/com/salesforce/dva/argus/util/Cron.java
View file
Expand Up @@ -28,7 +28,7 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

package com.salesforce.dva.argus.util;

import java.util.Date;
Expand All @@ -44,103 +44,103 @@
*/
public class Cron {

//~ Static fields/initializers *******************************************************************************************************************

private static final String ANNUALLY = "@ANNUALLY";
private static final String YEARLY = "@YEARLY";
private static final String MONTHLY = "@MONTHLY";
private static final String WEEKLY = "@WEEKLY";
private static final String DAILY = "@DAILY";
private static final String MIDNIGHT = "@MIDNIGHT";
private static final String HOURLY = "@HOURLY";

//~ Constructors *********************************************************************************************************************************

private Cron() { }

//~ Methods **************************************************************************************************************************************

/**
* Determines if the given CRON entry is runnable at this current moment in time. This mimics the original implementation of the CRON table.
*
* <p>This implementation supports only the following types of entries:</p>
*
* <ol>
* <li>Standard Entries having form: &lt;minutes&gt; &lt;hours&gt; &lt;days&gt; &lt;months&gt; &lt;days of week&gt;
*
* <ul>
* <li>* : All</li>
* <li>*\/n : Only mod n</li>
* <li>n : Numeric</li>
* <li>n-n : Range</li>
* <li>n,n,...,n : List</li>
* <li>n,n-n,...,n : List having ranges</li>
* </ul>
* </li>
* <li>Special Entries
*
* <ul>
* <li>@annually : equivalent to "0 0 1 1 *"</li>
* <li>@yearly : equivalent to "0 0 1 1 *"</li>
* <li>@monthly : equivalent to "0 0 1 * *"</li>
* <li>@weekly : equivalent to "0 0 * * 0"</li>
* <li>@daily : equivalent to "0 0 * * *"</li>
* <li>@midnight : equivalent to "0 0 * * *"</li>
* <li>@hourly : equivalent to "0 * * * *"</li>
* </ul>
* </li>
* </ol>
*
* @param entry The CRON entry to evaluate.
* @param atTime The time at which to evaluate the entry.
*
* @return true if the the current time is a valid runnable time with respect to the supplied entry.
*/
public static boolean shouldRun(String entry, Date atTime) {
entry = entry.trim().toUpperCase();
if (ANNUALLY.equals(entry) || (YEARLY.equals(entry))) {
entry = "0 0 1 1 *";
} else if (MONTHLY.equals(entry)) {
entry = "0 0 1 * *";
} else if (WEEKLY.equals(entry)) {
entry = "0 0 * * 0";
} else if (DAILY.equals(entry) || (MIDNIGHT.equals(entry))) {
entry = "0 0 * * *";
} else if (HOURLY.equals(entry)) {
entry = "0 * * * *";
}
return new CronTabEntry(entry).isRunnable(atTime);
}

/**
* Indicates if a CRON entry should run at the current moment in time.
*
* @param entry The CRON entry to evaluate.
*
* @return true if the the current time is a valid runnable time with respect to the supplied entry.
*/
public static boolean shouldRun(String entry) {
return Cron.shouldRun(entry, new Date());
}

/**
* Determines if an entry is valid CRON syntax.
*
* @param entry The CRON entry.
*
* @return True if the entry is valid CRON syntax.
*/
public static boolean isValid(String entry) {
boolean result = true;

try {
shouldRun(entry);
} catch (Exception ex) {
result = false;
}
return result;
}
//~ Static fields/initializers *******************************************************************************************************************

private static final String ANNUALLY = "@ANNUALLY";
private static final String YEARLY = "@YEARLY";
private static final String MONTHLY = "@MONTHLY";
private static final String WEEKLY = "@WEEKLY";
private static final String DAILY = "@DAILY";
private static final String MIDNIGHT = "@MIDNIGHT";
private static final String HOURLY = "@HOURLY";

//~ Constructors *********************************************************************************************************************************

private Cron() { }

//~ Methods **************************************************************************************************************************************

/**
* Determines if the given CRON entry is runnable at this current moment in time. This mimics the original implementation of the CRON table.
*
* <p>This implementation supports only the following types of entries:</p>
*
* <ol>
* <li>Standard Entries having form: &lt;minutes&gt; &lt;hours&gt; &lt;days&gt; &lt;months&gt; &lt;days of week&gt;
*
* <ul>
* <li>* : All</li>
* <li>*\/n : Only mod n</li>
* <li>n : Numeric</li>
* <li>n-n : Range</li>
* <li>n,n,...,n : List</li>
* <li>n,n-n,...,n : List having ranges</li>
* </ul>
* </li>
* <li>Special Entries
*
* <ul>
* <li>@annually : equivalent to "0 0 1 1 *"</li>
* <li>@yearly : equivalent to "0 0 1 1 *"</li>
* <li>@monthly : equivalent to "0 0 1 * *"</li>
* <li>@weekly : equivalent to "0 0 * * 0"</li>
* <li>@daily : equivalent to "0 0 * * *"</li>
* <li>@midnight : equivalent to "0 0 * * *"</li>
* <li>@hourly : equivalent to "0 * * * *"</li>
* </ul>
* </li>
* </ol>
*
* @param entry The CRON entry to evaluate.
* @param atTime The time at which to evaluate the entry.
*
* @return true if the the current time is a valid runnable time with respect to the supplied entry.
*/
public static boolean shouldRun(String entry, Date atTime) {
entry = entry.trim().toUpperCase();
if (ANNUALLY.equals(entry) || (YEARLY.equals(entry))) {
entry = "0 0 1 1 *";
} else if (MONTHLY.equals(entry)) {
entry = "0 0 1 * *";
} else if (WEEKLY.equals(entry)) {
entry = "0 0 * * 0";
} else if (DAILY.equals(entry) || (MIDNIGHT.equals(entry))) {
entry = "0 0 * * *";
} else if (HOURLY.equals(entry)) {
entry = "0 * * * *";
}
return new CronTabEntry(entry).isRunnable(atTime);
}

/**
* Indicates if a CRON entry should run at the current moment in time.
*
* @param entry The CRON entry to evaluate.
*
* @return true if the the current time is a valid runnable time with respect to the supplied entry.
*/
public static boolean shouldRun(String entry) {
return Cron.shouldRun(entry, new Date());
}

/**
* Determines if an entry is valid CRON syntax.
*
* @param entry The CRON entry.
*
* @return True if the entry is valid CRON syntax.
*/
public static boolean isValid(String entry) {
boolean result = true;

try {
shouldRun(entry);
} catch (Exception ex) {
result = false;
}
return result;
}

public static boolean isCronEntryValid(String cronEntry) {
String quartzCronEntry = convertToQuartzCronEntry(cronEntry);

Expand All @@ -152,9 +152,17 @@ public static boolean isCronEntryValid(String cronEntry) {
}
return true;
}

public static String convertToQuartzCronEntry(String cronEntry) {
return "0 " + cronEntry.substring(0, cronEntry.length() - 1) + "?";
}

public static String convertToQuartzCronEntry(String cronEntry) {
// adding seconds field
cronEntry = "0 " + cronEntry.trim();

// if day of the week is not specified, substitute it with ?, so as to prevent conflict with month field
if(cronEntry.charAt(cronEntry.length() - 1) == '*') {
return cronEntry.substring(0, cronEntry.length() - 1) + "?";
}else {
return cronEntry;
}
}
}
/* Copyright (c) 2016, Salesforce.com, Inc. All rights reserved. */
6 changes: 6 additions & 0 deletions ArgusCore/src/test/java/com/salesforce/dva/argus/util/CronTest.java
View file
Expand Up @@ -161,5 +161,11 @@ public void testTablesMixedList() {
}
}
}

@Test
public void testConvertCronEntryToQuartzCronEntry() {
assertEquals(Cron.convertToQuartzCronEntry(" * 5-17 * * * "), "0 * 5-17 * * ?");
assertEquals(Cron.convertToQuartzCronEntry("* 5-17 * * 1-5"), "0 * 5-17 * * 1-5");
}
}
/* Copyright (c) 2016, Salesforce.com, Inc. All rights reserved. */

0 comments on commit d8d3211

Please sign in to comment.