[bug] Delete duplicate alerts from Datadog

* This will now delete the duplicate copies and only retain one
  version per unique name. This was causing issues for things like
  muting as there would be multiple copies of the same alert in Datadog.
* Bump version to 0.0.14

lib/interferon.rb

@@ -193,7 +193,7 @@ def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
         existing_alert = existing_alerts[name]
         dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + name
         existing_alert['name'] = dry_run_alert_name
-        existing_alert['id'] = nil
+        existing_alert['id'] = [nil]
         existing_alerts[dry_run_alert_name] = existing_alerts.delete(name)
       end
 
@@ -216,7 +216,15 @@ def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
       to_remove = existing_alerts.dup
       alerts_queue.each do |name, alert_people_pair|
         alert = alert_people_pair[0]
-        to_remove.delete(alert['name'])
+        old_alerts = to_remove[alert['name']]
+
+        if not old_alerts.nil?
+          if old_alerts['id'].length == 1
+            to_remove.delete(alert['name'])
+          else
+            old_alerts['id'] = old_alerts['id'].drop(0)
+          end
+        end
       end
 
       # Clean up alerts not longer being generated
@@ -227,8 +235,10 @@ def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
 
       # Clean up dry-run created alerts
       (created_alerts + existing_dry_run_alerts).each do |alert_id_pair|
-        alert_id = alert_id_pair[1]
-        dest.remove_alert_by_id(alert_id)
+        alert_ids = alert_id_pair[1]
+        alert_ids.each do |alert_id|
+          dest.remove_alert_by_id(alert_id)
+        end
       end
 
     end
@@ -246,7 +256,15 @@ def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
       to_remove = existing_alerts.dup
       alerts_queue.each do |name, alert_people_pair|
         alert = alert_people_pair[0]
-        to_remove.delete(alert['name'])
+        old_alerts = to_remove[alert['name']]
+
+        if not old_alerts.nil?
+          if alert['id'].length == 1
+            to_remove.delete(alert['name'])
+          else
+            old_alerts['id'] = old_alerts['id'].drop(0)
+          end
+        end
       end
 
       # Clean up alerts not longer being generated
@@ -380,7 +398,7 @@ def self.same_alerts(dest, alert_people_pair, alert_api_json)
         :message => dest.generate_message(alert['message'], people).strip,
         :notify_no_data => alert['notify_no_data'],
         :silenced => alert['silenced'] || alert['silenced_until'] > Time.now,
-        :timeout => alert['timeout'] || nil,
+        :timeout => alert['timeout'] || nil && [1, alert['timeout'].to_i / 3600].max,
         :no_data_timeframe => alert['no_data_timeframe'] || nil
       }
 

lib/interferon/destinations/datadog.rb

@@ -75,7 +75,16 @@ def existing_alerts
         alerts = resp[1]['alerts']
 
         # key alerts by name
-        @existing_alerts = Hash[alerts.map{ |a| [a['name'], a] }]
+        @existing_alerts = {}
+        alerts.each do |alert|
+          existing_alert = @existing_alerts[alert['name']]
+          if existing_alert.nil?
+            alert['id'] = [alert['id']]
+            @existing_alerts[alert['name']] = alert
+          else
+            existing_alert['id'] << alert['id']
+          end
+        end
 
         # count how many are manually created
         @stats[:manually_created_alerts] = \
@@ -129,7 +138,7 @@ def create_alert(alert, people)
       else
         action = :updating
         @stats[:alerts_to_be_updated] += 1
-        id = existing_alert['id']
+        id = existing_alert['id'][0]
 
         new_alert_text = "Query:\n#{datadog_query}\nMessage:\n#{message}"
         existing_alert_text = "Query:\n#{existing_alert['query']}\nMessage:\n#{existing_alert['message']}\n"
@@ -162,7 +171,7 @@ def create_alert(alert, people)
         @stats[:alerts_silenced] += 1 if alert_opts[:silenced]
       end
 
-      id = resp[1].nil? ? nil : resp[1]['id']
+      id = resp[1].nil? ? nil : [resp[1]['id']]
       # lets key alerts by their name
       return [alert['name'], id]
     end
@@ -173,13 +182,15 @@ def remove_alert(alert)
         log.info("deleting alert: #{alert['name']}")
 
         if not @dry_run
-          resp = @dog.delete_alert(alert['id'])
-          code = resp[0].to_i
-          log_datadog_response_code(resp, code, :deleting)
-
-          if !(code >= 300 || code == -1)
-            # assume this was a success
-            @stats[:alerts_deleted] += 1
+          alert['id'].each do |alert_id|
+            resp = @dog.delete_alert(alert_id)
+            code = resp[0].to_i
+            log_datadog_response_code(resp, code, :deleting)
+
+            if !(code >= 300 || code == -1)
+              # assume this was a success
+              @stats[:alerts_deleted] += 1
+            end
           end
         end
       else

lib/interferon/version.rb

@@ -1,3 +1,3 @@
 module Interferon
-  VERSION = "0.0.13"
+  VERSION = "0.0.14"
 end