Added timed out and disabled states for checks

app/controllers/blazer/checks_controller.rb

@@ -3,7 +3,8 @@ module Blazer
     before_action :set_check, only: [:edit, :update, :destroy, :run]
 
     def index
-      @checks = Blazer::Check.joins(:query).includes(:query).order("state, blazer_queries.name, blazer_checks.id").to_a
+      state_order = [nil, "disabled", "error", "timed out", "failing", "passing"]
+      @checks = Blazer::Check.joins(:query).includes(:query).order("blazer_queries.name, blazer_checks.id").to_a.sort_by { |q| state_order.index(q.state) || 99 }
       @checks.select! { |c| "#{c.query.name} #{c.emails}".downcase.include?(params[:q]) } if params[:q]
     end
 

app/models/blazer/check.rb

@@ -5,21 +5,34 @@ module Blazer
     validates :query_id, presence: true
 
     def split_emails
-      emails.to_s.split(",").map(&:strip)
+      emails.to_s.downcase.split(",").map(&:strip)
     end
 
     def update_state(rows, error)
-      invert = self.respond_to?(:invert) && self.invert
+      invert = respond_to?(:invert) && self.invert
       self.state =
         if error
-          "error"
+          if error == Blazer::TIMEOUT_MESSAGE
+            "timed out"
+          else
+            "error"
+          end
         elsif rows.any?
           invert ? "passing" : "failing"
         else
           invert ? "failing" : "passing"
         end
 
-      self.last_run_at = Time.now if self.respond_to?(:last_run_at=)
+      self.last_run_at = Time.now if respond_to?(:last_run_at=)
+
+      if respond_to?(:timeouts=)
+        if state == "timed out"
+          self.timeouts += 1
+          self.state = "disabled" if timeouts >= 3
+        else
+          self.timeouts = 0
+        end
+      end
 
       # do not notify on creation, except when not passing
       if (state_was || state != "passing") && state != state_was && emails.present?

app/views/blazer/checks/index.html.erb

@@ -3,7 +3,7 @@
 <p style="float: right;"><%= link_to "New Check", new_check_path, class: "btn btn-info" %></p>
 <%= render partial: "blazer/nav" %>
 
-<% colors = {failing: "red", passing: "#5cb85c", error: "#666"} %>
+<% colors = {failing: "red", passing: "#5cb85c", error: "#666", timed_out: "orange", disabled: "#000"} %>
 <table class="table">
   <thead>
     <tr>
@@ -20,7 +20,7 @@
         <td><%= link_to check.query.name, check.query %></td>
         <td>
           <% if check.state %>
-            <small style="font-weight: bold; color: <%= colors[check.state.to_sym] %>;"><%= check.state.upcase %></small>
+            <small style="font-weight: bold; color: <%= colors[check.state.parameterize("_").to_sym] %>;"><%= check.state.upcase %></small>
           <% end %>
         </td>
         <td><%= check.schedule if check.respond_to?(:schedule) %></td>

lib/blazer.rb

@@ -106,7 +106,7 @@ module Blazer
 
   def self.send_failing_checks
     emails = {}
-    Blazer::Check.includes(:query).where(state: %w[failing error]).find_each do |check|
+    Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check|
       check.split_emails.each do |email|
         (emails[email] ||= []) << check
       end

lib/generators/blazer/templates/install.rb

@@ -38,6 +38,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration
       t.text :emails
       t.boolean :invert
       t.timestamp :last_run_at
+      t.integer :timeouts, default: 0
       t.timestamps
     end
   end