Revert "Revert checks for now"

This reverts commit 90686e36b423394811b838580b4da5902e2e3b32.

README.md

@@ -21,6 +21,7 @@ Works with PostgreSQL and MySQL
 - **Smart Variables** - no need to remember ids
 - **Charts** - visualize the data
 - **Audits** - all queries are tracked
+- **Checks & Alerts** - get emailed when bad data appears [master]
 
 ## Installation
 
@@ -67,7 +68,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO blazer;
 COMMIT;
 ```
 
-It is **highly, highly recommended** to protect sensitive information with views.  Documentation coming soon.
+It is recommended to protect sensitive information with views.  Documentation coming soon.
 
 ### MySQL
 
@@ -78,7 +79,7 @@ GRANT SELECT, SHOW VIEW ON database_name.* TO blazer@’127.0.0.1′ IDENTIFIED
 FLUSH PRIVILEGES;
 ```
 
-It is **highly, highly recommended** to protect sensitive information with views.  Documentation coming soon.
+It is recommended to protect sensitive information with views.  Documentation coming soon.
 
 ## Authentication
 
@@ -96,11 +97,31 @@ ENV["BLAZER_PASSWORD"] = "secret"
 ### Devise
 
 ```ruby
-authenticate :user, lambda{|user| user.admin? } do
+authenticate :user, lambda { |user| user.admin? } do
   mount Blazer::Engine, at: "blazer"
 end
 ```
 
+## Checks [master]
+
+Set up checks to run every hour.
+
+```sh
+rake blazer:run_checks
+```
+
+Be sure to set a host in `config/environments/production.rb` for emails to work.
+
+```ruby
+config.action_mailer.default_url_options = {host: "blazerme.herokuapp.com"}
+```
+
+We also recommend setting up failing checks to be sent once a day.
+
+```sh
+rake blazer:send_failing_checks
+```
+
 ## Customization
 
 Change time zone
@@ -145,6 +166,27 @@ If there are at least 2 columns and the first is a timestamp and all other colum
 
 If there are 2 columns and the first column is a string and the second column is a numeric, a pie chart will be generated
 
+## Upgrading
+
+### [master]
+
+Add a migration for checks
+
+```sh
+rails g migration create_blazer_checks
+```
+
+with
+
+```ruby
+create_table :blazer_checks do |t|
+  t.references :blazer_query
+  t.string :state
+  t.text :emails
+  t.timestamps
+end
+```
+
 ## TODO
 
 - better readme

app/assets/stylesheets/blazer/selectize.default.css

@@ -1,6 +1,6 @@
 /**
- * selectize.default.css (v0.10.1) - Default Theme
- * Copyright (c) 2013 Brian Reavis & contributors
+ * selectize.default.css (v0.12.1) - Default Theme
+ * Copyright (c) 2013–2015 Brian Reavis & contributors
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
  * file except in compliance with the License. You may obtain a copy of the License at:
@@ -189,6 +189,7 @@
   border: 1px solid #aaaaaa;
 }
 .selectize-input > input {
+  display: inline-block !important;
   padding: 0 !important;
   min-height: 0 !important;
   max-height: none !important;

app/controllers/blazer/base_controller.rb

@@ -0,0 +1,22 @@
+module Blazer
+  class BaseController < ApplicationController
+    # skip all filters
+    skip_filter *_process_action_callbacks.map(&:filter)
+
+    protect_from_forgery with: :exception
+
+    if ENV["BLAZER_PASSWORD"]
+      http_basic_authenticate_with name: ENV["BLAZER_USERNAME"], password: ENV["BLAZER_PASSWORD"]
+    end
+
+    layout "blazer/application"
+
+    before_action :ensure_database_url
+
+    private
+
+    def ensure_database_url
+      render text: "BLAZER_DATABASE_URL required" if !ENV["BLAZER_DATABASE_URL"] && !Rails.env.development?
+    end
+  end
+end

app/controllers/blazer/checks_controller.rb

@@ -0,0 +1,51 @@
+module Blazer
+  class ChecksController < BaseController
+    before_action :set_check, only: [:show, :edit, :update, :destroy, :run]
+
+    def index
+      @checks = Blazer::Check.joins(:blazer_query).includes(:blazer_query).order("state, blazer_queries.name, blazer_checks.id").to_a
+      @checks.select! { |c| "#{c.blazer_query.name} #{c.emails}".downcase.include?(params[:q]) } if params[:q]
+    end
+
+    def new
+      @check = Blazer::Check.new
+    end
+
+    def create
+      @check = Blazer::Check.new(check_params)
+
+      if @check.save
+        redirect_to run_check_path(@check)
+      else
+        render :new
+      end
+    end
+
+    def update
+      if @check.update(check_params)
+        redirect_to run_check_path(@check)
+      else
+        render :edit
+      end
+    end
+
+    def destroy
+      @check.destroy
+      redirect_to checks_path
+    end
+
+    def run
+      @query = @check.blazer_query
+    end
+
+    private
+
+    def check_params
+      params.require(:check).permit(:blazer_query_id, :emails)
+    end
+
+    def set_check
+      @check = Blazer::Check.find(params[:id])
+    end
+  end
+end

app/controllers/blazer/queries_controller.rb

@@ -1,23 +1,12 @@
 module Blazer
-  class QueriesController < ApplicationController
-    # skip all filters
-    skip_filter *_process_action_callbacks.map(&:filter)
-
-    protect_from_forgery with: :exception
-
-    if ENV["BLAZER_PASSWORD"]
-      http_basic_authenticate_with name: ENV["BLAZER_USERNAME"], password: ENV["BLAZER_PASSWORD"]
-    end
-
-    layout "blazer/application"
-
-    before_action :ensure_database_url
+  class QueriesController < BaseController
     before_action :set_query, only: [:show, :edit, :update, :destroy]
 
     def index
       @queries = Blazer::Query.order(:name)
       @queries = @queries.includes(:creator) if Blazer.user_class
       @trending_queries = Blazer::Audit.group(:query_id).where("created_at > ?", 2.days.ago).having("COUNT(DISTINCT user_id) >= 3").uniq.count(:user_id)
+      @checks = Blazer::Check.group(:blazer_query_id).count
     end
 
     def new
@@ -42,9 +31,9 @@ module Blazer
       @smart_vars = {}
       @sql_errors = []
       @bind_vars.each do |var|
-        query = smart_variables[var]
+        query = Blazer.smart_variables[var]
         if query
-          rows, error = run_statement(query)
+          rows, error = Blazer.run_statement(query)
           @smart_vars[var] = rows.map { |v| v.values.reverse }
           @sql_errors << error if error
         end
@@ -69,7 +58,13 @@ module Blazer
           audit.save!
         end
 
-        @rows, @error = run_statement(@statement)
+        @rows, @error = Blazer.run_statement(@statement)
+
+        if @query && !@error.to_s.include?("canceling statement due to statement timeout")
+          @query.blazer_checks.each do |check|
+            check.update_state(@rows, @error)
+          end
+        end
 
         @columns = {}
         if @rows.any?
@@ -88,19 +83,19 @@ module Blazer
 
         @filename = @query.name.parameterize if @query
 
-        @min_width_types = (@rows.first || {}).select { |k, v| v.is_a?(Time) || v.is_a?(String) || smart_columns[k] }.keys
+        @min_width_types = (@rows.first || {}).select { |k, v| v.is_a?(Time) || v.is_a?(String) || Blazer.smart_columns[k] }.keys
 
         @boom = {}
         @columns.keys.each do |key|
-          query = smart_columns[key]
+          query = Blazer.smart_columns[key]
           if query
             values = @rows.map { |r| r[key] }.compact.uniq
-            rows, error = run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
+            rows, error = Blazer.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
             @boom[key] = Hash[rows.map(&:values)]
           end
         end
 
-        @linked_columns = linked_columns
+        @linked_columns = Blazer.linked_columns
       end
 
       respond_to do |format|
@@ -128,10 +123,6 @@ module Blazer
 
     private
 
-    def ensure_database_url
-      render text: "BLAZER_DATABASE_URL required" if !ENV["BLAZER_DATABASE_URL"] && !Rails.env.development?
-    end
-
     def set_query
       @query = Blazer::Query.find(params[:id].to_s.split("-").first)
     end
@@ -151,28 +142,6 @@ module Blazer
       end
     end
 
-    def run_statement(statement)
-      rows = []
-      error = nil
-      begin
-        Blazer::Connection.transaction do
-          Blazer::Connection.connection.execute("SET statement_timeout = #{Blazer.timeout * 1000}") if Blazer.timeout && postgresql?
-          result = Blazer::Connection.connection.select_all(statement)
-          result.each do |untyped_row|
-            row = {}
-            untyped_row.each do |k, v|
-              row[k] = result.column_types.empty? ? v : result.column_types[k].send(:type_cast, v)
-            end
-            rows << row
-          end
-          raise ActiveRecord::Rollback
-        end
-      rescue ActiveRecord::StatementInvalid => e
-        error = e.message.sub(/.+ERROR: /, "")
-      end
-      [rows, error]
-    end
-
     def extract_vars(statement)
       statement.scan(/\{.*?\}/).map { |v| v[1...-1] }.uniq
     end
@@ -198,33 +167,5 @@ module Blazer
       params.except(:controller, :action, :id, :host, :query, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement)
     end
     helper_method :variable_params
-
-    def settings
-      YAML.load(File.read(Rails.root.join("config", "blazer.yml")))
-    end
-
-    def linked_columns
-      settings["linked_columns"] || {}
-    end
-
-    def smart_columns
-      settings["smart_columns"] || {}
-    end
-
-    def smart_variables
-      settings["smart_variables"] || {}
-    end
-
-    def tables
-      default_schema = postgresql? ? "public" : Blazer::Connection.connection_config[:database]
-      schema = Blazer::Connection.connection_config[:schema] || default_schema
-      rows, error = run_statement(Blazer::Connection.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema = ?", schema]))
-      Hash[rows.group_by { |r| r["table_name"] }.map { |t, f| [t, f.sort_by { |f| f["ordinal_position"] }.map { |f| f.slice("column_name", "data_type") }] }.sort_by { |t, _f| t }]
-    end
-    helper_method :tables
-
-    def postgresql?
-      Blazer::Connection.connection.adapter_name == "PostgreSQL"
-    end
   end
 end

app/helpers/blazer/queries_helper.rb → app/helpers/blazer/base_helper.rb

@@ -1,5 +1,5 @@
 module Blazer
-  module QueriesHelper
+  module BaseHelper
     def title(title = nil)
       if title
         content_for(:title) { title }

app/mailers/blazer/check_mailer.rb

@@ -0,0 +1,21 @@
+module Blazer
+  class CheckMailer < ActionMailer::Base
+    include ActionView::Helpers::TextHelper
+
+    default from: Blazer.from_email if Blazer.from_email
+
+    def state_change(check, state, state_was, rows_count, error)
+      @check = check
+      @state = state
+      @state_was = state_was
+      @rows_count = rows_count
+      @error = error
+      mail to: check.emails, subject: "Check #{state.titleize}: #{check.blazer_query.name}"
+    end
+
+    def failing_checks(email, checks)
+      @checks = checks
+      mail to: email, subject: "#{pluralize(checks.size, "Check")} Failing"
+    end
+  end
+end

app/models/blazer/check.rb

@@ -0,0 +1,28 @@
+module Blazer
+  class Check < ActiveRecord::Base
+    belongs_to :blazer_query, class_name: "Blazer::Query"
+
+    validates :blazer_query_id, presence: true
+
+    def split_emails
+      emails.to_s.split(",").map(&:strip)
+    end
+
+    def update_state(rows, error)
+      self.state =
+        if error
+          "error"
+        elsif rows.any?
+          "failing"
+        else
+          "passing"
+        end
+
+      # do not notify on creation, except when not passing
+      if (state_was || state != "passing") && state != state_was && emails.present?
+        Blazer::CheckMailer.state_change(self, state, state_was, rows.size, error).deliver_later
+      end
+      save!
+    end
+  end
+end

app/models/blazer/query.rb

@@ -1,6 +1,7 @@
 module Blazer
   class Query < ActiveRecord::Base
     belongs_to :creator, class_name: Blazer.user_class.to_s if Blazer.user_class
+    has_many :blazer_checks, class_name: "Blazer::Check", foreign_key: "blazer_query_id", dependent: :destroy
 
     validates :name, presence: true
     validates :statement, presence: true

app/views/blazer/check_mailer/failing_checks.html.erb

@@ -0,0 +1,6 @@
+<ul>
+  <% @checks.each do |check| %>
+    <li><%= link_to check.blazer_query.name, query_url(check.blazer_query_id) %></li>
+  <% end %>
+</ul>
+<p><%= link_to "Manage checks", checks_url %></p>

app/views/blazer/check_mailer/state_change.html.erb

@@ -0,0 +1,6 @@
+<p><%= link_to "View", query_url(@check.blazer_query_id) %></p>
+<% if @error %>
+  <p><%= @error %></p>
+<% elsif @rows_count > 0 %>
+  <p><%= pluralize(@rows_count, "row") %></p>
+<% end %>

app/views/blazer/checks/_form.html.erb

@@ -0,0 +1,28 @@
+<p class="text-muted">Checks are designed to identify bad data. A check fails if there are any results.</p>
+
+<% if @check.errors.any? %>
+  <div class="alert alert-danger"><%= @check.errors.full_messages.first %></div>
+<% end %>
+
+<%= form_for @check do |f| %>
+  <div class="form-group">
+    <%= f.label :blazer_query_id, "Query" %>
+    <div class="hide">
+      <%= f.select :blazer_query_id, Blazer::Query.order(:name).map { |q| [q.name, q.id] }, {include_blank: true} %>
+    </div>
+    <script>
+      $("#check_blazer_query_id").selectize().parents(".hide").removeClass("hide");
+    </script>
+  </div>
+  <div class="form-group">
+    <%= f.label :emails %>
+    <%= f.text_field :emails, placeholder: "Optional, comma separated", class: "form-control" %>
+  </div>
+  <p class="text-muted">Emails are sent when a check starts failing, and when it starts passing again.
+  <p>
+    <% if @check.persisted? %>
+      <%= link_to "Delete", check_path(@check), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>
+    <% end %>
+    <%= f.submit "Save", class: "btn btn-success" %>
+  </p>
+<% end %>

app/views/blazer/checks/edit.html.erb

@@ -0,0 +1 @@
+<%= render partial: "form" %>

app/views/blazer/checks/index.html.erb

@@ -0,0 +1,41 @@
+<% title "Checks" %>
+
+<p style="float: right;"><%= link_to "New Check", new_check_path, class: "btn btn-info" %></p>
+<p>
+  <%= link_to "Home", root_path, class: "btn btn-primary", style: "margin-right: 10px;" %>
+</p>
+
+<% colors = {failing: "red", passing: "#5cb85c", error: "#666"} %>
+<table class="table">
+  <thead>
+    <tr>
+      <th>Query</th>
+      <th style="width: 15%;">State</th>
+      <th style="width: 20%;">Emails</th>
+      <th style="width: 15%;"></th>
+    </tr>
+  </thead>
+  <tbody>
+    <% @checks.each do |check| %>
+      <tr>
+        <td><%= link_to check.blazer_query.name, check.blazer_query %></td>
+        <td>
+          <% if check.state %>
+            <small style="font-weight: bold; color: <%= colors[check.state.to_sym] %>;"><%= check.state.upcase %></small>
+          <% end %>
+        </td>
+        <td>
+          <ul class="list-unstyled" style="margin-bottom: 0;">
+            <% check.split_emails.each do |email| %>
+              <li><%= email %></li>
+            <% end %>
+          </ul>
+        </td>
+        <td style="text-align: right; padding: 1px;">
+          <%= link_to "Edit", edit_check_path(check), class: "btn btn-info" %>
+          <%= link_to "Run Now", run_check_path(check), class: "btn btn-primary" %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

app/views/blazer/checks/new.html.erb

@@ -0,0 +1 @@
+<%= render partial: "form" %>

app/views/blazer/checks/run.html.erb

@@ -0,0 +1,9 @@
+<p style="text-muted">Running check...</p>
+
+<script>
+  $.post("<%= run_queries_path %>", <%= json_escape({statement: @query.statement, query_id: @query.id}.to_json).html_safe %>, function (data) {
+    setTimeout( function () {
+      window.location.href = "<%= checks_path %>";
+    }, 200);
+  });
+</script>

app/views/blazer/queries/_form.html.erb

@@ -16,7 +16,7 @@
           <%= link_to "Back", :back %>
           <span class="text-muted" style="margin-left: 20px;"> Use {start_time} and {end_time} for time ranges</span>
         </div>
-        <%= select_tag :table_names, options_for_select([["Preview table", nil]] + tables.keys), style: "margin-right: 20px; width: 240px;" %>
+        <%= select_tag :table_names, options_for_select([["Preview table", nil]] + Blazer.tables.keys), style: "margin-right: 20px; width: 240px;" %>
         <%= link_to "Run", "#", class: "btn btn-info", id: "run", style: "vertical-align: top;" %>
       </div>
     </div>

app/views/blazer/queries/index.html.erb

@@ -2,6 +2,7 @@
   <div id="header" style="margin-bottom: 20px;">
     <div class="pull-right">
       <%= link_to "New Query", new_query_path, class: "btn btn-info" %>
+      <%= link_to "Checks", checks_path, class: "btn btn-primary" %>
     </div>
     <input type="text" placeholder="Start typing a query or person" style="width: 300px; display: inline-block;" autofocus=true class="search form-control" />
   </div>
@@ -25,6 +26,9 @@
             <% if @trending_queries[query.id] %>
               <small style="font-weight: bold; color: #f60;">TRENDING</small>
             <% end %>
+            <% if @checks[query.id] %>
+              <small style="font-weight: bold; color: #f60;">CHECK</small>
+            <% end %>
             <div class="hide"><%= query.name.gsub(/\s+/, "") %></div>
           </td>
           <td class="creator text-right text-muted">

app/views/blazer/queries/run.html.erb

@@ -6,12 +6,12 @@
   <p class="text-muted"><%= pluralize(@rows.size, "row") %></p>
   <% if @rows.any? %>
     <% values = @rows.first.values %>
-    <% if values.size >= 2 && values.first.is_a?(Time) && values[1..-1].all?{|v| v.is_a?(Numeric) } %>
+    <% if values.size >= 2 && (values.first.is_a?(Time) || values.first.is_a?(Date)) && values[1..-1].all?{|v| v.is_a?(Numeric) } %>
       <% time_k = @columns.keys.first %>
-      <%= line_chart @columns.keys[1..-1].map{|k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }} } %>
-    <% elsif values.size == 3 && values.first.is_a?(Time) && values[1].is_a?(String) && values[2].is_a?(Numeric) %>
+      <%= line_chart @columns.keys[1..-1].map{|k| {name: k, data: @rows.map{|r| [r[time_k].in_time_zone(Blazer.time_zone), r[k]] }} } %>
+    <% elsif values.size == 3 && (values.first.is_a?(Time) || values.first.is_a?(Date)) && values[1].is_a?(String) && values[2].is_a?(Numeric) %>
       <% keys = @columns.keys %>
-      <%= line_chart @rows.group_by { |v| v[keys[1]] }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] } } } %>
+      <%= line_chart @rows.group_by { |v| v[keys[1]] }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]].in_time_zone(Blazer.time_zone), v2[keys[2]]] } } } %>
     <% elsif values.size == 2 && values.first.is_a?(String) && values.last.is_a?(Numeric) %>
       <%= pie_chart @rows.map(&:values), library: {sliceVisibilityThreshold: 1 / 40.0} %>
     <% end %>

config/routes.rb

@@ -2,5 +2,8 @@ Blazer::Engine.routes.draw do
   resources :queries, except: [:index] do
     post :run, on: :collection # err on the side of caution
   end
+  resources :checks, except: [:show] do
+    get :run, on: :member
+  end
   root to: "queries#index"
 end

lib/blazer.rb

@@ -2,6 +2,7 @@ require "csv"
 require "chartkick"
 require "blazer/version"
 require "blazer/engine"
+require "blazer/tasks"
 
 module Blazer
   class << self
@@ -10,6 +11,7 @@ module Blazer
     attr_accessor :user_name
     attr_accessor :user_class
     attr_accessor :timeout
+    attr_accessor :from_email
   end
   self.audit = true
   self.user_name = :name
@@ -18,4 +20,80 @@ module Blazer
   def self.time_zone=(time_zone)
     @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
   end
+
+  def self.settings
+    @settings ||= YAML.load(File.read(Rails.root.join("config", "blazer.yml")))
+  end
+
+  def self.linked_columns
+    settings["linked_columns"] || {}
+  end
+
+  def self.smart_columns
+    settings["smart_columns"] || {}
+  end
+
+  def self.smart_variables
+    settings["smart_variables"] || {}
+  end
+
+  def self.run_statement(statement)
+    rows = []
+    error = nil
+    begin
+      Blazer::Connection.transaction do
+        Blazer::Connection.connection.execute("SET statement_timeout = #{Blazer.timeout * 1000}") if Blazer.timeout && postgresql?
+        result = Blazer::Connection.connection.select_all(statement)
+        result.each do |untyped_row|
+          row = {}
+          untyped_row.each do |k, v|
+            row[k] = result.column_types.empty? ? v : result.column_types[k].send(:type_cast, v)
+          end
+          rows << row
+        end
+        raise ActiveRecord::Rollback
+      end
+    rescue ActiveRecord::StatementInvalid => e
+      error = e.message.sub(/.+ERROR: /, "")
+    end
+    [rows, error]
+  end
+
+  def self.tables
+    default_schema = postgresql? ? "public" : Blazer::Connection.connection_config[:database]
+    schema = Blazer::Connection.connection_config[:schema] || default_schema
+    rows, error = run_statement(Blazer::Connection.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema = ?", schema]))
+    Hash[rows.group_by { |r| r["table_name"] }.map { |t, f| [t, f.sort_by { |f| f["ordinal_position"] }.map { |f| f.slice("column_name", "data_type") }] }.sort_by { |t, _f| t }]
+  end
+
+  def self.postgresql?
+    Blazer::Connection.connection.adapter_name == "PostgreSQL"
+  end
+
+  def self.run_checks
+    Blazer::Check.includes(:blazer_query).find_each do |check|
+      rows = nil
+      error = nil
+      tries = 0
+      # try 3 times on timeout errors
+      begin
+        rows, error = run_statement(check.blazer_query.statement)
+        tries += 1
+      end while error && error.include?("canceling statement due to statement timeout") && tries < 3
+      check.update_state(rows, error)
+    end
+  end
+
+  def self.send_failing_checks
+    emails = {}
+    Blazer::Check.includes(:blazer_query).where(state: %w[failing error]).find_each do |check|
+      check.split_emails.each do |email|
+        (emails[email] ||= []) << check
+      end
+    end
+
+    emails.each do |email, checks|
+      Blazer::CheckMailer.failing_checks(email, checks).deliver_later
+    end
+  end
 end

lib/blazer/tasks.rb

@@ -0,0 +1,12 @@
+require "rake"
+
+namespace :blazer do
+  desc "run checks"
+  task run_checks: :environment do
+    Blazer.run_checks
+  end
+
+  task send_failing_checks: :environment do
+    Blazer.send_failing_checks
+  end
+end

lib/generators/blazer/templates/install.rb

@@ -14,5 +14,12 @@ class <%= migration_class_name %> < ActiveRecord::Migration
       t.text :statement
       t.timestamp :created_at
     end
+
+    create_table :blazer_checks do |t|
+      t.references :blazer_query
+      t.string :state
+      t.text :emails
+      t.timestamps
+    end
   end
 end