Preserve columns with the same name - closes #43

app/controllers/blazer/dashboards_controller.rb

@@ -37,8 +37,8 @@ module Blazer
         @data_sources.each do |data_source|
           query = data_source.smart_variables[var]
           if query
-            rows, error, cached_at = data_source.run_statement(query)
-            ((@smart_vars[var] ||= []).concat(rows.map { |v| v.values.reverse })).uniq!
+            columns, rows, error, cached_at = data_source.run_statement(query)
+            ((@smart_vars[var] ||= []).concat(rows.map { |v| v.reverse })).uniq!
             @sql_errors << error if error
           end
         end

app/controllers/blazer/queries_controller.rb

@@ -51,8 +51,8 @@ module Blazer
       @bind_vars.each do |var|
         query = data_source.smart_variables[var]
         if query
-          rows, error, cached_at = data_source.run_statement(query)
-          @smart_vars[var] = rows.map { |v| v.values.reverse }
+          columns, rows, error, cached_at = data_source.run_statement(query)
+          @smart_vars[var] = rows.map { |v| v.reverse }
           @sql_errors << error if error
         end
       end
@@ -85,7 +85,7 @@ module Blazer
           audit.save!
         end
 
-        @rows, @error, @cached_at = @data_source.run_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])
+        @columns, @rows, @error, @cached_at = @data_source.run_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])
 
         if @query && @error != Blazer::TIMEOUT_MESSAGE
           @query.checks.each do |check|
@@ -93,11 +93,12 @@ module Blazer
           end
         end
 
-        @columns = {}
+        @first_row = @rows.first || []
+        @column_types = []
         if @rows.any?
-          @rows.first.each do |key, value|
-            @columns[key] =
-              case value
+          @columns.each_with_index do |column, i|
+            @column_types << (
+              case @first_row[i]
               when Integer
                 "int"
               when Float
@@ -105,20 +106,20 @@ module Blazer
               else
                 "string-ins"
               end
+            )
           end
         end
 
         @filename = @query.name.parameterize if @query
-
-        @min_width_types = (@rows.first || {}).select { |k, v| v.is_a?(Time) || v.is_a?(String) || @data_source.smart_columns[k] }.keys
+        @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }
 
         @boom = {}
-        @columns.keys.each do |key|
+        @columns.each_with_index do |key, i|
           query = @data_source.smart_columns[key]
           if query
-            values = @rows.map { |r| r[key] }.compact.uniq
-            rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
-            @boom[key] = Hash[rows.map(&:values).map { |k, v| [k.to_s, v] }]
+            values = @rows.map { |r| r[i] }.compact.uniq
+            columns, rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
+            @boom[key] = Hash[rows.map { |k, v| [k.to_s, v] }]
           end
         end
 
@@ -126,15 +127,17 @@ module Blazer
 
         @markers = []
         [["latitude", "longitude"], ["lat", "lon"]].each do |keys|
-          if (keys - (@rows.first || {}).keys).empty?
+          lat_index = @columns.index(keys.first)
+          lon_index = @columns.index(keys.last)
+          if lat_index && lon_index
             @markers =
               @rows.select do |r|
-                r[keys.first] && r[keys.last]
+                r[lat_index] && r[lon_index]
               end.map do |r|
                 {
-                  title: r.except(*keys).map{ |k, v| "<strong>#{k}:</strong> #{v}" }.join("<br />").truncate(140),
-                  latitude: r[keys.first],
-                  longitude: r[keys.last]
+                  title: r.each_with_index.map{ |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{@columns[i]}:</strong> #{v}" }.compact.join("<br />").truncate(140),
+                  latitude: r[lat_index],
+                  longitude: r[lon_index]
                 }
               end
           end
@@ -146,7 +149,7 @@ module Blazer
           render layout: false
         end
         format.csv do
-          send_data csv_data(@rows), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
+          send_data csv_data(@columns, @rows), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
         end
       end
     end
@@ -181,7 +184,7 @@ module Blazer
     end
 
     def tables
-      @tables = Blazer.data_sources[params[:data_source]].tables.keys
+      @tables = Blazer.data_sources[params[:data_source]].tables
       render partial: "tables", layout: false
     end
 
@@ -224,13 +227,11 @@ module Blazer
       params.require(:query).permit(:name, :description, :statement, :data_source)
     end
 
-    def csv_data(rows)
+    def csv_data(columns, rows)
       CSV.generate do |csv|
-        if rows.any?
-          csv << rows.first.keys
-        end
+        csv << columns
         rows.each do |row|
-          csv << row.map { |k, v| v.is_a?(Time) ? blazer_time_value(k, v) : v }
+          csv << row.each_with_index.map { |v, i| v.is_a?(Time) ? blazer_time_value(columns[i], v) : v }
         end
       end
     end

app/helpers/blazer/base_helper.rb

@@ -17,8 +17,8 @@ module Blazer
     end
 
     def blazer_column_types(columns, rows, boom)
-      columns.map do |k, _|
-        v = (rows.find { |r| r[k] } || {})[k]
+      columns.each_with_index.map do |k, i|
+        v = rows.find { |r| r[i] }[i]
         if boom[k]
           "string"
         elsif v.is_a?(Numeric)

app/views/blazer/queries/run.html.erb

@@ -24,13 +24,13 @@
     <p class="text-muted"><%= pluralize(@rows.size, "row") %></p>
   <% end %>
   <% if @rows.any? %>
-    <% values = @rows.first.values %>
+    <% values = @rows.first %>
     <% chart_id = SecureRandom.hex %>
     <% column_types = blazer_column_types(@columns, @rows, @boom) %>
     <% chart_options = {id: chart_id, min: nil} %>
     <% series_library = {} %>
-    <% @columns.keys.select { |k| k.downcase == "target" }.each do |key| %>
-      <% series_library[key] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
+    <% @columns.select { |k| k.downcase == "target" }.each_with_index do |key, i| %>
+      <% series_library[i] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
     <% end %>
     <% if blazer_maps? && @markers.any? %>
       <div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>
@@ -62,32 +62,30 @@
         map.fitBounds(featureLayer.getBounds());
       </script>
     <% elsif values.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" } %>
-      <% time_k = @columns.keys.first %>
-      <%= line_chart @columns.keys[1..-1].map{ |k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }, library: series_library[k]} }, chart_options %>
+      <%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: k, data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %>
     <% elsif values.size == 3 && column_types == ["time", "string", "numeric"] %>
-      <% keys = @columns.keys %>
-      <%= line_chart @rows.group_by { |r| k = keys[1]; v = r[k]; (@boom[k] || {})[v.to_s] || v }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] }, library: series_library[name]} }, chart_options %>
+      <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
     <% elsif values.size >= 2 && column_types == ["string"] + (values.size - 1).times.map { "numeric" } %>
-      <% keys = @columns.keys %>
-      <%= column_chart (values.size - 1).times.map { |i| name = @columns.keys[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[keys[0]] || {})[r[keys[0]].to_s] || r[keys[0]], r[keys[i + 1]]] } } }, id: chart_id %>
+      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>
     <% elsif @only_chart %>
       <% if @rows.size == 1 && @rows.first.size == 1 %>
-        <p style="font-size: 160px;"><%= blazer_format_value(@rows.first.keys.first, @rows.first.values.first) %></p>
+        <p style="font-size: 160px;"><%= blazer_format_value(@columns.first, @rows.first.first) %></p>
       <% else %>
         <% @no_chart = true %>
       <% end %>
     <% end %>
 
     <% unless @only_chart && !@no_chart %>
-      <% header_width = 100 / @rows.first.keys.size.to_f %>
+      <% header_width = 100 / @columns.size.to_f %>
       <div class="results-container">
-        <% if @columns.keys == ["QUERY PLAN"] %>
-          <pre><code><%= @rows.map { |r| r["QUERY PLAN"] }.join("\n") %></code></pre>
+        <% if @columns == ["QUERY PLAN"] %>
+          <pre><code><%= @rows.map { |r| r[0] }.join("\n") %></code></pre>
         <% else %>
           <table class="table results-table" style="margin-bottom: 0;">
             <thead>
               <tr>
-                <% @columns.each do |key, type| %>
+                <% @columns.each_with_index do |key, i| %>
+                  <% type = @column_types[i] %>
                   <th style="width: <%= header_width %>%;" data-sort="<%= type %>">
                     <div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;">
                       <%= key %>
@@ -99,7 +97,8 @@
             <tbody>
               <% @rows.each do |row| %>
                 <tr>
-                  <% row.each do |k, v| %>
+                  <% row.each_with_index do |v, i| %>
+                    <% k = @columns[i] %>
                     <td>
                       <% if v.is_a?(Time) %>
                         <% v = blazer_time_value(k, v) %>

lib/blazer.rb

@@ -72,7 +72,7 @@ module Blazer
       ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
         # try 3 times on timeout errors
         while tries <= 3
-          rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)
+          columns, rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)
           if error == Blazer::TIMEOUT_MESSAGE
             Rails.logger.info "[blazer timeout] query=#{check.query.name}"
             tries += 1

lib/blazer/data_source.rb

@@ -58,16 +58,18 @@ module Blazer
     end
 
     def run_statement(statement, options = {})
+      columns = nil
       rows = nil
       error = nil
       cached_at = nil
       cache_key = self.cache_key(statement) if cache
       if cache && !options[:refresh_cache]
         value = Blazer.cache.read(cache_key)
-        rows, cached_at = Marshal.load(value) if value
+        columns, rows, cached_at = Marshal.load(value) if value
       end
 
       unless rows
+        columns = []
         rows = []
 
         comment = "blazer"
@@ -95,13 +97,10 @@ module Blazer
             end
 
             result = connection_model.connection.select_all("#{statement} /*#{comment}*/")
+            columns = result.columns
             cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
-            result.each do |untyped_row|
-              row = {}
-              untyped_row.each do |k, v|
-                row[k] = result.column_types.empty? ? v : result.column_types[k].send(cast_method, v)
-              end
-              rows << row
+            result.rows.each do |untyped_row|
+              rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| result.column_types[c].send(cast_method, untyped_row[i]) })
             end
           rescue ActiveRecord::StatementInvalid => e
             error = e.message.sub(/.+ERROR: /, "")
@@ -109,10 +108,10 @@ module Blazer
           end
         end
 
-        Blazer.cache.write(cache_key, Marshal.dump([rows, Time.now]), expires_in: cache.to_f * 60) if !error && cache
+        Blazer.cache.write(cache_key, Marshal.dump([columns, rows, Time.now]), expires_in: cache.to_f * 60) if !error && cache
       end
 
-      [rows, error, cached_at]
+      [columns, rows, error, cached_at]
     end
 
     def clear_cache(statement)
@@ -120,7 +119,7 @@ module Blazer
     end
 
     def cache_key(statement)
-      ["blazer", "v2", id, Digest::MD5.hexdigest(statement)].join("/")
+      ["blazer", "v3", id, Digest::MD5.hexdigest(statement)].join("/")
     end
 
     def schemas
@@ -129,8 +128,8 @@ module Blazer
     end
 
     def tables
-      rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema IN (?)", schemas]))
-      Hash[rows.group_by { |r| r["table_name"] }.map { |t, f| [t, f.sort_by { |f| f["ordinal_position"] }.map { |f| f.slice("column_name", "data_type") }] }.sort_by { |t, _f| t }]
+      columns, rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema IN (?)", schemas]))
+      rows.map(&:first)
     end
 
     def postgresql?