Haruka Yoshihara
null+****@clear*****
Wed Dec 12 12:36:34 JST 2012
Haruka Yoshihara 2012-12-12 12:36:34 +0900 (Wed, 12 Dec 2012) New Revision: d260f39b870194cd27e56785086064533f1d9f3e https://github.com/groonga/groonga-query-log/commit/d260f39b870194cd27e56785086064533f1d9f3e Log: Import classes under GroongaQueryLog from groonga repository TODO: add tests Original definition of classes exists in groonga/tools/groonga-query-log-analyzer. Added files: lib/groonga/query-log/analyzer.rb lib/groonga/query-log/reporter.rb lib/groonga/query-log/reporter/console.rb lib/groonga/query-log/reporter/html.rb lib/groonga/query-log/reporter/json.rb lib/groonga/query-log/sized-grouped-operations.rb lib/groonga/query-log/sized-statistics.rb Copied files: lib/groonga/query-log/streamer.rb (from lib/groonga/query-log.rb) Modified files: lib/groonga/query-log.rb Modified: lib/groonga/query-log.rb (+7 -0) =================================================================== --- lib/groonga/query-log.rb 2012-11-27 13:54:15 +0900 (9d93c69) +++ lib/groonga/query-log.rb 2012-12-12 12:36:34 +0900 (0641ced) @@ -17,4 +17,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA require "groonga/query-log/version" +require "groonga/query-log/analyzer" require "groonga/query-log/parser" +require "groonga/query-log/reporter/console" +require "groonga/query-log/reporter/json" +require "groonga/query-log/reporter/html" +require "groonga/query-log/sized-grouped-operations" +require "groonga/query-log/sized-statistics" +require "groonga/query-log/streamer" Added: lib/groonga/query-log/analyzer.rb (+210 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/analyzer.rb 2012-12-12 12:36:34 +0900 (4dcc949) @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require 'optparse' +require "groonga/query-log/reporter" + +module Groonga + module QueryLog + class Analyzer + class Error < StandardError + end + + class UnsupportedReporter < Error + end + + def initialize + setup_options + end + + def run(argv=nil) + log_paths = @option_parser.parse!(argv || ARGV) + + stream = @options[:stream] + dynamic_sort = @options[:dynamic_sort] + statistics = Groonga::QueryLog::SizedStatistics.new + statistics.apply_options(@options) + parser = Groonga::QueryLog::Parser.new + if stream + streamer = Groonga::QueryLog::Streamer.new(create_reporter(statistics)) + streamer.start + process_statistic = lambda do |statistic| + streamer << statistic + end + elsif dynamic_sort + process_statistic = lambda do |statistic| + statistics << statistic + end + else + full_statistics = [] + process_statistic = lambda do |statistic| + full_statistics << statistic + end + end + if log_paths.empty? + parser.parse(ARGF, &process_statistic) + else + log_paths.each do |log_path| + File.open(log_path) do |log| + parser.parse(log, &process_statistic) + end + end + end + if stream + streamer.finish + return + end + statistics.replace(full_statistics) unless dynamic_sort + + reporter = create_reporter(statistics) + reporter.apply_options(@options) + reporter.report + end + + private + def setup_options + @options = {} + @options[:n_entries] = 10 + @options[:order] = "-elapsed" + @options[:color] = :auto + @options[:output] = "-" + @options[:slow_operation_threshold] = 0.1 + @options[:slow_response_threshold] = 0.2 + @options[:reporter] = "console" + @options[:dynamic_sort] = true + @options[:stream] = false + @options[:report_summary] = true + + @option_parser = OptionParser.new do |parser| + parser.banner += " LOG1 ..." + + parser.on("-n", "--n-entries=N", + Integer, + "Show top N entries", + "(#{@options[:n_entries]})") do |n| + @options[:n_entries] = n + end + + available_orders = ["elapsed", "-elapsed", "start-time", "-start-time"] + parser.on("--order=ORDER", + available_orders, + "Sort by ORDER", + "available values: [#{available_orders.join(', ')}]", + "(#{@options[:order]})") do |order| + @options[:order] = order + end + + color_options = [ + [:auto, :auto], + ["-", false], + ["no", false], + ["false", false], + ["+", true], + ["yes", true], + ["true", true], + ] + parser.on("--[no-]color=[auto]", + color_options, + "Enable color output", + "(#{@options[:color]})") do |color| + if color.nil? + @options[:color] = true + else + @options[:color] = color + end + end + + parser.on("--output=PATH", + "Output to PATH.", + "'-' PATH means standard output.", + "(#{@options[:output]})") do |output| + @options[:output] = output + end + + parser.on("--slow-operation-threshold=THRESHOLD", + Float, + "Use THRESHOLD seconds to detect slow operations.", + "(#{@options[:slow_operation_threshold]})") do |threshold| + @options[:slow_operation_threshold] = threshold + end + + parser.on("--slow-response-threshold=THRESHOLD", + Float, + "Use THRESHOLD seconds to detect slow operations.", + "(#{@options[:sloq_response_threshold]})") do |threshold| + @options[:sloq_response_threshold] = threshold + end + + available_reporters = ["console", "json", "html"] + parser.on("--reporter=REPORTER", + available_reporters, + "Reports statistics by REPORTER.", + "available values: [#{available_reporters.join(', ')}]", + "(#{@options[:reporter]})") do |reporter| + @options[:reporter] = reporter + end + + parser.on("--[no-]dynamic-sort", + "Sorts dynamically.", + "Memory and CPU usage reduced for large query log.", + "(#{@options[:dynamic_sort]})") do |sort| + @options[:dynamic_sort] = sort + end + + parser.on("--[no-]stream", + "Outputs analyzed query on the fly.", + "NOTE: --n-entries and --order are ignored.", + "(#{@options[:stream]})") do |stream| + @options[:stream] = stream + end + + parser.on("--[no-]report-summary", + "Reports summary at the end.", + "(#{@options[:report_summary]})") do |report_summary| + @options[:report_summary] = report_summary + end + end + + def create_reporter(statistics) + case @options[:reporter] + when "json" + require 'json' + Groonga::QueryLog::JSONReporter.new(statistics) + when "html" + Groonga::QueryLog::HTMLReporter.new(statistics) + else + Groonga::QueryLog::ConsoleReporter.new(statistics) + end + end + + def create_stream_reporter + case @options[:reporter] + when "json" + require 'json' + Groonga::QueryLog::StreamJSONQueryLogReporter.new + when "html" + raise UnsupportedReporter, "HTML reporter doesn't support --stream." + else + Groonga::QueryLog::StreamConsoleQueryLogReporter.new + end + end + end + end + end +end Added: lib/groonga/query-log/reporter.rb (+99 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/reporter.rb 2012-12-12 12:36:34 +0900 (af189e0) @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +module Groonga + module QueryLog + class Reporter + include Enumerable + + attr_reader :output + def initialize(statistics) + @statistics = statistics + @report_summary = true + @output = $stdout + end + + def apply_options(options) + self.output = options[:output] || @output + unless options[:report_summary].nil? + @report_summary = options[:report_summary] + end + end + + def output=(output) + @output = output + @output = $stdout if @output == "-" + end + + def each + @statistics.each do |statistic| + yield statistic + end + end + + def report + setup do + report_summary if @report_summary + report_statistics + end + end + + def report_statistics + each do |statistic| + report_statistic(statistic) + end + end + + private + def setup + setup_output do + start + yield + finish + end + end + + def setup_output + original_output = @output + if****@outpu*****_a?(String) + File.open(@output, "w") do |output| + @output = output + yield(@output) + end + else + yield(@output) + end + ensure + @output = original_output + end + + def write(*args) + @output.write(*args) + end + + def format_time(time) + if time.nil? + "NaN" + else + time.strftime("%Y-%m-%d %H:%M:%S.%u") + end + end + end + end +end Added: lib/groonga/query-log/reporter/console.rb (+289 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/reporter/console.rb 2012-12-12 12:36:34 +0900 (5212540) @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require "groonga/query-log/reporter" + +module Groonga + module QueryLog + class ConsoleReporter < Reporter + class Color + NAMES = ["black", "red", "green", "yellow", + "blue", "magenta", "cyan", "white"] + + attr_reader :name + def initialize(name, options={}) + @name = name + @foreground = options[:foreground] + @foreground = true if****@foreg*****? + @intensity = options[:intensity] + @bold = options[:bold] + @italic = options[:italic] + @underline = options[:underline] + end + + def foreground? + @foreground + end + + def intensity? + @intensity + end + + def bold? + @bold + end + + def italic? + @italic + end + + def underline? + @underline + end + + def ==(other) + self.class === other and + [name, foreground?, intensity?, + bold?, italic?, underline?] == + [other.name, other.foreground?, other.intensity?, + other.bold?, other.italic?, other.underline?] + end + + def sequence + sequence = [] + if @name == "none" + elsif @name == "reset" + sequence << "0" + else + foreground_parameter = foreground? ? 3 : 4 + foreground_parameter += 6 if intensity? + sequence << "#{foreground_parameter}#{NAMES.index(@name)}" + end + sequence << "1" if bold? + sequence << "3" if italic? + sequence << "4" if underline? + sequence + end + + def escape_sequence + "\e[#{sequence.join(';')}m" + end + + def +(other) + MixColor.new([self, other]) + end + end + + class MixColor + attr_reader :colors + def initialize(colors) + @colors = colors + end + + def sequence + @colors.inject([]) do |result, color| + result + color.sequence + end + end + + def escape_sequence + "\e[#{sequence.join(';')}m" + end + + def +(other) + self.class.new([self, other]) + end + + def ==(other) + self.class === other and colors == other.colors + end + end + + def initialize(statistics) + super + @color = :auto + @reset_color = Color.new("reset") + @color_schema = { + :elapsed => {:foreground => :white, :background => :green}, + :time => {:foreground => :white, :background => :cyan}, + :slow => {:foreground => :white, :background => :red}, + } + end + + def apply_options(options) + super + @color = options[:color] || @color + end + + def report_statistics + write("\n") + write("Slow Queries:\n") + super + end + + def report_statistic(statistic) + @index += 1 + write("%*d) %s" % [@digit, @index, format_heading(statistic)]) + report_parameters(statistic) + report_operations(statistic) + end + + def start + @index = 0 + if****@stati*****? + @digit = 1 + else + @digit = Math.log10(@statistics.size).truncate + 1 + end + end + + def finish + end + + private + def setup + super do + setup_color do + yield + end + end + end + + def report_summary + write("Summary:\n") + write(" Threshold:\n") + write(" slow response : #{@statistics.slow_response_threshold}\n") + write(" slow operation : #{@statistics.slow_operation_threshold}\n") + write(" # of responses : #{@statistics.n_responses}\n") + write(" # of slow responses : #{@statistics.n_slow_responses}\n") + write(" responses/sec : #{@statistics.responses_per_second}\n") + write(" start time : #{format_time(@statistics.start_time)}\n") + write(" last time : #{format_time(@statistics.last_time)}\n") + write(" period(sec) : #{@statistics.period}\n") + slow_response_ratio =****@stati*****_response_ratio + write(" slow response ratio : %5.3f%%\n" % slow_response_ratio) + write(" total response time : #{@statistics.total_elapsed}\n") + report_slow_operations + end + + def report_slow_operations + write(" Slow Operations:\n") + total_elapsed_digit = nil + total_elapsed_decimal_digit = 6 + n_operations_digit = nil + @statistics.each_slow_operation do |grouped_operation| + total_elapsed = grouped_operation[:total_elapsed] + total_elapsed_digit ||= Math.log10(total_elapsed).truncate + 1 + n_operations = grouped_operation[:n_operations] + n_operations_digit ||= Math.log10(n_operations).truncate + 1 + parameters = [total_elapsed_digit + 1 + total_elapsed_decimal_digit, + total_elapsed_decimal_digit, + total_elapsed, + grouped_operation[:total_elapsed_ratio], + n_operations_digit, + n_operations, + grouped_operation[:n_operations_ratio], + grouped_operation[:name], + grouped_operation[:context]] + write(" [%*.*f](%5.2f%%) [%*d](%5.2f%%) %9s: %s\n" % parameters) + end + end + + def report_parameters(statistic) + command = statistic.command + write(" name: <#{command.name}>\n") + write(" parameters:\n") + command.arguments.each do |key, value| + write(" <#{key}>: <#{value}>\n") + end + end + + def report_operations(statistic) + statistic.each_operation do |operation| + relative_elapsed_in_seconds = operation[:relative_elapsed_in_seconds] + formatted_elapsed = "%8.8f" % relative_elapsed_in_seconds + if operation[:slow?] + formatted_elapsed = colorize(formatted_elapsed, :slow) + end + operation_report = " %2d) %s: %10s" % [operation[:i] + 1, + formatted_elapsed, + operation[:name]] + if operation[:n_records] + operation_report << "(%6d)" % operation[:n_records] + else + operation_report << "(%6s)" % "" + end + context = operation[:context] + if context + context = colorize(context, :slow) if operation[:slow?] + operation_report << " " << context + end + write("#{operation_report}\n") + end + write("\n") + end + + def guess_color_availability(output) + return false unless output.tty? + case ENV["TERM"] + when /term(?:-color)?\z/, "screen" + true + else + return true if ENV["EMACS"] == "t" + false + end + end + + def setup_color + color = @color + @color = guess_color_availability(@output) if @color == :auto + yield + ensure + @color = color + end + + def format_heading(statistic) + formatted_elapsed = colorize("%8.8f" % statistic.elapsed_in_seconds, + :elapsed) + "[%s-%s (%s)](%d): %s" % [format_time(statistic.start_time), + format_time(statistic.last_time), + formatted_elapsed, + statistic.return_code, + statistic.raw_command] + end + + def format_time(time) + colorize(super, :time) + end + + def colorize(text, schema_name) + return text unless @color + options = @color_schema[schema_name] + color = Color.new("none") + if options[:foreground] + color += Color.new(options[:foreground].to_s, :bold => true) + end + if options[:background] + color += Color.new(options[:background].to_s, :foreground => false) + end + "%s%s%s" % [color.escape_sequence, text, @reset_color.escape_sequence] + end + end + end +end Added: lib/groonga/query-log/reporter/html.rb (+330 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/reporter/html.rb 2012-12-12 12:36:34 +0900 (8d7e49d) @@ -0,0 +1,330 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require "erb" +require "groonga/query-log/reporter" + +module Groonga + module QueryLog + class HTMLReporter < Reporter + include ERB::Util + + def report_statistic(statistic) + write(",") if @index > 0 + write("\n") + write(format_statistic(statistic)) + @index += 1 + end + + def start + write(header) + end + + def finish + write(footer) + end + + def report_summary + summary_html = erb(<<-EOH, __LINE__ + 1, binding) + <h2>Summary</h2> + <div class="summary"> + <%= analyze_parameters %> + <%= metrics %> + <%= slow_operations %> + </div> + EOH + write(summary_html) + end + + def report_statistics + write(statistics_header) + super + write(statistics_footer) + end + + def report_statistic(statistic) + command = statistic.command + statistic_html = erb(<<-EOH, __LINE__ + 1, binding) + <div class="statistic-heading"> + <h3>Command</h3> + <div class="metrics"> + [<%= format_time(statistic.start_time) %> + - + <%= format_time(statistic.last_time) %> + (<%= format_elapsed(statistic.elapsed_in_seconds, + :slow? => statistic.slow?) %>)] + (<%= span({:class => "return-code"}, h(statistic.return_code)) %>) + </div> + <%= div({:class => "raw-command"}, h(statistic.raw_command)) %> + </div> + <div class="statistic-parameters"> + <h3>Parameters</h3> + <dl> + <dt>name</dt> + <dd><%= h(command.name) %></dd> + <% command.arguments.each do |key, value| %> + <dt><%= h(key) %></dt> + <dd><%= h(value) %></dd> + <% end %> + </dl> + </div> + <div class="statistic-operations"> + <h3>Operations</h3> + <ol> + <% statistic.each_operation do |operation| %> + <li> + <%= format_elapsed(operation[:relative_elapsed_in_seconds], + :slow? => operation[:slow?]) %>: + <%= span({:class => "name"}, h(operation[:name])) %>: + <%= span({:class => "context"}, h(operation[:context])) %> + </li> + <% end %> + </ol> + </div> + EOH + write(statistic_html) + end + + private + def erb(content, line, _binding=nil) + _erb = ERB.new(content, nil, "<>") + eval(_erb.src, _binding || binding, __FILE__, line) + end + + def header + erb(<<-EOH, __LINE__ + 1) + <html> + <head> + <title>groonga query analyzer</title> + <style> + table, + table tr, + table tr th, + table tr td + { + border: 1px solid black; + } + + span.slow + { + color: red; + } + + div.parameters + { + float: left; + padding: 2em; + } + + div.parameters h3 + { + text-align: center; + } + + div.parameters table + { + margin-right: auto; + margin-left: auto; + } + + div.statistics + { + clear: both; + } + + td.elapsed, + td.ratio, + td.n + { + text-align: right; + } + + td.name + { + text-align: center; + } + </style> + </head> + <body> + <h1>groonga query analyzer</h1> + EOH + end + + def footer + erb(<<-EOH, __LINE__ + 1) + </body> + </html> + EOH + end + + def statistics_header + erb(<<-EOH, __LINE__ + 1) + <h2>Slow Queries</h2> + <div> + EOH + end + + def statistics_footer + erb(<<-EOH, __LINE__ + 1) + </div> + EOH + end + + def analyze_parameters + erb(<<-EOH, __LINE__ + 1) + <div class="parameters"> + <h3>Analyze Parameters</h3> + <table> + <tr><th>Name</th><th>Value</th></tr> + <tr> + <th>Slow response threshold</th> + <td><%= h(@statistics.slow_response_threshold) %>sec</td> + </tr> + <tr> + <th>Slow operation threshold</th> + <td><%= h(@statistics.slow_operation_threshold) %>sec</td> + </tr> + </table> + </div> + EOH + end + + def metrics + erb(<<-EOH, __LINE__ + 1) + <div class="parameters"> + <h3>Metrics</h3> + <table> + <tr><th>Name</th><th>Value</th></tr> + <tr> + <th># of responses</th> + <td><%= h(@statistics.n_responses) %></td> + </tr> + <tr> + <th># of slow responses</th> + <td><%= h(@statistics.n_slow_responses) %></td> + </tr> + <tr> + <th>responses/sec</th> + <td><%= h(@statistics.responses_per_second) %></td> + </tr> + <tr> + <th>start time</th> + <td><%= format_time(@statistics.start_time) %></td> + </tr> + <tr> + <th>last time</th> + <td><%= format_time(@statistics.last_time) %></td> + </tr> + <tr> + <th>period</th> + <td><%= h(@statistics.period) %>sec</td> + </tr> + <tr> + <th>slow response ratio</th> + <td><%= h(@statistics.slow_response_ratio) %>%</td> + </tr> + <tr> + <th>total response time</th> + <td><%= h(@statistics.total_elapsed) %>sec</td> + </tr> + </table> + </div> + EOH + end + + def slow_operations + erb(<<-EOH, __LINE__ + 1) + <div class="statistics"> + <h3>Slow Operations</h3> + <table class="slow-operations"> + <tr> + <th>total elapsed(sec)</th> + <th>total elapsed(%)</th> + <th># of operations</th> + <th># of operations(%)</th> + <th>operation name</th> + <th>context</th> + </tr> + <% @statistics.each_slow_operation do |grouped_operation| %> + <tr> + <td class="elapsed"> + <%= format_elapsed(grouped_operation[:total_elapsed]) %> + </td> + <td class="ratio"> + <%= format_ratio(grouped_operation[:total_elapsed_ratio]) %> + </td> + <td class="n"> + <%= h(grouped_operation[:n_operations]) %> + </td> + <td class="ratio"> + <%= format_ratio(grouped_operation[:n_operations_ratio]) %> + </td> + <td class="name"><%= h(grouped_operation[:name]) %></td> + <td class="context"> + <%= format_context(grouped_operation[:context]) %> + </td> + </tr> + <% end %> + </table> + </div> + EOH + end + + def format_time(time) + span({:class => "time"}, h(super)) + end + + def format_elapsed(elapsed, options={}) + formatted_elapsed = span({:class => "elapsed"}, h("%8.8f" % elapsed)) + if options[:slow?] + formatted_elapsed = span({:class => "slow"}, formatted_elapsed) + end + formatted_elapsed + end + + def format_ratio(ratio) + h("%5.2f%%" % ratio) + end + + def format_context(context) + h(context).gsub(/,/, ",<wbr />") + end + + def tag(name, attributes, content) + html = "<#{name}" + html_attributes = attributes.collect do |key, value| + "#{h(key)}=\"#{h(value)}\"" + end + html << " #{html_attributes.join(' ')}" unless attributes.empty? + html << ">" + html << content + html << "</#{name}>" + html + end + + def span(attributes, content) + tag("span", attributes, content) + end + + def div(attributes, content) + tag("div", attributes, content) + end + end + end +end Added: lib/groonga/query-log/reporter/json.rb (+76 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/reporter/json.rb 2012-12-12 12:36:34 +0900 (0df77db) @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require "groonga/query-log/reporter" + +module Groonga + module QueryLog + class JSONReporter < Reporter + def report_statistic(statistic) + write(",") if @index > 0 + write("\n") + write(format_statistic(statistic)) + @index += 1 + end + + def start + @index = 0 + write("[") + end + + def finish + write("\n") + write("]\n") + end + + def report_summary + # TODO + end + + private + def format_statistic(statistic) + data = { + "start_time" => statistic.start_time.to_i, + "last_time" => statistic.last_time.to_i, + "elapsed" => statistic.elapsed_in_seconds, + "return_code" => statistic.return_code, + } + command = statistic.command + arguments = command.arguments.collect do |key, value| + {"key" => key, "value" => value} + end + data["command"] = { + "raw" => statistic.raw_command, + "name" => command.name, + "parameters" => arguments, + } + operations = [] + statistic.each_operation do |operation| + operation_data = {} + operation_data["name"] = operation[:name] + operation_data["relative_elapsed"] = operation[:relative_elapsed_in_seconds] + operation_data["context"] = operation[:context] + operations << operation_data + end + data["operations"] = operations + JSON.generate(data) + end + end + end +end Added: lib/groonga/query-log/sized-grouped-operations.rb (+82 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/sized-grouped-operations.rb 2012-12-12 12:36:34 +0900 (f8bad3f) @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +module Groonga + module QueryLog + class SizedGroupedOperations < Array + def initialize + @max_size = 10 + @sorter = create_sorter + end + + def apply_options(options) + @max_size = options[:n_entries] + end + + def each + i = 0 + super do |grouped_operation| + break if i >= @max_size + i += 1 + yield(grouped_operation) + end + end + + def <<(operation) + each do |grouped_operation| + if grouped_operation[:name] == operation[:name] and + grouped_operation[:context] == operation[:context] + elapsed = operation[:relative_elapsed_in_seconds] + grouped_operation[:total_elapsed] += elapsed + grouped_operation[:n_operations] += 1 + replace(sort_by(&@sorter)) + return self + end + end + + grouped_operation = { + :name => operation[:name], + :context => operation[:context], + :n_operations => 1, + :total_elapsed => operation[:relative_elapsed_in_seconds], + } + buffer_size = @max_size * 100 + if size < buffer_size + super(grouped_operation) + replace(sort_by(&@sorter)) + else + if****@sorte*****(grouped_operation) < @sorter.call(last) + super(grouped_operation) + sorted_operations = sort_by(&@sorter) + sorted_operations.pop + replace(sorted_operations) + end + end + self + end + + private + def create_sorter + lambda do |grouped_operation| + -grouped_operation[:total_elapsed] + end + end + end + end +end Added: lib/groonga/query-log/sized-statistics.rb (+165 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/sized-statistics.rb 2012-12-12 12:36:34 +0900 (b6f0ea8) @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +module Groonga + module QueryLog + class SizedStatistics < Array + attr_reader :n_responses, :n_slow_responses, :n_slow_operations + attr_reader :slow_operations, :total_elapsed + attr_reader :start_time, :last_time + attr_accessor :slow_operation_threshold, :slow_response_threshold + def initialize + @max_size = 10 + self.order = "-elapsed" + @slow_operation_threshold = 0.1 + @slow_response_threshold = 0.2 + @start_time = nil + @last_time = nil + @n_responses = 0 + @n_slow_responses = 0 + @n_slow_operations = 0 + @slow_operations = SizedGroupedOperations.new + @total_elapsed = 0 + @collect_slow_statistics = true + end + + def order=(new_order) + @order = new_order + @sorter = create_sorter + end + + def apply_options(options) + @max_size = options[:n_entries] || @max_size + self.order = options[:order] || @order + @slow_operation_threshold = + options[:slow_operation_threshold] || @slow_operation_threshold + @slow_response_threshold = + options[:slow_response_threshold] || @slow_response_threshold + unless options[:report_summary].nil? + @collect_slow_statistics = options[:report_summary] + end + @slow_operations.apply_options(options) + end + + def <<(statistic) + update_statistic(statistic) + if size < @max_size + super(statistic) + replace(self) + else + if****@sorte*****(statistic) < @sorter.call(last) + super(statistic) + replace(self) + end + end + self + end + + def replace(other) + sorted_other = other.sort_by(&@sorter) + if sorted_other.size > @max_size + super(sorted_other[0, @max_size]) + else + super(sorted_other) + end + end + + def responses_per_second + _period = period + if _period.zero? + 0 + else + @n_responses.to_f / _period + end + end + + def slow_response_ratio + if @n_responses.zero? + 0 + else + (@n_slow_responses.to_f / @n_responses) * 100 + end + end + + def period + if @start_time and @last_time + @last_time - @start_time + else + 0 + end + end + + def each_slow_operation + @slow_operations.each do |grouped_operation| + total_elapsed = grouped_operation[:total_elapsed] + n_operations = grouped_operation[:n_operations] + ratios = { + :total_elapsed_ratio => total_elapsed / @total_elapsed * 100, + :n_operations_ratio => n_operations / @n_slow_operations.to_f * 100, + } + yield(grouped_operation.merge(ratios)) + end + end + + private + def create_sorter + case @order + when "-elapsed" + lambda do |statistic| + -statistic.elapsed + end + when "elapsed" + lambda do |statistic| + statistic.elapsed + end + when "-start-time" + lambda do |statistic| + -statistic.start_time + end + else + lambda do |statistic| + statistic.start_time + end + end + end + + def update_statistic(statistic) + statistic.slow_response_threshold = @slow_response_threshold + statistic.slow_operation_threshold = @slow_operation_threshold + @start_time ||= statistic.start_time + @start_time = [@start_time, statistic.start_time].min + @last_time ||= statistic.last_time + @last_time = [@last_time, statistic.last_time].max + @n_responses += 1 + @total_elapsed += statistic.elapsed_in_seconds + return unless @collect_slow_statistics + if statistic.slow? + @n_slow_responses += 1 + if statistic.select_command? + statistic.each_operation do |operation| + next unless operation[:slow?] + @n_slow_operations += 1 + @slow_operations << operation + end + end + end + end + end + end +end Copied: lib/groonga/query-log/streamer.rb (+23 -3) 62% =================================================================== --- lib/groonga/query-log.rb 2012-11-27 13:54:15 +0900 (9d93c69) +++ lib/groonga/query-log/streamer.rb 2012-12-12 12:36:34 +0900 (650c422) @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2011-2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2012 Haruka Yoshihara <yoshihara �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,5 +17,24 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -require "groonga/query-log/version" -require "groonga/query-log/parser" +module Groonga + module QueryLog + class Streamer + def initialize(reporter) + @reporter = reporter + end + + def start + @reporter.start + end + + def <<(statistic) + @reporter.report_statistic(statistic) + end + + def finish + @reporter.finish + end + end + end +end -------------- next part -------------- HTML����������������������������...Download