null+****@clear*****
null+****@clear*****
2011年 6月 2日 (木) 16:20:24 JST
Kouhei Sutou 2011-06-02 07:20:24 +0000 (Thu, 02 Jun 2011)
New Revision: 455a1020dd64f7365ddafe56be76c17ac760d287
Log:
[query-log][analyzer] reduce memory usage. refs #939
Modified files:
test/unit/tools/test-query-log-analyzer.rb
tools/groonga-query-log-analyzer.rb
Modified: test/unit/tools/test-query-log-analyzer.rb (+5 -6)
===================================================================
--- test/unit/tools/test-query-log-analyzer.rb 2011-06-02 06:43:09 +0000 (057ccab)
+++ test/unit/tools/test-query-log-analyzer.rb 2011-06-02 07:20:24 +0000 (be53d1c)
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2009-2011 Kouhei Sutou <kou****@clear*****>
+# Copyright (C) 2011 Kouhei Sutou <kou****@clear*****>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -80,10 +80,6 @@ module QueryLogAalyzerTest
end
class StatisticStepParseTest < Test::Unit::TestCase
- def setup
- @parser = GroongaQueryLogAnaylzer::QueryLogParser.new
- end
-
def test_context
steps = statistics.first.steps.collect do |step|
[step[:name], step[:context]]
@@ -126,7 +122,10 @@ EOL
end
def statistics
- @parser.parse(StringIO.new(log))
+ statistics = GroongaQueryLogAnaylzer::SizedStatistics.new(100, "-elapsed")
+ parser = GroongaQueryLogAnaylzer::QueryLogParser.new(statistics)
+ parser.parse(StringIO.new(log))
+ statistics
end
end
end
Modified: tools/groonga-query-log-analyzer.rb (+50 -54)
===================================================================
--- tools/groonga-query-log-analyzer.rb 2011-06-02 06:43:09 +0000 (9a6bd83)
+++ tools/groonga-query-log-analyzer.rb 2011-06-02 07:20:24 +0000 (8baf6bf)
@@ -13,20 +13,15 @@ class GroongaQueryLogAnaylzer
def run(argv=nil)
log_paths = @option_parser.parse!(argv || ARGV)
- parser = QueryLogParser.new
- threads = []
+ statistics = SizedStatistics.new(@options[:n_entries], @options[:order])
+ parser = QueryLogParser.new(statistics)
log_paths.each do |log_path|
- threads << Thread.new do
- File.open(log_path) do |log|
- parser.parse(log)
- end
+ File.open(log_path) do |log|
+ parser.parse(log)
end
end
- threads.each do |thread|
- thread.join
- end
- reporter = create_reporter(parser.statistics)
+ reporter = create_reporter(statistics)
reporter.apply_options(@options)
reporter.report
end
@@ -273,20 +268,59 @@ class GroongaQueryLogAnaylzer
end
class SizedStatistics < Array
- def initialize(size)
+ def initialize(size, order)
@size = size
+ @order = order
+ @sorter = sorter
+ end
+
+ def <<(other)
+ if size < @size - 1
+ super(other)
+ elsif size == @size - 1
+ super(other)
+ sort_by!(&@sorter)
+ else
+ if****@sorte*****(other) < @sorter.call(last)
+ super(other)
+ sort_by!(&@sorter)
+ pop
+ end
+ end
+ self
+ end
+
+ private
+ def sorter
+ case @order
+ when "elapsed"
+ lambda do |statistic|
+ -statistic.elapsed
+ end
+ when "-elapsed"
+ lambda do |statistic|
+ -statistic.elapsed
+ end
+ when "-start-time"
+ lambda do |statistic|
+ -statistic.start_time
+ end
+ else
+ lambda do |statistic|
+ statistic.start_time
+ end
+ end
end
end
class QueryLogParser
attr_reader :statistics
- def initialize
+ def initialize(statistics)
@mutex = Mutex.new
- @statistics = []
+ @statistics = statistics
end
def parse(input)
- statistics = []
current_statistics = {}
input.each_line do |line|
case line
@@ -299,12 +333,9 @@ class GroongaQueryLogAnaylzer
time_stamp = Time.local(year, month, day, hour, minutes, seconds,
micro_seconds)
parse_line(statistics, current_statistics,
- time_stamp, context_id, type, rest)
+ time_stamp, context_id, type, rest)
end
end
- @mutex.synchronize do
- @statistics.concat(statistics)
- end
end
private
@@ -344,64 +375,29 @@ class GroongaQueryLogAnaylzer
attr_accessor :n_entries, :slow_threshold
def initialize(statistics)
@statistics = statistics
- @order = "-elapsed"
@n_entries = 10
@slow_threshold = 0.05
@output = $stdout
- @sorted_statistics = nil
end
def apply_options(options)
- self.order = options[:order] || @order
self.n_entries = options[:n_entries] || @n_entries
self.slow_threshold = options[:slow_threshold] || @slow_threshold
self.output = options[:output] || @output
end
- def order=(order)
- return if @order == order
- @order = order
- @sorted_statistics = nil
- end
-
def output=(output)
@output = output
@output = $stdout if @output == "-"
end
- def sorted_statistics
- @sorted_statistics ||=****@stati*****_by(&sorter)
- end
-
def each
- sorted_statistics.each_with_index do |statistic, i|
- break if i >= @n_entries
+ @statistics.each do |statistic|
yield statistic
end
end
private
- def sorter
- case @order
- when "elapsed"
- lambda do |statistic|
- -statistic.elapsed
- end
- when "-elapsed"
- lambda do |statistic|
- -statistic.elapsed
- end
- when "-start-time"
- lambda do |statistic|
- -statistic.start_time
- end
- else
- lambda do |statistic|
- statistic.start_time
- end
- end
- end
-
def slow?(elapsed)
elapsed >= @slow_threshold
end