Kouhei Sutou 2019-05-09 18:54:34 +0900 (Thu, 09 May 2019) Revision: aca507441b5ba74b1fe7c8fb87af8c6de5286556 https://github.com/groonga/grntest/commit/aca507441b5ba74b1fe7c8fb87af8c6de5286556 Message: Use log parse gems Removed files: lib/grntest/log-entry.rb lib/grntest/log-parser.rb lib/grntest/query-log-entry.rb lib/grntest/query-log-parser.rb Modified files: grntest.gemspec lib/grntest/executors/base-executor.rb lib/grntest/test-runner.rb Modified: grntest.gemspec (+6 -4) =================================================================== --- grntest.gemspec 2018-11-26 17:15:55 +0900 (3d554db) +++ grntest.gemspec 2019-05-09 18:54:34 +0900 (b7ca420) @@ -1,6 +1,6 @@ -# -*- mode: ruby; coding: utf-8 -*- +# -*- ruby -*- # -# Copyright (C) 2012-2013 Kouhei Sutou <kou****@clear*****> +# Copyright (C) 2012-2019 Kouhei Sutou <kou****@clear*****> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,10 +43,12 @@ Gem::Specification.new do |spec| spec.executables = Dir.glob("*") end - spec.add_runtime_dependency("json") - spec.add_runtime_dependency("msgpack") spec.add_runtime_dependency("diff-lcs") spec.add_runtime_dependency("groonga-command-parser") + spec.add_runtime_dependency("groonga-log") + spec.add_runtime_dependency("groonga-query-log", ">= 1.4.1") + spec.add_runtime_dependency("json") + spec.add_runtime_dependency("msgpack") spec.add_development_dependency("bundler") spec.add_development_dependency("rake") Modified: lib/grntest/executors/base-executor.rb (+59 -12) =================================================================== --- lib/grntest/executors/base-executor.rb 2018-11-26 17:15:55 +0900 (014a38e) +++ lib/grntest/executors/base-executor.rb 2019-05-09 18:54:34 +0900 (cc1a82d) @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2018 Kouhei Sutou <kou****@clear*****> +# Copyright (C) 2012-2019 Kouhei Sutou <kou****@clear*****> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,11 +17,11 @@ require "pathname" require "fileutils" require "shellwords" +require "groonga-log" +require "groonga-query-log" require "groonga/command/parser" require "grntest/error" -require "grntest/log-parser" -require "grntest/query-log-parser" require "grntest/execution-context" require "grntest/response-parser" require "grntest/template-evaluator" @@ -277,7 +277,11 @@ module Grntest end def normalize_log_level(level) - level[0] + case level + when "info" + level = "information" + end + level.to_sym end def execute_directive_sleep(line, content, options) @@ -424,10 +428,10 @@ module Grntest def extract_important_messages(log) important_messages = [] - parser = LogParser.new + parser = GroongaLog::Parser.new in_crash = false parser.parse(log) do |entry| - if entry.log_level == "C" + if entry.log_level == :critical case entry.message when "-- CRASHED!!! --" in_crash = true @@ -441,7 +445,8 @@ module Grntest next if !in_crash and backtrace_log_message?(entry.message) end next if thread_log_message?(entry.message) - important_messages << "\#|#{entry.log_level}| #{entry.message}" + formatted_log_level = format_log_level(entry.log_level) + important_messages << "\#|#{formatted_log_level}| #{entry.message}" end important_messages.join("\n") end @@ -462,8 +467,37 @@ module Grntest end def important_log_level?(log_level) - ["E", "A", "C", "e", "w"].include?(log_level) or - @custom_important_log_levels.include?(log_level) + case log_level + when :emergency, :alert, :critical, :error, :warning + true + when *@custom_important_log_levels + true + else + false + end + end + + def format_log_level(log_level) + case log_level + when :emergency + "E" + when :alert + "A" + when :critical + "C" + when :error + "e" + when :warning + "w" + when :notice + "n" + when :information + "i" + when :debug + "d" + when :dump + "-" + end end def backtrace_log_message?(message) @@ -539,9 +573,22 @@ module Grntest def log_query_log_content(content) return unles****@conte*****_query_log? - parser = QueryLogParser.new - parser.parse(content) do |entry| - log_query("\##{entry.mark}#{entry.message}") + parser = GroongaQueryLog::Parser.new + parser.parse(content) do |statistic| + relative_elapsed_time = "000000000000000" + command = statistic.command + if command + command[:output_type] = nil if command.output_type == :json + log_query("\#>#{command.to_command_format}") + end + statistic.each_operation do |operation| + message = operation[:raw_message] + if operation[:name] == "cache" + message = message.gsub(/\(\d+\)/, "(0)") + end + log_query("\#:#{relative_elapsed_time} #{message}") + end + log_query("\#<#{relative_elapsed_time} rc=#{statistic.return_code}") end end Deleted: lib/grntest/log-entry.rb (+0 -19) 100644 =================================================================== --- lib/grntest/log-entry.rb 2018-11-26 17:15:55 +0900 (fe810ef) +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (C) 2014 Kouhei Sutou <kou****@clear*****> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -module Grntest - class LogEntry < Struct.new(:timestamp, :log_level, :message) - end -end Deleted: lib/grntest/log-parser.rb (+0 -48) 100644 =================================================================== --- lib/grntest/log-parser.rb 2018-11-26 17:15:55 +0900 (861ce72) +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 2014 Kouhei Sutou <kou****@clear*****> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -require "grntest/log-entry" - -module Grntest - class LogParser - def parse(log) - timestamp = nil - log_level = nil - message = nil - emit_entry = lambda do - if timestamp - entry = LogEntry.new(timestamp, log_level, message.chomp) - yield(entry) - end - end - log.each_line do |line| - case line - when /\A(\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2}\.\d+)\| - ([a-zA-Z])\| - (?:\d+:)? - \s*/x - emit_entry.call - timestamp = $1 - log_level = $2 - message = $POSTMATCH - else - message ||= "" - message << line - end - end - emit_entry.call - end - end -end Deleted: lib/grntest/query-log-entry.rb (+0 -19) 100644 =================================================================== --- lib/grntest/query-log-entry.rb 2018-11-26 17:15:55 +0900 (2765e1a) +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (C) 2015 Kouhei Sutou <kou****@clear*****> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -module Grntest - class QueryLogEntry < Struct.new(:timestamp, :mark, :message) - end -end Deleted: lib/grntest/query-log-parser.rb (+0 -71) 100644 =================================================================== --- lib/grntest/query-log-parser.rb 2018-11-26 17:15:55 +0900 (ed1aebf) +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (C) 2015-2017 Kouhei Sutou <kou****@clear*****> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -require "grntest/query-log-entry" - -module Grntest - class QueryLogParser - def parse(log) - log.each_line do |line| - case line - when /\A(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)\|.+?\|(.)/ - timestamp = $1 - mark = $2 - message = normalize_message(mark, $POSTMATCH.chomp) - entry = QueryLogEntry.new(timestamp, mark, message) - yield(entry) - end - end - end - - private - def normalize_message(mark, message) - case mark - when ">" - message = normalize_command(message) - else - message = normalize_elapsed_time(message) - message = normalize_cache_content(message) - end - message - end - - def normalize_command(message) - command = nil - Groonga::Command::Parser.parse(message) do |status, *args| - case status - when :on_command - command = args[0] - when :on_load_start - command = args[0] - end - end - if command.output_type == :json - command[:output_type] = nil - end - command.to_command_format - end - - def normalize_elapsed_time(message) - message.gsub(/\A\d{15} /, "0" * 15 + " ") - end - - def normalize_cache_content(message) - message.gsub(/\A(0{15}) (cache\()\d+(\))\z/) do - "#{$1} #{$2}0#{$3}" - end - end - end -end Modified: lib/grntest/test-runner.rb (+3 -3) =================================================================== --- lib/grntest/test-runner.rb 2018-11-26 17:15:55 +0900 (93acf9e) +++ lib/grntest/test-runner.rb 2019-05-09 18:54:34 +0900 (e557326) @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2018 Kouhei Sutou <kou****@clear*****> +# Copyright (C) 2012-2019 Kouhei Sutou <kou****@clear*****> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,11 +19,11 @@ require "tempfile" require "timeout" require "socket" +require "groonga-log" require "json" require "grntest/platform" require "grntest/error" -require "grntest/log-parser" require "grntest/executors" require "grntest/base-result" @@ -808,7 +808,7 @@ http { end def check_memory_leak(context) - parser = LogParser.new + parser = GroongaLog::Parser.new parser.parse(context.log) do |entry| next unless /^grn_fin \((\d+)\)$/ =~ entry.message n_leaked_objects = $1.to_i -------------- next part -------------- An HTML attachment was scrubbed... URL: <https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20190509/be869a1e/attachment-0001.html>
